nswtopo 2.0.0.pre.beta1

data/lib/nswtopo/gis/arcgis_server/connection.rb

@@ -0,0 +1,52 @@
+module NSWTopo
+  module ArcGISServer
+    class Connection
+      def initialize(http, service_path)
+        @http, @service_path, @headers = http, service_path, { "User-Agent" => "Ruby/#{RUBY_VERSION}", "Referer" => "%s://%s" % [http.use_ssl? ? "https" : "http", http.address] }
+        http.max_retries = 0
+      end
+
+      def repeatedly_request(request)
+        intervals ||= 5.times.map(&1.4142.method(:**))
+        response = @http.request(request)
+        response.error! unless Net::HTTPSuccess === response
+        yield response
+      rescue *ERRORS, Error => error
+        interval = intervals.shift
+        interval ? sleep(interval) : raise(error)
+        retry
+      end
+
+      def get(relative_path, **query, &block)
+        path = Pathname(@service_path).join(relative_path).to_s
+        path << ?? << URI.encode_www_form(query) unless query.empty?
+        request = Net::HTTP::Get.new(path, @headers)
+        repeatedly_request(request, &block)
+      end
+
+      def post(relative_path, **query, &block)
+        path = Pathname(@service_path).join(relative_path).to_s
+        request = Net::HTTP::Post.new(path, @headers)
+        request.body = URI.encode_www_form(query)
+        repeatedly_request(request, &block)
+      end
+
+      def process_json(response)
+        JSON.parse(response.body).tap do |result|
+          # raise Error, result["error"].values_at("message", "details").compact.join(?\n) if result["error"]
+          raise Error, result["error"]["message"] if result["error"]
+        end
+      rescue JSON::ParserError
+        raise Error, "unexpected ArcGIS response format"
+      end
+
+      def get_json(relative_path = "", **query)
+        get relative_path, query.merge(f: "json"), &method(:process_json)
+      end
+
+      def post_json(relative_path = "", **query)
+        post relative_path, query.merge(f: "json"), &method(:process_json)
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/arcgis_server.rb

@@ -0,0 +1,155 @@
+require_relative 'arcgis_server/connection'
+
+module NSWTopo
+  module ArcGISServer
+    Error = Class.new RuntimeError
+    ERRORS = [Timeout::Error, Errno::ENETUNREACH, Errno::ETIMEDOUT, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError]
+    SERVICE = /^(?:MapServer|FeatureServer|ImageServer)$/
+
+    def self.check_uri(url)
+      uri = URI.parse url
+      return unless URI::HTTP === uri
+      instance, (id, *) = uri.path.split(?/).slice_after(SERVICE).take(2)
+      return unless instance.last =~ SERVICE
+      return unless !id || id =~ /^\d+$/
+      return uri, instance.join(?/), id
+    rescue URI::Error
+    end
+
+    def self.===(string)
+      uri, service_path, id = check_uri string
+      uri != nil
+    end
+
+    def self.start(url, &block)
+      uri, service_path, id = check_uri url
+      raise "invalid ArcGIS server URL: %s" % url unless uri
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", read_timeout: 600) do |http|
+        connection = Connection.new http, service_path
+        service = connection.get_json
+        projection = case
+        when wkt  = service.dig("spatialReference", "wkt") then Projection.new(wkt)
+        when wkid = service.dig("spatialReference", "latestWkid") then Projection.new("EPSG:#{wkid}")
+        when wkid = service.dig("spatialReference", "wkid") then Projection.new("EPSG:#{wkid == 102100 ? 3857 : wkid}")
+        else raise Error, "no spatial reference found: #{uri}"
+        end
+        yield connection, service, projection, *id
+      end
+    rescue *ERRORS => error
+      raise Error, error.message
+    end
+
+    def arcgis_layer(url, where: nil, layer: nil, per_page: nil, margin: {})
+      ArcGISServer.start url do |connection, service, projection, id|
+        id = service["layers"].find do |info|
+          layer.to_s == info["name"]
+        end&.dig("id") if layer
+        id ? nil : layer ? raise("no such ArcGIS layer: %s" % layer) : raise("not an ArcGIS layer url: %s" % url)
+
+        layer = connection.get_json id.to_s
+        query_path = "#{id}/query"
+        max_record_count, fields, types, type_id_field, geometry_type, capabilities = layer.values_at "maxRecordCount", "fields", "types", "typeIdField", "geometryType", "capabilities"
+        raise Error, "no query capability available: #{url}" unless capabilities =~ /Query|Data/
+
+        if type_id_field && types
+          type_id_field = fields.find do |field|
+            field.values_at("alias", "name").include? type_id_field
+          end&.fetch("name")
+          type_values = types.map do |type|
+            type.values_at "id", "name"
+          end.to_h
+          subtype_coded_values = types.map do |type|
+            type.values_at "id", "domains"
+          end.map do |id, domains|
+            coded_values = domains.map do |name, domain|
+              [name, domain["codedValues"]]
+            end.select(&:last).map do |name, pairs|
+              values = pairs.map do |pair|
+                pair.values_at "code", "name"
+              end.to_h
+              [name, values]
+            end.to_h
+            [id, coded_values]
+          end.to_h
+        end
+
+        coded_values = fields.map do |field|
+          [field["name"], field.dig("domain", "codedValues")]
+        end.select(&:last).map do |name, pairs|
+          values = pairs.map do |pair|
+            pair.values_at "code", "name"
+          end.to_h
+          [name, values]
+        end.to_h
+
+        geometry = { rings: @map.bounding_box(margin).reproject_to(projection).coordinates.map(&:reverse) }.to_json
+        where = Array(where).map { |clause| "(#{clause})"}.join " AND "
+        query = { geometry: geometry, geometryType: "esriGeometryPolygon", returnIdsOnly: true, where: where }
+
+        object_ids = connection.get_json(query_path, query)["objectIds"]
+        next GeoJSON::Collection.new projection unless object_ids
+
+        features = Enumerator.new do |yielder|
+          per_page, total = [*per_page, *max_record_count, 500].min, object_ids.length
+          while object_ids.any?
+            yield total - object_ids.length, total if block_given? && total > 0
+            yielder << begin
+              connection.get_json query_path, outFields: ?*, objectIds: object_ids.take(per_page).join(?,)
+            rescue Error => error
+              (per_page /= 2) > 0 ? retry : raise(error)
+            end
+            object_ids.shift per_page
+          end
+        end.inject [] do |features, page|
+          features += page["features"]
+        end.map do |feature|
+          next unless geometry = feature["geometry"]
+          attributes = feature.fetch "attributes", {}
+
+          values = attributes.map do |name, value|
+            case
+            when type_id_field == name
+              type_values[value]
+            when decode = subtype_coded_values&.dig(attributes[type_id_field], name)
+              decode[value]
+            when decode = coded_values.dig(name)
+              decode[value]
+            when %w[null Null NULL <null> <Null> <NULL>].include?(value)
+              nil
+            else value
+            end
+          end
+          attributes = attributes.keys.zip(values).to_h
+
+          case geometry_type
+          when "esriGeometryPoint"
+            point = geometry.values_at "x", "y"
+            next unless point.all?
+            next GeoJSON::Point.new point, attributes
+          when "esriGeometryMultipoint"
+            points = geometry["points"]
+            next unless points&.any?
+            next GeoJSON::MultiPoint.new points.transpose.take(2).transpose, attributes
+          when "esriGeometryPolyline"
+            raise Error, "ArcGIS curve geometries not supported" if geometry.key? "curvePaths"
+            paths = geometry["paths"]
+            next unless paths&.any?
+            next GeoJSON::LineString.new paths[0], attributes if paths.one?
+            next GeoJSON::MultiLineString.new paths, attributes
+          when "esriGeometryPolygon"
+            raise Error, "ArcGIS curve geometries not supported" if geometry.key? "curveRings"
+            rings = geometry["rings"]
+            next unless rings&.any?
+            rings.each(&:reverse!) unless rings[0].anticlockwise?
+            next GeoJSON::Polygon.new rings, attributes if rings.one?
+            next GeoJSON::MultiPolygon.new rings.slice_before(&:anticlockwise?).to_a, attributes
+          else
+            raise Error, "unsupported ArcGIS geometry type: #{geometry_type}"
+          end
+        end.compact
+
+        GeoJSON::Collection.new projection, features
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/dem.rb

@@ -0,0 +1,70 @@
+module NSWTopo
+  module DEM
+    include GDALGlob, Log
+
+    def get_dem(temp_dir, dem_path)
+      txt_path = temp_dir / "dem.txt"
+      vrt_path = temp_dir / "dem.vrt"
+      te = @map.bounds(margin: margin).transpose.flatten
+
+      Dir.chdir(@source ? @source.parent : Pathname.pwd) do
+        log_update "%s: examining DEM" % @name
+        gdal_rasters @path do |index, total|
+          log_update "%s: examining DEM file %i of %i" % [@name, index, total]
+        end
+      end.tap do |rasters|
+        raise "no elevation data found at specified path" if rasters.none?
+        log_update "%s: extracting DEM raster" % @name
+      end.group_by do |path, info|
+        Projection.new info.dig("coordinateSystem", "wkt")
+      end.map.with_index do |(projection, rasters), index|
+        raise "DEM data not in planar projection with metre units" unless projection.proj4.split(?\s).any?("+units=m")
+
+        paths, resolutions = rasters.map do |path, info|
+          [path, info["geoTransform"].values_at(1, 2).norm]
+        end.sort_by(&:last).transpose
+
+        txt_path.write paths.reverse.join(?\n)
+        @resolution ||= resolutions.first
+
+        indexed_dem_path = temp_dir / "dem.#{index}.tif"
+        OS.gdalbuildvrt "-overwrite", "-input_file_list", txt_path, vrt_path
+        OS.gdalwarp "-t_srs", @map.projection, "-te", *te, "-tr", @resolution, @resolution, "-r", "bilinear", vrt_path, indexed_dem_path
+        indexed_dem_path
+      end.tap do |dem_paths|
+        txt_path.write dem_paths.join(?\n)
+        OS.gdalbuildvrt "-overwrite", "-input_file_list", txt_path, vrt_path
+        OS.gdal_translate vrt_path, dem_path
+      end
+    end
+
+    def blur_dem(dem_path, blur_path)
+      sigma = @smooth * @map.scale / 1000.0
+      half = (3 * sigma / @resolution).ceil
+
+      coeffs = (-half..half).map do |n|
+        n * @resolution / sigma
+      end.map do |x|
+        Math::exp(-x**2)
+      end
+
+      vrt = OS.gdalbuildvrt "/vsistdout/", dem_path
+      xml = REXML::Document.new vrt
+      xml.elements.each("//ComplexSource") do |complex_source|
+        kernel_filtered_source = complex_source.parent.add_element("KernelFilteredSource")
+        complex_source.elements.each("SourceFilename|OpenOptions|SourceBand|SourceProperties|SrcRect|DstRect") do |element|
+          kernel_filtered_source.add_element element
+        end
+        kernel = kernel_filtered_source.add_element("Kernel", "normalized" => 1)
+        kernel.add_element("Size").text = coeffs.size
+        kernel.add_element("Coefs").text = coeffs.join ?\s
+        complex_source.parent.delete complex_source
+      end
+
+      log_update "%s: smoothing DEM raster" % @name
+      OS.gdal_translate "/vsistdin/", blur_path do |stdin|
+        stdin.write xml
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/esri_hdr.rb

@@ -0,0 +1,77 @@
+module NSWTopo
+  class ESRIHdr
+    def initialize(path_or_object, *args)
+      @header = case path_or_object
+      when Pathname then path_or_object.sub_ext(".hdr").each_line.map(&:upcase).map(&:split).to_h
+      when ESRIHdr then path_or_object.header.dup
+      end
+
+      @format = case @header.values_at "PIXELTYPE", "NBITS", "BYTEORDER"
+      when %w[SIGNEDINT 8 I]    then "c*"
+      when %w[SIGNEDINT 8 M]    then "c*"
+      when %w[SIGNEDINT 16 I]   then "s<*"
+      when %w[SIGNEDINT 16 M]   then "s>*"
+      when %w[SIGNEDINT 32 I]   then "l<*"
+      when %w[SIGNEDINT 32 M]   then "l>*"
+      when %w[UNSIGNEDINT 8 I]  then "C*"
+      when %w[UNSIGNEDINT 8 M]  then "C*"
+      when %w[UNSIGNEDINT 16 I] then "S<*"
+      when %w[UNSIGNEDINT 16 M] then "S>*"
+      when %w[UNSIGNEDINT 32 I] then "L<*"
+      when %w[UNSIGNEDINT 32 M] then "L>*"
+      when %w[FLOAT 32 I]       then "e*"
+      when %w[FLOAT 32 M]       then "g*"
+      end
+
+      @nodata = case path_or_object
+      when Pathname
+        case @header.values_at "PIXELTYPE", "NBITS"
+        when %w[SIGNEDINT 8]    then args.take(1).pack("c").unpack("c").first
+        when %w[UNSIGNEDINT 8]  then args.take(1).pack("C").unpack("C").first
+        when %w[SIGNEDINT 16]   then args.take(1).pack("s").unpack("s").first
+        when %w[UNSIGNEDINT 16] then args.take(1).pack("S").unpack("S").first
+        when %w[SIGNEDINT 32]   then args.take(1).pack("l").unpack("l").first
+        when %w[UNSIGNEDINT 32] then args.take(1).pack("L").unpack("L").first
+        when %w[FLOAT 32]       then args.first
+        else abort @header.inspect
+        end if args.any?
+      when ESRIHdr then path_or_object.nodata
+      end
+
+      @values = case path_or_object
+      when Pathname
+        path_or_object.sub_ext(".bil").binread.unpack(@format).map do |value|
+          value == @nodata ? nil : value
+        end
+      when ESRIHdr then args[0]
+      end
+    end
+
+    def write(path)
+      @header.map do |pair|
+        "%-#{@header.keys.map(&:length).max}s  %s\n" % pair
+      end.join('').tap do |text|
+        path.sub_ext(".hdr").write text
+      end
+      @values.map do |value|
+        value || @nodata
+      end.pack(@format).tap do |data|
+        path.sub_ext(".bil").binwrite data
+      end
+    end
+
+    attr_reader :header, :values, :nodata
+
+    def nrows
+      @nrows ||= @header["NROWS"].to_i
+    end
+
+    def ncols
+      @ncols ||= @header["NCOLS"].to_i
+    end
+
+    def rows
+      @values.each_slice ncols
+    end
+  end
+end

data/lib/nswtopo/gis/gdal_glob.rb

@@ -0,0 +1,41 @@
+module NSWTopo
+  module GDALGlob
+    def gdal_raster?(format)
+      @gdal_formats ||= OS.gdalinfo("--formats").each_line.drop(1).map do |line|
+        line.strip.split(?\s).first
+      end - %w[PDF]
+      @gdal_formats.include? format
+    end
+
+    def gdal_rasters(path)
+      paths = Array(path).flat_map do |path|
+        Pathname.glob Pathname(path).expand_path
+      end
+
+      total = nil
+      Enumerator.new do |yielder|
+        while path = paths.pop
+          OS.gdalmanage("identify", "-r", "-u", path).each_line.map do |line|
+            line.chomp.split ": "
+          end.each do |component, format|
+            case
+            when gdal_raster?(format) then yielder << component
+            when path == component 
+            when component !~ /\.zip$/
+            else paths << "/vsizip/#{component}"
+            end
+          end
+        end
+      end.entries.tap do |paths|
+        total = paths.length
+      end.map.with_index do |path, index|
+        yield [index + 1, total] if block_given?
+        info = JSON.parse OS.gdalinfo("-json", path)
+        next unless info["geoTransform"]
+        next unless wkt = info.dig("coordinateSystem", "wkt")
+        next path, info
+      rescue JSON::ParserError
+      end.compact
+    end
+  end
+end

data/lib/nswtopo/gis/geojson/collection.rb

@@ -0,0 +1,94 @@
+module NSWTopo
+  module GeoJSON
+    class Collection
+      def initialize(projection = Projection.wgs84, features = [])
+        @projection, @features = projection, features
+      end
+      attr_reader :projection, :features
+
+      def self.load(json, projection = nil)
+        collection = JSON.parse(json)
+        proj4 = collection.dig "crs", "properties", "name"
+        projection ||= proj4 ? Projection.new(proj4) : Projection.wgs84
+        collection["features"].map do |feature|
+          geometry, properties = feature.values_at "geometry", "properties"
+          type, coordinates = geometry.values_at "type", "coordinates"
+          raise Error, "unsupported geometry type: #{type}" unless TYPES === type
+          GeoJSON.const_get(type).new coordinates, properties
+        end.yield_self do |features|
+          new projection, features
+        end
+      rescue JSON::ParserError
+        raise Error, "invalid GeoJSON data"
+      end
+
+      def <<(feature)
+        tap { @features << feature }
+      end
+      alias push <<
+
+      include Enumerable
+      def each(&block)
+        block_given? ? tap { @features.each(&block) } : @features.each
+      end
+
+      def reproject_to(projection)
+        return self if self.projection == projection
+        json = OS.ogr2ogr "-t_srs", projection, "-f", "GeoJSON", "-lco", "RFC7946=NO", "/vsistdout/", "GeoJSON:/vsistdin/" do |stdin|
+          stdin.puts to_json
+        end
+        Collection.load json, projection
+      end
+
+      def reproject_to_wgs84
+        reproject_to Projection.wgs84
+      end
+
+      def to_h
+        {
+          "type" => "FeatureCollection",
+          "crs" => { "type" => "name", "properties" => { "name" => @projection } },
+          "features" => map(&:to_h)
+        }
+      end
+
+      extend Forwardable
+      delegate %i[coordinates properties] => :first
+      delegate %i[reject! select!] => :@features
+
+      def to_json(**extras)
+        to_h.merge(extras).to_json
+      end
+
+      def explode
+        Collection.new @projection, flat_map(&:explode)
+      end
+
+      def multi
+        Collection.new @projection, map(&:multi)
+      end
+
+      def merge(other)
+        raise Error, "can't merge different projections" unless @projection == other.projection
+        Collection.new @projection, @features + other.features
+      end
+
+      def merge!(other)
+        raise Error, "can't merge different projections" unless @projection == other.projection
+        tap { @features.concat other.features }
+      end
+
+      def clip!(hull)
+        @features.map! do |feature|
+          feature.clip hull
+        end.compact!
+        self
+      end
+
+      # TODO: what about empty collections?
+      def bounds
+        map(&:bounds).transpose.map(&:flatten).map(&:minmax)
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/geojson/line_string.rb

@@ -0,0 +1,11 @@
+module NSWTopo
+  module GeoJSON
+    class LineString
+      delegate %i[length offset buffer smooth samples] => :multi
+
+      def bounds
+        @coordinates.transpose.map(&:minmax)
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/geojson/multi_line_string.rb

@@ -0,0 +1,63 @@
+module NSWTopo
+  module GeoJSON
+    class MultiLineString
+      include StraightSkeleton
+
+      def clip(hull)
+        lines = [hull, hull.perps].transpose.inject(@coordinates) do |result, (vertex, perp)|
+          result.inject([]) do |clipped, points|
+            clipped + [*points, points.last].segments.inject([[]]) do |lines, segment|
+              inside = segment.map { |point| point.minus(vertex).dot(perp) >= 0 }
+              case
+              when inside.all?
+                lines.last << segment[0]
+              when inside[0]
+                lines.last << segment[0]
+                lines.last << segment.along(vertex.minus(segment[0]).dot(perp) / segment.difference.dot(perp))
+              when inside[1]
+                lines << []
+                lines.last << segment.along(vertex.minus(segment[0]).dot(perp) / segment.difference.dot(perp))
+              end
+              lines
+            end
+          end
+        end.select(&:many?)
+        lines.none? ? nil : lines.one? ? LineString.new(*lines, @properties) : MultiLineString.new(lines, @properties)
+      end
+
+      def length
+        @coordinates.sum(&:path_length)
+      end
+
+      def offset(*margins, **options)
+        linestrings = margins.inject Nodes.new(@coordinates) do |nodes, margin|
+          nodes.progress limit: margin, **options.slice(:rounding_angle, :cutoff_angle)
+        end.readout
+        MultiLineString.new linestrings, @properties
+      end
+
+      def buffer(*margins, **options)
+        MultiLineString.new(@coordinates + @coordinates.map(&:reverse), @properties).offset(*margins, **options)
+      end
+
+      def smooth(margin, **options)
+        linestrings = Nodes.new(@coordinates).tap do |nodes|
+          nodes.progress **options.slice(:rounding_angle).merge(limit: margin)
+          nodes.progress **options.slice(:rounding_angle, :cutoff_angle).merge(limit: -2 * margin)
+          nodes.progress **options.slice(:rounding_angle, :cutoff_angle).merge(limit: margin)
+        end.readout
+        MultiLineString.new linestrings, @properties
+      end
+
+      def samples(interval)
+        points = @coordinates.map do |linestring|
+          distance = linestring.path_length
+          linestring.sample_at(interval, along: true).map do |point, along|
+            [point, (2 * along - distance).abs - distance]
+          end
+        end.flatten(1).sort_by(&:last).map(&:first)
+        MultiPoint.new points, @properties
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/geojson/multi_point.rb

@@ -0,0 +1,12 @@
+module NSWTopo
+  module GeoJSON
+    class MultiPoint
+      def clip(hull)
+        points = [hull, hull.perps].transpose.inject(@coordinates) do |result, (vertex, perp)|
+          result.select { |point| point.minus(vertex).dot(perp) >= 0 }
+        end
+        points.none? ? nil : points.one? ? Point.new(*points, @properties) : MultiPoint.new(points, @properties)
+      end
+    end
+  end
+end