nswtopo 2.0.0.pre.beta1 → 3.0

data/lib/nswtopo/geometry/r_tree.rb

@@ -3,12 +3,12 @@ class RTree
     @nodes, @bounds, @object = nodes, bounds, object
   end
 
-  def overlaps?(bounds)
+  def overlaps?(bounds, buffer)
     return false if @bounds.empty?
     return true unless bounds
     bounds.zip(@bounds).all? do |bound1, bound2|
       bound1.zip(bound2.rotate).each.with_index.all? do |limits, index|
-        limits.rotate(index).inject(&:<=)
+        limits.rotate(index).inject(&:-) <= buffer
       end
     end
   end
@@ -31,17 +31,23 @@ class RTree
     end
   end
 
-  def search(bounds, searched = Set.new)
+  def search(bounds, buffer: 0, searched: Set.new)
     Enumerator.new do |yielder|
-      unless searched.include? self
-        if overlaps? bounds
-          @nodes.each do |node|
-            node.search(bounds, searched).each { |object| yielder << object }
-          end
-          yielder << @object if @nodes.empty?
+      next if searched.include? self
+      if overlaps? bounds, buffer
+        @nodes.each do |node|
+          node.search(bounds, buffer: buffer, searched: searched).inject(yielder, &:<<)
         end
-        searched << self
+        yielder << @object if @nodes.empty?
       end
+      searched << self
     end
   end
+
+  def each(&block)
+    @nodes.each do |node|
+      node.each(&block)
+    end
+    yield @bounds, @object if @object
+  end
 end

data/lib/nswtopo/geometry/segment.rb

@@ -1,18 +1,18 @@
 module Segment
   def segments
-    self[0..-2].zip self[1..-1]
+    each_cons(2).entries
   end
 
   def ring
     zip rotate
   end
 
-  def difference
+  def diff
     last.minus first
   end
 
   def distance
-    difference.norm
+    diff.norm
   end
 
   def along(fraction)

data/lib/nswtopo/geometry/straight_skeleton/nodes.rb

@@ -11,13 +11,13 @@ module StraightSkeleton
       @active, @indices = Set[], Hash.new.compare_by_identity
       data.to_d.map do |points|
         next points unless points.length > 2
-        points.inject [] do |points, point|
-          points.last == point ? points : points << point
+        points.each.with_object [] do |point, points|
+          points << point unless points.last == point
         end
       end.map.with_index do |(*points, point), index|
         points.first == point ? [points, :ring, (index unless points.hole?)] : [points << point, :segments, nil]
       end.each do |points, pair, index|
-        normals = points.send(pair).map(&:difference).map(&:normalised).map(&:perp)
+        normals = points.send(pair).map(&:diff).map(&:normalised).map(&:perp)
         points.map do |point|
           Vertex.new self, point
         end.each do |node|
@@ -137,7 +137,7 @@ module StraightSkeleton
           end
           pending.subtract nodes
           nodes << nodes.first if nodes.first == nodes.last.next
-          result << nodes
+          result << nodes unless nodes.one?
         end
       end
     end
@@ -184,10 +184,9 @@ module StraightSkeleton
         end
       end.each do |point, nodes|
         @active.subtract nodes
-        nodes.inject [] do |events, node|
+        nodes.each.with_object [] do |node, events|
           events << [:incoming, node.prev] if node.prev
           events << [:outgoing, node.next] if node.next
-          events
         end.sort_by do |event, node|
           case event
           when :incoming then [-@direction * node.normals[1].angle,        1]

data/lib/nswtopo/geometry/vector_sequence.rb

@@ -1,6 +1,6 @@
 module VectorSequence
   def perps
-    ring.map(&:difference).map(&:perp)
+    ring.map(&:diff).map(&:perp)
   end
 
   def signed_area
@@ -23,7 +23,7 @@ module VectorSequence
   end
 
   def convex?
-    ring.map(&:difference).ring.all? do |directions|
+    ring.map(&:diff).ring.all? do |directions|
       directions.inject(&:cross) >= 0
     end
   end
@@ -101,7 +101,7 @@ module VectorSequence
   end
 
   def path_length
-    segments.map(&:difference).sum(&:norm)
+    segments.map(&:diff).sum(&:norm)
   end
 
   def trim(margin)
@@ -130,9 +130,10 @@ module VectorSequence
     trim(0.5 * (path_length - length))
   end
 
-  def sample_at(interval, along: false, angle: false)
+  def sample_at(interval, along: false, angle: false, offset: nil)
     Enumerator.new do |yielder|
-      segments.inject [0.5, 0] do |(alpha, sum), segment|
+      alpha = (0.5 + Float(offset || 0) / interval) % 1.0
+      segments.inject [alpha, 0] do |(alpha, sum), segment|
         loop do
           fraction = alpha * interval / segment.distance
           break unless fraction < 1
@@ -140,7 +141,7 @@ module VectorSequence
           sum += alpha * interval
           yielder << case
           when along then [segment[0], sum]
-          when angle then [segment[0], segment.difference.angle]
+          when angle then [segment[0], segment.diff.angle]
           else segment[0]
           end
           alpha = 1.0

data/lib/nswtopo/gis/arcgis/connection.rb

@@ -0,0 +1,56 @@
+module NSWTopo
+  module ArcGIS
+    class Connection
+      ERRORS = [Timeout::Error, Errno::ENETUNREACH, Errno::ETIMEDOUT, Errno::EINVAL, Errno::ECONNRESET, Errno::ECONNREFUSED, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError]
+      Error = Class.new RuntimeError
+
+      def initialize(uri, prefix_path = nil)
+        @http = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", read_timeout: 600)
+        @prefix, @headers = Pathname(prefix_path.to_s), { "User-Agent" => "Ruby/#{RUBY_VERSION}", "Referer" => "%s://%s" % [@http.use_ssl? ? "https" : "http", @http.address] }
+        @http.max_retries = 0
+      rescue *ERRORS => error
+        raise Error, error.message
+      end
+
+      def repeatedly_request(request, &block)
+        intervals ||= 4.times.map(&1.4142.method(:**))
+        @http.request(request).tap(&:value).then(&block)
+      rescue *ERRORS, Error => error
+        intervals.any? ? sleep(intervals.shift) : raise(Error, error.message)
+        retry
+      end
+
+      def get(relative_path, **query, &block)
+        path = @prefix.join(relative_path.to_s).to_s
+        path << ?? << URI.encode_www_form(query) unless query.empty?
+        request = Net::HTTP::Get.new(path, @headers)
+        repeatedly_request(request, &block)
+      end
+
+      def post(relative_path, **query, &block)
+        path = @prefix.join(relative_path.to_s).to_s
+        request = Net::HTTP::Post.new(path, @headers)
+        request.body = URI.encode_www_form(query)
+        repeatedly_request(request, &block)
+      end
+
+      def process_json(response)
+        JSON.parse(response.body).tap do |result|
+          next unless error = result["error"]
+          # raise Error, error.values_at("message", "details").compact.join(?\n)
+          raise Error, error.values_at("message", "code").map(&:to_s).reject(&:empty?).first
+        end
+      rescue JSON::ParserError
+        raise Error, "unexpected ArcGIS response format"
+      end
+
+      def get_json(relative_path = "", **query)
+        get relative_path, **query, f: "json", &method(:process_json)
+      end
+
+      def post_json(relative_path = "", **query)
+        post relative_path, **query, f: "json", &method(:process_json)
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/arcgis/layer/map.rb

@@ -0,0 +1,163 @@
+module NSWTopo
+  module ArcGIS
+    module Map
+      TILE = 1000
+      NoUniqueFieldError = Class.new RuntimeError
+
+      def pages(per_page)
+        objectid_field = @layer["fields"].find do |field|
+          field["type"] == "esriFieldTypeOID"
+        end&.fetch("name")
+
+        raise "ArcGIS layer does not support dynamic layers: #{@name}" unless @service["supportsDynamicLayers"]
+        raise "ArcGIS layer does not support SVG output: #{@name}" unless @service["supportedImageFormatTypes"].split(?,).include? "SVG"
+        raise "ArcGIS layer does not have an objectid field: #{@name}" unless objectid_field
+
+        @unique ||= @type_field
+        @unique ||= @layer["fields"].find do |field|
+          field.values_at("name", "alias").map(&:downcase).include? @layer.dig("drawingInfo", "renderer", "field1")&.downcase
+        end&.fetch("name")
+        @unique ||= @coded_values.min_by do |name, lookup|
+          lookup.length
+        end&.first
+        raise NoUniqueFieldError unless @unique
+
+        @count = classify(@unique).sum(&:last)
+        return [GeoJSON::Collection.new(projection: projection, name: @name)].each if @count.zero?
+
+        @fields ||= @layer["fields"].select do |field|
+          Layer::FIELD_TYPES === field["type"]
+        end.map do |field|
+          field["name"]
+        end
+
+        include_objectid = @fields.include? objectid_field
+        min, chunk, table = 0, 10000, {}
+        loop do
+          break unless table.length < @count
+          page, where = {}, ["#{objectid_field}>=#{min}", "#{objectid_field}<#{min + chunk}", *@where]
+          Set[*@fields, *extra_field].delete(objectid_field).each_slice(2) do |fields|
+            classify(objectid_field, *fields, where: where).each do |attributes, count|
+              objectid = attributes.delete objectid_field
+              page[objectid] ||= include_objectid ? { objectid_field => objectid } : {}
+              page[objectid].merge! attributes
+            end
+          end
+        rescue Connection::Error
+          (chunk /= 2) > 0 ? retry : raise
+        else
+          table.merge! page
+          min += chunk
+        end
+
+        parent = @layer
+        scale = loop do
+          break parent["minScale"] if parent["minScale"]&.nonzero?
+          break parent["effectiveMinScale"] if parent["effectiveMinScale"]&.nonzero?
+          break unless parent_id = parent.dig("parentLayer", "id")
+          parent = get_json parent_id
+        end || begin
+          case @service["units"]
+          when "esriMeters" then 100000
+          else raise "can't get features from layer: #{@name}"
+          end
+        end
+
+        bounds = @layer["extent"].values_at("xmin", "xmax", "ymin", "ymax").each_slice(2)
+        cx, cy = bounds.map { |bound| 0.5 * bound.sum }
+        bbox, size = %W[#{cx},#{cy},#{cx},#{cy} #{TILE},#{TILE}]
+        dpi = bounds.map { |b0, b1| 0.0254 * TILE * scale / (b1 - b0) }.min * 0.999
+
+        renderer = case @geometry_type
+        when "esriGeometryPoint"
+          { type: "simple", symbol: { color: [0,0,0,255], size: 1, type: "esriSMS", style: "esriSMSSquare" } }
+        when "esriGeometryPolyline"
+          { type: "simple", symbol: { color: [0,0,0,255], width: 1, type: "esriSLS", style: "esriSLSSolid" } }
+        when "esriGeometryPolygon"
+          { type: "simple", symbol: { color: [0,0,0,255], width: 0, type: "esriSFS", style: "esriSFSSolid" } }
+        else
+          raise "unsupported ArcGIS geometry type: #{@geometry_type}"
+        end
+        dynamic_layer = { source: { type: "mapLayer", mapLayerId: @id }, drawingInfo: { showLabels: false, renderer: renderer } }
+
+        sets = table.group_by(&:last).map(&:last).sort_by(&:length)
+
+        Enumerator::Lazy.new(sets) do |yielder, objectids_properties|
+          while objectids_properties.any?
+            begin
+              objectids, properties = objectids_properties.take(per_page).transpose
+              dynamic_layers = [dynamic_layer.merge(definitionExpression: "#{objectid_field} IN (#{objectids.join ?,})")]
+              export = get_json "export", format: "svg", dynamicLayers: dynamic_layers.to_json, bbox: bbox, size: size, mapScale: scale, dpi: dpi
+              href = URI.parse export["href"]
+              xml = Connection.new(href).get(href.path, &:body)
+              xmin, xmax, ymin, ymax = export["extent"].values_at "xmin", "xmax", "ymin", "ymax"
+            rescue Connection::Error
+              (per_page /= 2) > 0 ? retry : raise
+            end
+
+            REXML::Document.new(xml).elements.collect("svg//g[@transform]//g[@transform][path[@d]]") do |group|
+              a, b, c, d, e, f = group.attributes["transform"].match(/matrix\((.*)\)/)[1].split(?\s).map(&:to_f)
+              coords = []
+              group.elements["path[@d]"].attributes["d"].gsub(/\ *([MmZzLlHhVvCcSsQqTtAa])\ */) do
+                ?\s + $1 + ?\s
+              end.strip.split(?\s).slice_before(/[MmZzLlHhVvCcSsQqTtAa]/).each do |command, *numbers|
+                raise "can't handle SVG path data command: #{command}" unless numbers.length.even?
+                coordinates = numbers.each_slice(2).map do |x, y|
+                  fx, fy = [(a * Float(x) + c * Float(y) + e) / TILE, (b * Float(x) + d * Float(y) + f) / TILE]
+                  [fx * xmax + (1 - fx) * xmin, fy * ymin + (1 - fy) * ymax]
+                end
+                case command
+                when ?Z then next
+                when ?M then coords << coordinates
+                when ?L then coords.last.concat coordinates
+                when ?C
+                  coordinates.each_slice(3) do |points|
+                    raise "unexpected SVG response (bad path data)" unless points.length == 3
+                    curves = [[coords.last.last, *points]]
+                    while curve = curves.shift
+                      next if curve.first == curve.last
+                      if curve.values_at(0,-1).distance < 0.99 * curve.segments.map(&:distance).sum
+                        reduced = 3.times.inject [ curve ] do |reduced|
+                          reduced << reduced.last.each_cons(2).map do |(x0, y0), (x1, y1)|
+                            [0.5 * (x0 + x1), 0.5 * (y0 + y1)]
+                          end
+                        end
+                        curves.unshift reduced.map(&:last).reverse
+                        curves.unshift reduced.map(&:first)
+                      else
+                        coords.last << curve.last
+                      end
+                    end
+                  end
+                else raise "can't handle SVG path data command: #{command}"
+                end
+              end
+              coords
+            end.tap do |coords|
+              lengths = [properties.length, coords.length]
+              raise "unexpected SVG response (expected %i features, received %i)" % lengths if lengths.inject(&:<)
+            end.zip(properties).map do |coords, properties|
+              case @geometry_type
+              when "esriGeometryPoint"
+                raise "unexpected SVG response (bad point symbol)" unless coords.map(&:length) == [ 4 ]
+                point = coords[0].transpose.map { |coords| coords.sum / coords.length }
+                next GeoJSON::Point.new point, properties
+              when "esriGeometryPolyline"
+                next GeoJSON::LineString.new coords[0], properties if @mixed && coords.one?
+                next GeoJSON::MultiLineString.new coords, properties
+              when "esriGeometryPolygon"
+                coords.each(&:reverse!) unless coords[0].anticlockwise?
+                polys = coords.slice_before(&:anticlockwise?).entries
+                next GeoJSON::Polygon.new polys.first, properties if @mixed && polys.one?
+                next GeoJSON::MultiPolygon.new polys, properties
+              end
+            end.tap do |features|
+              yielder << GeoJSON::Collection.new(projection: projection, name: @name, features: features)
+            end
+            objectids_properties.shift per_page
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/arcgis/layer/query.rb

@@ -0,0 +1,87 @@
+module NSWTopo
+  module ArcGIS
+    module Query
+      UniqueFieldError = Class.new RuntimeError
+
+      def base_query(**options)
+        case
+        when @unique
+          raise UniqueFieldError
+        when @geometry
+          raise "polgyon geometry required" unless @geometry.polygon?
+          options[:geometry] = { rings: @geometry.reproject_to(projection).coordinates.map(&:reverse) }.to_json
+          options[:geometryType] = "esriGeometryPolygon"
+          options[:where] = join_clauses(*@where) if @where
+        when @where
+          options[:where] = join_clauses(*@where)
+        else
+          oid_field = @layer["fields"].find do |field|
+            field["type"] == "esriFieldTypeOID"
+          end&.fetch("name")
+          options[:where] = oid_field ? "#{oid_field} IS NOT NULL" : "1=1"
+        end
+        options
+      end
+
+      def count
+        @count ||= get_json("#{@id}/query", **base_query, returnCountOnly: true).dig("count")
+      end
+
+      def pages(per_page)
+        objectids = get_json("#{@id}/query", **base_query, returnIdsOnly: true)["objectIds"] || []
+        @count = objectids.count
+        return [GeoJSON::Collection.new(projection: projection, name: @name)].each if @count.zero?
+
+        @fields ||= @layer["fields"].select do |field|
+          Layer::FIELD_TYPES === field["type"]
+        end.map do |field|
+          field["name"]
+        end
+
+        Enumerator.new do |yielder|
+          out_fields = [*@fields, *extra_field].join ?,
+          while objectids.any?
+            begin
+              get_json "#{@id}/query", outFields: out_fields, objectIds: objectids.take(per_page).join(?,)
+            rescue Connection::Error
+              (per_page /= 2) > 0 ? retry : raise
+            end.fetch("features", []).map do |feature|
+              next unless geometry = feature["geometry"]
+              properties = feature.fetch("attributes", {})
+
+              case @geometry_type
+              when "esriGeometryPoint"
+                point = geometry.values_at "x", "y"
+                next unless point.all?
+                next GeoJSON::Point.new point, properties
+              when "esriGeometryMultipoint"
+                points = geometry["points"]
+                next unless points&.any?
+                next GeoJSON::MultiPoint.new points.transpose.take(2).transpose, properties
+              when "esriGeometryPolyline"
+                raise "ArcGIS curve geometries not supported" if geometry.key? "curvePaths"
+                paths = geometry["paths"]
+                next unless paths&.any?
+                next GeoJSON::LineString.new paths[0], properties if @mixed && paths.one?
+                next GeoJSON::MultiLineString.new paths, properties
+              when "esriGeometryPolygon"
+                raise "ArcGIS curve geometries not supported" if geometry.key? "curveRings"
+                rings = geometry["rings"]
+                next unless rings&.any?
+                rings.each(&:reverse!) unless rings[0].anticlockwise?
+                polys = rings.slice_before(&:anticlockwise?).entries
+                next GeoJSON::Polygon.new polys.first, properties if @mixed && polys.one?
+                next GeoJSON::MultiPolygon.new polys, properties
+              else
+                raise "unsupported ArcGIS geometry type: #{@geometry_type}"
+              end
+            end.compact.tap do |features|
+              yielder << GeoJSON::Collection.new(projection: projection, features: features, name: @name)
+            end
+            objectids.shift per_page
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/arcgis/layer/renderer.rb

@@ -0,0 +1,66 @@
+module NSWTopo
+  module ArcGIS
+    module Renderer
+      TooManyFieldsError = Class.new RuntimeError
+      NoGeometryError = Class.new RuntimeError
+
+      def classify(*fields, where: @where)
+        raise TooManyFieldsError unless fields.size <= 3
+        raise NoGeometryError, "ArcGIS layer does not support spatial filtering: #{@name}" if @geometry
+
+        types = fields.map do |name|
+          @layer["fields"].find do |field|
+            field["name"] == name
+          end&.fetch("type")
+        end
+
+        counts, values = 2.times do |repeat|
+          counts, values = %w[| ~ ^ #].find do |delimiter|
+            classification_def = { type: "uniqueValueDef", uniqueValueFields: fields.take(repeat) + fields, fieldDelimiter: delimiter }
+            unique_values = get_json("#{@id}/generateRenderer", where: join_clauses(*where), classificationDef: classification_def.to_json).fetch("uniqueValueInfos")
+
+            values = unique_values.map do |info|
+              info["value"].split(delimiter).map(&:strip)
+            end
+            next unless values.all? do |values|
+              values.length == fields.length + repeat
+            end
+            repeat.times { values.each(&:shift) }
+            counts = unique_values.map do |info|
+              info["count"]
+            end
+            break counts, values
+          end
+          raise "couldn't delimit values" unless values
+          next if 0 == repeat && fields.one? && (counts.all?(1) || counts.all?(0))
+          break counts, values
+        end
+
+        values.map do |values|
+          values.zip(types).map do |value, type|
+            case
+            when value == "<Null>" then nil
+            when value == "" then nil
+            when type == "esriFieldTypeOID" then Integer(value)
+            when type == "esriFieldTypeInteger" then Integer(value)
+            when type == "esriFieldTypeSmallInteger" then Integer(value)
+            when type == "esriFieldTypeDouble" then Float(value)
+            when type == "esriFieldTypeSingle" then Float(value)
+            when type == "esriFieldTypeString" then String(value)
+            when type == "esriFieldTypeGUID" then String(value)
+            when type == "esriFieldTypeDate"
+              begin
+                Time.strptime(value, "%m/%d/%Y %l:%M:%S %p").to_i * 1000
+              rescue ArgumentError
+              end
+            end
+          rescue ArgumentError
+            raise "could not interpret #{value.inspect} as #{type}"
+          end.then do |values|
+            fields.zip values
+          end.to_h
+        end.zip counts
+      end
+    end
+  end
+end

data/lib/nswtopo/gis/arcgis/layer/statistics.rb

@@ -0,0 +1,15 @@
+module NSWTopo
+  module ArcGIS
+    module Statistics
+      def classify(*fields)
+        statistics = fields.map.with_index do |name, index|
+          { statisticType: "count", onStatisticField: name, outStatisticFieldName: "COUNT_#{index}" }
+        end
+        field_counts = get_json "#{@id}/query", **base_query, outStatistics: statistics.to_json, groupByFieldsForStatistics: fields.join(?,)
+        field_counts["features"].map do |feature|
+          [feature["attributes"].slice(*fields), feature["attributes"]["COUNT_0"]]
+        end
+      end
+    end
+  end
+end