nswtopo 2.0.0.pre.beta1

data/lib/nswtopo/layer/vegetation.rb

@@ -0,0 +1,73 @@
+module NSWTopo
+  module Vegetation
+    include Raster, GDALGlob
+    CREATE = %w[mapping contrast colour]
+
+    def get_raster(temp_dir)
+      txt_path = temp_dir / "source.txt"
+      vrt_path = temp_dir / "source.vrt"
+
+      min, max = minmax = @mapping&.values_at("min", "max")
+      low, high, factor = { "low" => 0, "high" => 100, "factor" => 0.0 }.merge(@contrast || {}).values_at "low", "high", "factor"
+      woody, nonwoody = { "woody" => "#A6F1A6", "non-woody" => "#FFFFFF" }.merge(@colour || {}).values_at("woody", "non-woody").map { |string| Colour.new string }
+
+      colour_table = (0..255).map do |index|
+        case
+        when minmax&.all?(Integer) && minmax.all?(0..255)
+          (100.0 * (index - min) / (max - min)).clamp(0.0, 100.0)
+        when @mapping&.keys&.all?(Integer)
+          @mapping.fetch(index, 0)
+        else raise "no vegetation colour mapping specified for #{name}"
+        end
+      end.map do |percent|
+        (Float(percent - low) / (high - low)).clamp(0.0, 1.0)
+      end.map do |x|
+        next x if factor.zero?
+        [x, 1.0].map do |x|
+          [x, 0.0].map do |x|
+            1 / (1 + Math::exp(factor * (0.5 - x)))
+          end.inject(&:-)
+        end.inject(&:/) # sigmoid between 0..1
+      end.map do |x|
+        nonwoody.mix(woody, x)
+      end
+
+      Dir.chdir(@source ? @source.parent : Pathname.pwd) do
+        gdal_rasters @path
+      end.tap do |rasters|
+        raise "no vegetation data file specified" if rasters.none?
+      end.group_by do |path, info|
+        Projection.new info.dig("coordinateSystem", "wkt")
+      end.map.with_index do |(projection, rasters), index|
+        indexed_tif_path = temp_dir / "indexed.#{index}.tif"
+        indexed_vrt_path = temp_dir / "indexed.#{index}.vrt"
+        coloured_tif_path = temp_dir / "coloured.#{index}.tif"
+        tif_path = temp_dir / "output.#{index}.tif"
+
+        txt_path.write rasters.map(&:first).join(?\n)
+        OS.gdalbuildvrt "-overwrite", "-input_file_list", txt_path, vrt_path
+        OS.gdal_translate "-projwin", *@map.projwin(projection), "-r", "near", "-co", "TFW=YES", vrt_path, indexed_tif_path
+        OS.gdal_translate "-of", "VRT", indexed_tif_path, indexed_vrt_path
+
+        xml = REXML::Document.new indexed_vrt_path.read
+        raise "can't process vegetation data for #{@name}" unless xml.elements.each("/VRTDataset/VRTRasterBand/ColorTable", &:itself).one?
+        raise "can't process vegetation data for #{@name}" unless xml.elements.each("/VRTDataset/VRTRasterBand/ColorTable/Entry", &:itself).count == 256
+        xml.elements.collect("/VRTDataset/VRTRasterBand/ColorTable/Entry", &:itself).zip(colour_table) do |entry, colour|
+          entry.attributes["c1"], entry.attributes["c2"], entry.attributes["c3"], entry.attributes["c4"] = *colour.triplet, 255
+        end
+        xml.elements.each("/VRTDataset/VRTRasterBand/NoDataValue", &:remove)
+        indexed_vrt_path.write xml
+        OS.gdal_translate "-expand", "rgb", indexed_vrt_path, coloured_tif_path
+
+        OS.gdalwarp "-s_srs", projection, "-t_srs", @map.projection, "-r", "bilinear", coloured_tif_path, tif_path
+        next tif_path, Numeric === @resolution ? @resolution : @map.get_raster_resolution(tif_path)
+      end.transpose.tap do |tif_paths, resolutions|
+        @resolution = resolutions.min
+        txt_path.write tif_paths.join(?\n)
+        OS.gdalbuildvrt "-overwrite", "-input_file_list", txt_path, vrt_path
+      end
+
+      return @resolution, vrt_path
+    end
+  end
+end

data/lib/nswtopo/layer.rb

@@ -0,0 +1,78 @@
+require_relative 'layer/raster'
+require_relative 'layer/vector'
+require_relative 'layer/vegetation'
+require_relative 'layer/import'
+require_relative 'layer/arcgis_raster'
+require_relative 'layer/feature'
+require_relative 'layer/contour'
+require_relative 'layer/spot'
+require_relative 'layer/overlay'
+require_relative 'layer/relief'
+require_relative 'layer/grid'
+require_relative 'layer/declination'
+require_relative 'layer/control'
+require_relative 'layer/labels'
+
+module NSWTopo
+  class Layer
+    TYPES = Set[Vegetation, Import, ArcGISRaster, Feature, Contour, Spot, Overlay, Relief, Grid, Declination, Control, Labels]
+
+    def initialize(name, map, params)
+      @type = begin
+        NSWTopo.const_get params["type"]
+      rescue NameError, TypeError
+      end
+
+      raise "unrecognised layer type: %s" % params["type"].inspect unless TYPES === @type
+      extend @type
+
+      @params = @type.const_defined?(:DEFAULTS) ? @type.const_get(:DEFAULTS).transform_keys(&:to_s).merge(params) : params
+      @name, @map, @source, @path, @resolution = Layer.sanitise(name), map, @params.delete("source"), @params.delete("path"), @params.delete("resolution")
+
+      @type.const_get(:CREATE).map(&:to_s).each do |attr|
+        instance_variable_set ?@ + attr.tr_s(?-, ?_), @params.delete(attr)
+      end if @type.const_defined?(:CREATE)
+    end
+
+    attr_reader :name, :params
+    alias to_s name
+
+    def level
+      case
+      when Vegetation   == @type then 0
+      when Import       == @type then 1
+      when ArcGISRaster == @type then 1
+      when Feature      == @type then 2
+      when Contour      == @type then 2
+      when Spot         == @type then 2
+      when Overlay      == @type then 3
+      when Relief       == @type then 4
+      when Grid         == @type then 5
+      when Declination  == @type then 6
+      when Control      == @type then 7
+      when Labels       == @type then 99
+      end
+    end
+
+    def <=>(other)
+      [self, other].map(&:level).inject(&:<=>)
+    end
+
+    def ==(other)
+      Layer === other && self.name == other.name
+    end
+
+    def uptodate?
+      mtimes = [@source&.mtime, @map.mtime(filename)]
+      mtimes.all? && mtimes.inject(&:<)
+    end
+
+    def pair
+      return name, params
+    end
+
+    def self.sanitise(name)
+      name&.tr_s '^_a-zA-Z0-9*\-', ?.
+    end
+  end
+end

data/lib/nswtopo/log.rb

@@ -0,0 +1,28 @@
+module NSWTopo
+  module Log
+    SUCCESS = $stdout.tty? ? "\r\e[2K\e[32mnswtopo:\e[0m %s" : "nswtopo: %s"
+    FAILURE = $stderr.tty? ? "\r\e[2K\e[31mnswtopo:\e[0m %s" : "nswtopo: %s"
+    NEUTRAL = $stdout.tty? ? "\r\e[2Knswtopo: %s" : "nswtopo: %s"
+    UPDATE  = "\r\e[2K%s"
+
+    def log_success(message)
+      puts SUCCESS % message
+    end
+
+    def log_neutral(message)
+      puts NEUTRAL % message
+    end
+
+    def log_update(message)
+      print UPDATE % message if $stdout.tty?
+    end
+
+    def log_warn(message)
+      warn FAILURE % message
+    end
+
+    def log_abort(message)
+      abort FAILURE % message
+    end
+  end
+end

data/lib/nswtopo/map.rb

@@ -0,0 +1,296 @@
+module NSWTopo
+  class Map
+    include Formats, Dither, Zip, Log, Safely
+
+    def initialize(archive, proj4:, scale:, centre:, extents:, rotation:, layers: {})
+      @archive, @scale, @centre, @extents, @rotation, @layers = archive, scale, centre, extents, rotation, layers
+      @projection = Projection.new proj4
+      ox, oy = bounding_box.coordinates[0][3]
+      @affine = [[1, 0], [0, -1], [-ox, oy]].map do |vector|
+        vector.rotate_by_degrees(-@rotation).times(1000.0 / @scale)
+      end.transpose
+    end
+    attr_reader :projection, :scale, :centre, :extents, :rotation
+
+    extend Forwardable
+    delegate %i[write mtime read] => :@archive
+
+    def self.init(archive, scale: 25000, rotation: 0.0, bounds: nil, coords: nil, dimensions: nil, margins: nil)
+      wgs84_points = case
+      when coords && bounds
+        raise "can't specify both bounds file and map coordinates"
+      when coords
+        coords
+      when bounds
+        gps = GPS.load bounds
+        margins ||= [15, 15] unless dimensions || gps.polygons.any?
+        case
+        when gps.polygons.any?
+          gps.polygons.map(&:coordinates).flatten(1).inject(&:+)
+        when gps.linestrings.any?
+          gps.linestrings.map(&:coordinates).inject(&:+)
+        when gps.points.any?
+          gps.points.map(&:coordinates)
+        else
+          raise "no features found in %s" % bounds
+        end
+      else
+        raise "no bounds file or map coordinates specified"
+      end
+
+      wgs84_centre = wgs84_points.transpose.map(&:minmax).map(&:sum).times(0.5)
+      projection = Projection.azimuthal_equidistant *wgs84_centre
+
+      case rotation
+      when "auto"
+        raise "can't specify both map dimensions and auto-rotation" if dimensions
+        points = GeoJSON.multipoint(wgs84_points).reproject_to(projection).coordinates
+        centre, extents, rotation = points.minimum_bounding_box(*margins)
+        rotation *= -180.0 / Math::PI
+      when "magnetic"
+        rotation = declination(*wgs84_centre)
+      else
+        raise "map rotation must be between ±45°" unless rotation.abs <= 45
+      end
+
+      case
+      when centre
+      when dimensions
+        raise "can't specify both margins and map dimensions" if margins
+        extents = dimensions.map do |dimension|
+          dimension * 0.001 * scale
+        end
+        centre = GeoJSON.point(wgs84_centre).reproject_to(projection).coordinates
+      else
+        points = GeoJSON.multipoint(wgs84_points).reproject_to(projection).coordinates
+        centre, extents = points.map do |point|
+          point.rotate_by_degrees rotation
+        end.transpose.map(&:minmax).map do |min, max|
+          [0.5 * (max + min), max - min]
+        end.transpose
+        centre.rotate_by_degrees! -rotation
+      end
+
+      wgs84_centre = GeoJSON.point(centre, projection: projection).reproject_to_wgs84.coordinates
+      projection = Projection.transverse_mercator *wgs84_centre
+
+      extents = extents.zip(margins).map do |extent, margin|
+        extent + 2 * margin * 0.001 * scale
+      end if margins
+
+      case
+      when extents.all?(&:positive?)
+      when coords
+        raise "not enough information to calculate map size – add more coordinates, or specify map dimensions or margins"
+      when bounds
+        raise "not enough information to calculate map size – check bounds file, or specify map dimensions or margins"
+      end
+
+      new(archive, proj4: projection.proj4, scale: scale, centre: [0, 0], extents: extents, rotation: rotation).save
+    end
+
+    def self.load(archive)
+      new archive, **YAML.load(archive.read "map.yml")
+    end
+
+    def save
+      tap { @archive.write "map.yml", YAML.dump(proj4: @projection.proj4, scale: @scale, centre: @centre, extents: @extents, rotation: @rotation, layers: @layers) }
+    end
+
+    def layers
+      @layers.map do |name, params|
+        Layer.new(name, self, params)
+      end
+    end
+
+    def raster_dimensions_at(ppi: nil, resolution: nil)
+      resolution ||= 0.0254 * @scale / ppi
+      ppi ||= 0.0254 * @scale / resolution
+      return (@extents / resolution).map(&:ceil), ppi, resolution
+    end
+
+    def wgs84_centre
+      GeoJSON.point(@centre, projection: @projection).reproject_to_wgs84.coordinates
+    end
+
+    def bounding_box(mm: nil, metres: nil)
+      margin = mm ? mm * 0.001 * @scale : metres ? metres : 0
+      ring = @extents.map do |extent|
+        [-0.5 * extent - margin, 0.5 * extent + margin]
+      end.inject(&:product).map do |offset|
+        @centre.plus offset.rotate_by_degrees(-@rotation)
+      end.values_at(0,2,3,1,0)
+      GeoJSON.polygon [ring], projection: projection
+    end
+
+    def bounds(margin: {}, projection: nil)
+      bounding_box(margin).yield_self do |bbox|
+        projection ? bbox.reproject_to(projection) : bbox
+      end.coordinates.first.transpose.map(&:minmax)
+    end
+
+    def projwin(projection)
+      bounds(projection: projection).flatten.values_at(0,3,1,2)
+    end
+
+    def write_world_file(path, resolution: nil, ppi: nil)
+      resolution ||= 0.0254 * @scale / ppi
+      top_left = bounding_box.coordinates[0][3]
+      WorldFile.write top_left, resolution, -@rotation, path
+    end
+
+    def coords_to_mm(point)
+      @affine.map do |row|
+        row.dot [*point, 1.0]
+      end
+    end
+
+    def get_raster_resolution(raster_path)
+      metre_diagonal = bounding_box.coordinates.first.values_at(0, 2)
+      pixel_diagonal = OS.gdaltransform "-i", "-t_srs", @projection, raster_path do |stdin|
+        metre_diagonal.each do |point|
+          stdin.puts point.join(?\s)
+        end
+      end.each_line.map do |line|
+        line.split(?\s).take(2).map(&:to_f)
+      end
+      metre_diagonal.distance / pixel_diagonal.distance
+    rescue OS::Error
+      raise "invalid raster"
+    end
+
+    def self.declination(longitude, latitude)
+      today = Date.today
+      query = { lat1: latitude.abs, lat1Hemisphere: latitude < 0 ? ?S : ?N, lon1: longitude.abs, lon1Hemisphere: longitude < 0 ? ?W : ?E, model: "WMM", startYear: today.year, startMonth: today.month, startDay: today.day, resultFormat: "xml" }
+      uri = URI::HTTPS.build host: "www.ngdc.noaa.gov", path: "/geomag-web/calculators/calculateDeclination", query: URI.encode_www_form(query)
+      xml = Net::HTTP.get uri
+      text = REXML::Document.new(xml).elements["//declination"]&.text
+      text ? text.to_f : raise
+    rescue RuntimeError, SystemCallError, EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, SocketError
+      raise "couldn't get magnetic declination value"
+    end
+
+    def declination
+      Map.declination *wgs84_centre
+    end
+
+    def add(*layers, after: nil, before: nil, replace: nil, overwrite: false)
+      [%w[before after replace], [before, after, replace]].transpose.select(&:last).each do |option, name|
+        next if self.layers.any? { |other| other.name == name }
+        raise "no such layer: %s" % name
+      end.map(&:first).combination(2).each do |options|
+        raise OptionParser::AmbiguousOption,  "can't specify --%s and --%s simultaneously" % options
+      end
+
+      layers.inject [self.layers, false, replace || after, []] do |(layers, changed, follow, errors), layer|
+        index = layers.index layer unless replace || after || before
+        if overwrite || !layer.uptodate?
+          layer.create
+          log_success "%s layer: %s" % [layer.empty? ? "empty" : "added", layer.name]
+        else
+          log_neutral "kept existing layer: %s" % layer.name
+          next layers, changed, layer.name, errors if index
+        end
+        layers.delete layer
+        case
+        when index
+        when follow
+          index = layers.index { |other| other.name == follow }
+          index += 1
+        when before
+          index = layers.index { |other| other.name == before }
+        else
+          index = layers.index { |other| (other <=> layer) > 0 } || -1
+        end
+        next layers.insert(index, layer), true, layer.name, errors
+      rescue ArcGISServer::Error, RuntimeError => error
+        log_warn ArcGISServer::Error === error ? "couldn't download layer: #{layer.name}" : error.message
+        next layers, changed, follow, errors << error
+      end.tap do |ordered_layers, changed, follow, errors|
+        if changed
+          @layers.replace Hash[ordered_layers.map(&:pair)]
+          replace ? delete(replace) : save
+        end
+        raise PartialFailureError, "failed to create %s" % [layers.one? ? "layer" : errors.one? ? "1 layer" : "#{errors.length} layers"] if errors.any?
+      end
+    end
+
+    def delete(*names)
+      raise OptionParser::MissingArgument, "no layers specified" unless names.any?
+      names.inject Set[] do |matched, name|
+        matches = @layers.keys.grep(name)
+        raise "no such layer: #{name}" if String === name && matches.none?
+        matched.merge matches
+      end.tap do |names|
+        raise "no matching layers found" unless names.any?
+      end.each do |name|
+        params = @layers.delete name
+        @archive.delete Layer.new(name, self, params).filename
+        log_success "deleted layer: %s" % name
+      end
+      save
+    end
+
+    def info(empty: nil)
+      StringIO.new.tap do |io|
+        io.puts "%-11s 1:%i" %            ["scale:",      @scale]
+        io.puts "%-11s %imm × %imm" %     ["dimensions:", *@extents.times(1000.0 / @scale)]
+        io.puts "%-11s %.1fkm × %.1fkm" % ["extent:",     *@extents.times(0.001)]
+        io.puts "%-11s %.1fkm²" %         ["area:",       @extents.inject(&:*) * 0.000001]
+        io.puts "%-11s %.1f°" %           ["rotation:",   @rotation]
+        layers.reject(&empty ? :nil? : :empty?).inject("layers:") do |heading, layer|
+          io.puts "%-11s %s" % [heading, layer]
+          nil
+        end
+      end.string
+    end
+    alias to_s info
+
+    def render(*paths, worldfile: false, force: false, external: nil, **options)
+      @archive.delete "map.svg" if force
+      Dir.mktmppath do |temp_dir|
+        rasters = Hash.new do |rasters, opts|
+          png_path = temp_dir / "raster.#{rasters.size}.png"
+          pgw_path = temp_dir / "raster.#{rasters.size}.pgw"
+          rasterise png_path, external: external, **opts
+          write_world_file pgw_path, opts
+          rasters[opts] = png_path
+        end
+        dithers = Hash.new do |dithers, opts|
+          png_path = temp_dir / "dither.#{dithers.size}.png"
+          pgw_path = temp_dir / "dither.#{dithers.size}.pgw"
+          FileUtils.cp rasters[opts], png_path
+          dither png_path
+          write_world_file pgw_path, opts
+          dithers[opts] = png_path
+        end
+
+        outputs = paths.map.with_index do |path, index|
+          ext = path.extname.delete_prefix ?.
+          name = path.basename(path.extname)
+          out_path = temp_dir / "output.#{index}.#{ext}"
+          send "render_#{ext}", out_path, name: name, external: external, **options do |dither: false, **opts|
+            (dither ? dithers : rasters)[opts]
+          end
+          next out_path, path
+        end
+
+        safely "saving, please wait..." do
+          outputs.each do |out_path, path|
+            FileUtils.cp out_path, path
+            log_success "created %s" % path
+          end
+
+          paths.select do |path|
+            %w[.png .tif .jpg].include? path.extname
+          end.group_by do |path|
+            path.parent / path.basename(path.extname)
+          end.keys.each do |base|
+            write_world_file Pathname("#{base}.wld"), ppi: options.fetch(:ppi, Formats::PPI)
+            Pathname("#{base}.prj").write "#{@projection}\n"
+          end if worldfile
+        end
+      end
+    end
+  end
+end

data/lib/nswtopo/os.rb

@@ -0,0 +1,75 @@
+module NSWTopo
+  module OS
+    Error = Class.new RuntimeError
+    Missing = Class.new RuntimeError
+
+    GDAL = %w[
+      gdal_contour
+      gdal_grid
+      gdal_rasterize
+      gdal_translate
+      gdaladdo
+      gdalbuildvrt
+      gdaldem
+      gdalenhance
+      gdalinfo
+      gdallocationinfo
+      gdalmanage
+      gdalserver
+      gdalsrsinfo
+      gdaltindex
+      gdaltransform
+      gdalwarp
+      gnmanalyse
+      gnmmanage
+      nearblack
+      ogr2ogr
+      ogrinfo
+      ogrlineref
+      ogrtindex
+      testepsg
+    ]
+    ImageMagick = %w[
+      animate
+      compare
+      composite
+      conjure
+      convert
+      display
+      identify
+      import
+      mogrify
+      montage
+      stream
+    ]
+    SQLite3 = %w[sqlite3]
+    PNGQuant = %w[pngquant]
+    GIMP = %w[gimp]
+    Zip = %w[zip]
+    SevenZ = %w[7z]
+
+    extend self
+
+    %w[GDAL ImageMagick SQLite3 PNGQuant GIMP Zip SevenZ].each do |package|
+      OS.const_get(package).each do |command|
+        define_method command do |*args, &block|
+          Open3.popen3 command, *args.map(&:to_s) do |stdin, stdout, stderr, thread|
+            thr_in = Thread.new do
+              block.call(stdin) if block
+            rescue Errno::EPIPE
+            ensure
+              stdin.close
+            end
+            thr_out = Thread.new { stdout.read }
+            thr_err = Thread.new { stderr.read }
+            [thr_in, thr_out, thr_err].each(&:join)
+            raise Error, "#{command}: #{thr_err.value.empty? ? thr_out.value : thr_err.value}" unless thread.value.success?
+            thr_out.value
+          end
+        rescue Errno::ENOENT
+          raise Missing, "#{package} not installed"
+        end
+      end
+    end
+  end
+end

data/lib/nswtopo/safely.rb

@@ -0,0 +1,13 @@
+module NSWTopo
+  module Safely
+    include Log
+    def safely(message)
+      yield
+    rescue Interrupt => interrupt
+      log_warn message
+      retry
+    ensure
+      raise interrupt if interrupt
+    end
+  end
+end

data/lib/nswtopo/version.rb

@@ -0,0 +1,4 @@
+module NSWTopo
+  VERSION = "2.0.0-beta1"
+  MIN_VERSION = "2.0.0-beta1"
+end

data/lib/nswtopo/zip.rb

@@ -0,0 +1,15 @@
+module NSWTopo
+  module Zip
+    def zip(directory, archive)
+      Enumerator.new do |yielder|
+        yielder << ->(dir) { OS.zip "-r", archive.expand_path, *Pathname.glob('*') }
+        yielder << ->(dir) { OS.send "7z", "a", "-tzip", "-r", archive.expand_path, *Pathname.glob('*') }
+        raise "no zip utility installed"
+      end.each do |zip|
+        Dir.chdir(directory, &zip)
+        break
+      rescue OS::Missing
+      end
+    end
+  end
+end