aipp 0.2.1 → 0.2.2

data/lib/aipp/downloader.rb

@@ -0,0 +1,115 @@
+module AIPP
+
+  # AIP downloader infrastructure
+  #
+  # The downloader operates in the +storage+ directory where it creates two
+  # subdirectories "archive" and "work". The initializer looks for +archive+
+  # in "archives" and (if found) unzips its contents into "work". When reading
+  # a +document+, the downloader looks for the +document+ in "work" and
+  # (unless found) downloads it from +url+. HTML documents are parsed to
+  # +Nokogiri::HTML5::Document+, PDF documents are parsed to +AIPP::PDF+.
+  # Finally, the contents of "work" are written back to +archive+.
+  #
+  # @example
+  #   AIPP::Downloader.new(storage: options[:storage], archive: "2018-11-08") do |downloader|
+  #     html = downloader.read(
+  #       document: 'ENR-5.1',
+  #       url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/FRANCE/AIRAC-2018-11-08/html/eAIP/FR-ENR-5.1-fr-FR.html'
+  #     )
+  #     pdf = downloader.read(
+  #       document: 'VAC-LFMV',
+  #       url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/Atlas-VAC/PDF_AIPparSSection/VAC/AD/AD-2.LFMV.pdf'
+  #     )
+  #   end
+  class Downloader
+
+    # @return [Pathname] directory to operate within
+    attr_reader :storage
+
+    # @return [String] name of the archive (without extension ".zip")
+    attr_reader :archive
+
+    # @return [Pathname] full path to the archive
+    attr_reader :archive_file
+
+    # @param storage [Pathname] directory to operate within
+    # @param archive [String] name of the archive (without extension ".zip")
+    def initialize(storage:, archive:)
+      @storage, @archive = storage, archive
+      fail(ArgumentError, 'bad storage directory') unless Dir.exist? storage
+      @archive_file = archives_path.join("#{@archive}.zip")
+      prepare
+      unzip if @archive_file.exist?
+      yield self
+      zip
+    ensure
+      teardown
+    end
+
+    # Download and read +document+
+    #
+    # @param document [String] document to read (without extension)
+    # @param url [String] URL to download the document from
+    # @param type [Symbol, nil] document type: +nil+ (default) to derive it from
+    #   the URL, :html, or :pdf
+    # @return [Nokogiri::HTML5::Document, AIPP::PDF]
+    def read(document:, url:, type: nil)
+      type ||= Pathname(URI(url).path).extname[1..-1].to_sym
+      file = work_path.join([document, type].join('.'))
+      unless file.exist?
+        debug "Downloading #{document}"
+        IO.copy_stream(Kernel.open(url), file)
+      end
+      convert file
+    end
+
+    private
+
+    def archives_path
+      @storage.join('archives')
+    end
+
+    def work_path
+      @storage.join('work')
+    end
+
+    def prepare
+      teardown
+      archives_path.mkpath
+      work_path.mkpath
+    end
+
+    def teardown
+      if work_path.exist?
+        work_path.children.each(&:delete)
+        work_path.delete
+      end
+    end
+
+    def unzip
+      Zip::File.open(archive_file).each do |entry|
+        entry.extract(work_path.join(entry.name))
+      end
+    end
+
+    def zip
+      backup_file = archive_file.sub(/$/, '.old') if archive_file.exist?
+      archive_file.rename(backup_file) if backup_file
+      Zip::File.open(archive_file, Zip::File::CREATE) do |zip|
+        work_path.children.each do |entry|
+          zip.add(entry.basename.to_s, entry) unless entry.basename.to_s[0] == '.'
+        end
+      end
+      backup_file&.delete
+    end
+
+    def convert(file)
+      case file.extname
+        when '.html' then Nokogiri.HTML5(file)
+        when '.pdf' then AIPP::PDF.new(file)
+      else
+        fail(ArgumentError, "invalid document type")
+      end
+    end
+  end
+end

data/lib/aipp/executable.rb

@@ -8,7 +8,7 @@ module AIPP
       @options = options
       @options[:airac] = AIPP::AIRAC.new
       @options[:storage] = Pathname(Dir.home).join('.aipp')
-      @options[:verbose] = @options[:force] = @options[:pry_on_error] = false
+      @options[:force] = $PRY_ON_WARN = $PRY_ON_ERROR = false
       OptionParser.new do |o|
         o.banner = <<~END
           Download online AIP and convert it to #{options[:schema].upcase}.
@@ -18,10 +18,10 @@ module AIPP
         o.on('-r', '--region STRING', String, 'region (e.g. "LF")') { |v| @options[:region] = v.upcase }
         o.on('-a', '--aip STRING', String, 'process this AIP only (e.g. "ENR-5.1")') { |v| @options[:aip] = v.upcase }
         o.on('-s', '--storage DIR', String, 'storage directory (default: "~/.aipp")') { |v| @options[:storage] = Pathname(v) }
-        o.on('-v', '--[no-]verbose', 'verbose process output (default: false)') { |v| @options[:verbose] = v }
         o.on('-f', '--[no-]force', 'ignore XML schema validation (default: false)') { |v| @options[:force] = v }
-        o.on('-W', '--pry-on-warn ID', Integer, 'open pry on warn with ID (default: nil)') { |v| @options[:pry_on_warn] = v }
-        o.on('-E', '--[no-]pry-on-error', 'open pry on error (default: false)') { |v| @options[:pry_on_error] = v }
+        o.on('-v', '--[no-]verbose', 'verbose output and Ruby debug mode (default: false)') { |v| $DEBUG = v }
+        o.on('-w', '--pry-on-warn [ID]', Integer, 'open pry on warn with ID (default: nil)') { |v| $PRY_ON_WARN = v || true }
+        o.on('-e', '--[no-]pry-on-error', 'open pry on error (default: false)') { |v| $PRY_ON_ERROR = v }
         o.on('-A', '--about', 'show author/license information and exit') { about }
         o.on('-R', '--readme', 'show README and exit') { readme }
         o.on('-L', '--list', 'list implemented regions and AIPs') { list }
@@ -33,7 +33,7 @@ module AIPP
     #
     # @raise [RuntimeError] if the region does not exist
     def run
-      Pry::rescue do
+      Pry.rescue do
         fail(OptionParser::MissingArgument, :region) unless options[:region]
         AIPP::Parser.new(options: options).tap do |parser|
           parser.read_config
@@ -45,7 +45,7 @@ module AIPP
         end
       rescue => error
         puts "ERROR: #{error.message}"
-        Pry::rescued(error) if options[:pry_on_error]
+        Pry::rescued(error) if $PRY_ON_ERROR
       end
     end
 

data/lib/aipp/parser.rb

@@ -2,8 +2,6 @@ module AIPP
 
   # AIP parser infrastructure
   class Parser
-    using AIPP::Refinements
-    include AIPP::Progress
 
     # @return [Hash] passed command line arguments
     attr_reader :options
@@ -20,16 +18,16 @@ module AIPP
     def initialize(options:)
       @options = options
       @options[:storage] = options[:storage].join(options[:region])
-      @options[:storage].mkpath
+      @options[:storage].mkpath unless @options[:storage].exist?
       @config = {}
       @aixm = AIXM.document(region: @options[:region], effective_at: @options[:airac].date)
-      @cache = OpenStruct.new
       @dependencies = THash.new
+      @cache = OpenStruct.new
     end
 
     # Read the configuration from config.yml.
     def read_config
-      info("Reading config.yml", force: true)
+      info("Reading config.yml")
       @config = YAML.load_file(config_file, fallback: {}).transform_keys(&:to_sym) if config_file.exist?
       @config[:namespace] ||= SecureRandom.uuid
       @aixm.namespace = @config[:namespace]
@@ -37,29 +35,30 @@ module AIPP
 
     # Read the region directory and build the dependency list.
     def read_region
-      info("Reading region #{options[:region]}", force: true)
+      info("Reading region #{options[:region]}")
       dir = Pathname(__FILE__).dirname.join('regions', options[:region])
       fail("unknown region `#{options[:region]}'") unless dir.exist?
+      dir.glob('helpers/*.rb').each { |f| require f }
       dir.glob('*.rb').each do |file|
-        info("Requiring #{file.basename}")
+        debug("Requiring #{file.basename}")
         require file
-        if (aip = file.basename('.*').to_s) == 'helper'
-          extend [:AIPP, options[:region], :Helper].constantize
-        else
-          @dependencies[aip] = [:AIPP, options[:region], aip.classify, :DEPENDS].constantize
-        end
+        aip = file.basename('.*').to_s
+        @dependencies[aip] = ("AIPP::%s::%s::DEPENDS" % [options[:region], aip.remove(/\W/).classify]).constantize
       end
     end
 
     # Parse AIP by invoking the parser classes for the current region.
     def parse_aip
-      info("AIRAC #{options[:airac].id} effective #{options[:airac].date}", force: true, color: :green)
-      @dependencies.tsort(options[:aip]).each do |aip|
-        info("Parsing #{aip}", force: true)
-        [:AIPP, options[:region], aip.classify].constantize.new(
-          aip: aip,
-          parser: self
-        ).parse
+      info("AIRAC #{options[:airac].id} effective #{options[:airac].date}", color: :green)
+      AIPP::Downloader.new(storage: options[:storage], archive: options[:airac].date.xmlschema) do |downloader|
+        @dependencies.tsort(options[:aip]).each do |aip|
+          info("Parsing #{aip}")
+          ("AIPP::%s::%s" % [options[:region], aip.remove(/\W/).classify]).constantize.new(
+            aip: aip,
+            downloader: downloader,
+            parser: self
+          ).attach_patches.tap(&:parse).detach_patches
+        end
       end
     end
 
@@ -67,23 +66,24 @@ module AIPP
     #
     # @raise [RuntimeError] if the document is not valid
     def validate_aixm
-      info("Validating #{options[:schema].upcase}", force: true)
+      info("Validating #{options[:schema].upcase}")
       unless aixm.valid?
-        send(@options[:force] ? :warn : :fail, "invalid AIXM document:\n#{aixm.errors}")
+        message = "invalid AIXM document:\n" + aixm.errors.map(&:message).join("\n")
+        @options[:force] ? warn(message, pry: binding) : fail(message)
       end
     end
 
     # Write the AIXM document.
     def write_aixm
       file = "#{options[:region]}_#{options[:airac].date.xmlschema}.#{options[:schema]}"
-      info("Writing #{file}", force: true)
+      info("Writing #{file}")
       AIXM.send("#{options[:schema]}!")
       File.write(file, aixm.to_xml)
     end
 
     # Write the configuration to config.yml.
     def write_config
-      info("Writing config.yml", force: true)
+      info("Writing config.yml")
       File.write(config_file, config.to_yaml)
     end
 

data/lib/aipp/patcher.rb

@@ -0,0 +1,47 @@
+module AIPP
+  module Patcher
+
+    def self.included(klass)
+      klass.extend(ClassMethods)
+      klass.class_variable_set(:@@patches, {})
+    end
+
+    module ClassMethods
+      def patches
+        class_variable_get(:@@patches)
+      end
+
+      def patch(klass, attribute, &block)
+        (patches[self] ||= []) << [klass, attribute, block]
+      end
+    end
+
+    def attach_patches
+      parser = self
+      self.class.patches[self.class]&.each do |(klass, attribute, block)|
+        klass.instance_eval do
+          alias_method :"original_#{attribute}=", :"#{attribute}="
+          define_method(:"#{attribute}=") do |value|
+            catch :abort do
+              value = block.call(parser, self, value)
+              debug("PATCH: #{self.inspect}", color: :magenta)
+            end
+            send(:"original_#{attribute}=", value)
+          end
+        end
+      end
+      self
+    end
+
+    def detach_patches
+      self.class.patches[self.class]&.each do |(klass, attribute, _)|
+        klass.instance_eval do
+          alias_method :"#{attribute}=", :"original_#{attribute}="
+          remove_method :"original_#{attribute}="
+        end
+      end
+      self
+    end
+
+  end
+end

data/lib/aipp/pdf.rb

@@ -0,0 +1,123 @@
+module AIPP
+
+  # PDF to text reader with support for pages and fencing
+  #
+  # @example
+  #   pdf = AIPP::PDF.new("/path/to/file.pdf")
+  #   pdf.file   # => #<Pathname:/path/to/file.pdf>
+  #   pdf.from(100).to(200).each_line_with_position do |line, page, last|
+  #     line   # => line content (e.g. "first line")
+  #     page   # => page number (e.g. 1)
+  #     last   # => last line boolean (true for last line, false otherwise)
+  #   end
+  class PDF
+    attr_reader :file
+
+    def initialize(file, cache: true)
+      @file = file.is_a?(Pathname) ? file : Pathname(file)
+      @text, @page_ranges = cache ? read_cache : read
+      @from = 0
+      @to = @last = @text.length - 1
+    end
+
+    # @return [String]
+    def inspect
+      %Q(#<#{self.class} file=#{@file} range=#{range}>)
+    end
+
+    # Fence the PDF beginning with this index
+    #
+    # @param index [Integer, Symbol] either an integer position within the
+    #   +text+ string or +:begin+ to indicate "first existing position"
+    # @return [self]
+    def from(index)
+      index = 0 if index == :begin
+      fail ArgumentError unless (0..@to).include? index
+      @from = index
+      self
+    end
+
+    # Fence the PDF ending with this index
+    #
+    # @param index [Integer, Symbol] either an integer position within the
+    #   +text+ string or +:end+ to indicate "last existing position"
+    # @return [self]
+    def to(index)
+      index = @last if index == :end
+      fail ArgumentError unless (@from..@last).include? index
+      @to = index
+      self
+    end
+
+    # Get the current fencing range
+    #
+    # @return [Range<Integer>]
+    def range
+      (@from..@to)
+    end
+
+    # Text string of the PDF with fencing applied
+    #
+    # @return [String] PDF converted to string
+    def text
+      @text[range]
+    end
+
+    # Text split to individual lines
+    #
+    # @return [Array] lines
+    def lines
+      text.split(/(?<=[\n\f])/)
+    end
+
+    # Executes the block for every line and passes the line content, page
+    # number and end of document boolean.
+    #
+    # If no block is given, an enumerator is returned instead.
+    #
+    # @yieldparam line [String] content of the line
+    # @yieldparam page [Integer] page number the line is found on within the PDF
+    # @yieldparam last [Boolean] true for the last line, false otherwise
+    # @return [Enumerator]
+    def each_line
+      return enum_for(:each) unless block_given?
+      offset, last_line_index = @from, lines.count - 1
+      lines.each_with_index do |line, line_index|
+        yield(line, page_for(index: offset), line_index == last_line_index)
+        offset += line.length
+      end
+    end
+    alias_method :each, :each_line
+
+    private
+
+    def read
+      pages = ::PDF::Reader.new(@file).pages
+      [pages.map(&:text).join("\f"), page_ranges_for(pages)]
+    end
+
+    def read_cache
+      cache_file = "#{@file}.json"
+      if File.exist?(cache_file) && (File.stat(@file).mtime - File.stat(cache_file).mtime).abs < 1
+        JSON.load File.read(cache_file)
+      else
+        read.tap do |data|
+          File.write(cache_file, data.to_json)
+          FileUtils.touch(cache_file, mtime: File.stat(@file).mtime)
+        end
+      end
+    end
+
+    def page_ranges_for(pages)
+      [].tap do |page_ranges|
+        pages.each_with_index  do |page, index|
+          page_ranges << (page_ranges.last || 0) + page.text.length + index
+        end
+      end
+    end
+
+    def page_for(index:)
+      @page_ranges.index(@page_ranges.bsearch { |i| i >= index }) + 1
+    end
+  end
+end

data/lib/aipp/regions/LF/AD-1.3.rb

@@ -0,0 +1,162 @@
+module AIPP
+  module LF
+
+    # Aerodromes
+    class AD13 < AIP
+
+      include AIPP::LF::Helpers::Common
+
+      DEPENDS = %w(AD-2)
+
+      # Map names of id-less airports to unofficial ids
+      ID_LESS_AIRPORTS = {
+        "ALBE" => 'LF9001',
+        "BEAUMONT DE LOMAGNE" => 'LF9002',
+        "BERDOUES" => 'LF9003',
+        "BOULOC" => 'LF9004',
+        "BUXEUIL ST REMY / CREUSE" => 'LF9005',
+        "CALVIAC" => 'LF9006',
+        "CAYLUS" => 'LF9007',
+        "CORBONOD" => 'LF9008',
+        "L'ISLE EN DODON" => 'LF9009',
+        "LACAVE LE FRAU" => 'LF9010',
+        "LUCON CHASNAIS" => 'LF9011',
+        "PEYRELEVADE" => 'LF9012',
+        "SAINT CYR LA CAMPAGNE" => 'LF9013',
+        "SEPTFONDS" => 'LF9014',
+        "TALMONT VENDEE AIR PARK" => 'LF9015'
+      }
+
+      def parse
+        ad2_exists = false
+        tbody = prepare(html: read).css('tbody').first   # skip altiports
+        tbody.css('tr').to_enum.with_index(1).each do |tr, index|
+          if tr.attr(:id).match?(/-TXT_NAME-/)
+            write @airport if @airport && !ad2_exists
+            @airport = airport_from tr
+            debug "Parsing #{@airport.id}"
+            ad2_exists = false
+            if airport = select(:airport, id: @airport.id).first
+              ad2_exists = true
+              @airport = airport
+            end
+            add_usage_limitations_from tr
+            next
+          end
+          @airport.add_runway(runway_from(tr)) unless ad2_exists
+        rescue => error
+          warn("error parsing #{@airport.id} at ##{index}: #{error.message}", pry: error)
+        end
+        write @airport if @airport && !ad2_exists
+      end
+
+      private
+
+      def airport_from(tr)
+        tds = tr.css('td')
+        id = tds[0].text.strip.blank_to_nil || ID_LESS_AIRPORTS.fetch(tds[1].text.strip)
+        AIXM.airport(
+          source: source(position: tr.line),
+          organisation: organisation_lf,   # TODO: not yet implemented
+          id: id,
+          name: tds[1].text.strip,
+          xy: xy_from(tds[3].text)
+        ).tap do |airport|
+          airport.z = AIXM.z(tds[4].text.strip.to_i, :qnh)
+          airport.declination = tds[2].text.remove('°').strip.to_f
+          airport.transition_z = AIXM.z(5000, :qnh)   # TODO: default - exceptions may exist
+        end
+      end
+
+      def add_usage_limitations_from(tr)
+        raw_limitation = tr.css('td:nth-of-type(8)').text.cleanup.downcase
+        raw_conditions = tr.css('td:nth-of-type(6)').text.cleanup.downcase.split(%r([\s/]+))
+        limitation = case raw_limitation
+          when /ouv.+cap|milit/ then :permitted
+          when /usa.+restr|priv/ then :reservation_required
+        end
+        @airport.add_usage_limitation(limitation) do |l|
+          l.add_condition do |c|
+            c.realm = :military if raw_limitation.match?(/milit/)
+            c.origin = :national if raw_conditions.include?('ntl')
+            c.origin = :international if raw_conditions.include?('intl')
+            c.rule = :ifr if raw_conditions.include?('ifr')
+            c.rule = :vfr if raw_conditions.include?('vfr')
+            c.purpose = :scheduled if raw_conditions.include?('s')
+            c.purpose = :not_scheduled if raw_conditions.include?('ns')
+            c.purpose = :private if raw_conditions.include?('p')
+          end
+          l.remarks = "Usage restreint (voir VAC) / restricted use (see VAC)" if raw_limitation.match?(/usa.+restr/)
+          l.remarks = "Propriété privée / privately owned" if raw_limitation.match?(/priv/)
+        end
+      end
+
+      def runway_from(tr)
+        tds = tr.css('td')
+        surface = tds[1].css('span[id*="SURFACE"]').text
+        AIXM.runway(
+          name: tds[0].text.strip.split.join('/')
+        ).tap do |runway|
+          @runway = runway   # TODO: needed for now for surface composition patches to work
+          runway.length = AIXM.d(tds[1].css('span[id$="VAL_LEN"]').text.to_i, :m)
+          runway.width = AIXM.d(tds[1].css('span[id$="VAL_WID"]').text.to_i, :m)
+          runway.surface.composition = (COMPOSITIONS.fetch(surface)[:composition] unless surface.blank?)
+          runway.surface.preparation = (COMPOSITIONS.fetch(surface)[:preparation] unless surface.blank?)
+          runway.remarks = tds[7].text.cleanup.blank_to_nil
+          values = tds[2].text.remove('°').strip.split
+          runway.forth.geographic_orientation = AIXM.a(values.first.to_i)
+          runway.back.geographic_orientation = AIXM.a(values.last.to_i)
+          parts = tds[3].text.strip.split(/\n\s+\n\s+/)
+          runway.forth.xy = (xy_from(parts[0]) unless parts[0].blank?)
+          runway.back.xy = (xy_from(parts[1]) unless parts[1].blank?)
+          values = tds[4].text.strip.split
+          runway.forth.z = AIXM.z(values.first.to_i, :qnh)
+          runway.back.z = AIXM.z(values.last.to_i, :qnh)
+          displaced_thresholds = displaced_thresholds_from(tds[5])
+          runway.forth.displaced_threshold = displaced_thresholds.first
+          runway.back.displaced_threshold = displaced_thresholds.last
+        end
+      end
+
+      def displaced_thresholds_from(td)
+        values = td.text.strip.split
+        case values.count
+          when 1 then []
+          when 2 then [AIXM.xy(lat: values[0], long: values[1]), nil]
+          when 3 then [nil, AIXM.xy(lat: values[1], long: values[2])]
+          when 4 then [AIXM.xy(lat: values[0], long: values[1]), AIXM.xy(lat: values[2], long: values[3])]
+          else fail "cannot parse displaced thresholds"
+        end
+      end
+
+      patch AIXM::Component::Runway, :width do |parser, object, value|
+        throw :abort unless value.zero?
+        @fixtures ||= YAML.load_file(Pathname(__FILE__).dirname.join('AD-1.3.yml'))
+        airport_id = parser.instance_variable_get(:@airport).id
+        runway_name = object.name.to_s
+        throw :abort if (width = @fixtures.dig('runways', airport_id, runway_name, 'width')).nil?
+        AIXM.d(width.to_i, :m)
+      end
+
+      patch AIXM::Component::Runway::Direction, :xy do |parser, object, value|
+        throw :abort unless value.nil?
+        @fixtures ||= YAML.load_file(Pathname(__FILE__).dirname.join('AD-1.3.yml'))
+        airport_id = parser.instance_variable_get(:@airport).id
+        direction_name = object.name.to_s
+        throw :abort if (xy = @fixtures.dig('runways', airport_id, direction_name, 'xy')).nil?
+        lat, long = xy.split(/\s+/)
+        AIXM.xy(lat: lat, long: long)
+      end
+
+      patch AIXM::Component::Surface, :composition do |parser, object, value|
+        throw :abort unless value.blank?
+        @fixtures ||= YAML.load_file(Pathname(__FILE__).dirname.join('AD-1.3.yml'))
+        airport_id = parser.instance_variable_get(:@airport).id
+        runway_name = parser.instance_variable_get(:@runway).name
+        throw :abort if (composition = @fixtures.dig('runways', airport_id, runway_name, 'composition')).nil?
+        composition
+      end
+
+    end
+  end
+end