aipp 1.0.0 → 2.0.0

data/lib/aipp/executable.rb

@@ -1,60 +1,28 @@
 module AIPP
 
-  # Executable instantiated by the console tools
+  # @abstract
   class Executable
     include AIPP::Debugger
 
-    attr_reader :options
-
-    def initialize(**options)
-      @options = options.merge(
-        airac: AIRAC::Cycle.new,
-        region_options: [],
+    def initialize(exe_file)
+      AIPP.options.replace(
+        schema: exe_file.split('2').last.to_sym,
         storage: Pathname(Dir.home).join('.aipp'),
+        clean: false,
         force: false,
         mid: false,
+        quiet: false,
         verbose: false,
         debug_on_warning: false,
         debug_on_error: false
       )
-      OptionParser.new do |o|
-        o.banner = <<~END
-          Download online AIP and convert it to #{options[:schema].upcase}.
-          Usage: #{File.basename($0)} [options]
-        END
-        o.on('-d', '--airac (DATE|INTEGER)', String, %Q[AIRAC date or delta e.g. "+1" (default: "#{@options[:airac].date.xmlschema}")]) { @options[:airac] = airac_for(_1) }
-        o.on('-r', '--region STRING', String, 'region (e.g. "LF")') { @options[:region] = _1.upcase }
-        o.on('-a', '--aip STRING', String, 'process this AIP only (e.g. "ENR-5.1")') { @options[:aip] = _1 }
-        if options[:schema] == :ofmx
-          o.on('-g', '--[no-]grouped-obstacles', 'group obstacles (default: false)') { @options[:grouped_obstacles] = _1 }
-          o.on('-m', '--[no-]mid', 'insert mid attributes into all Uid elements (default: false)') { @options[:mid] = _1 }
-        end
-        o.on('-o', '--region-options STRING', String, %Q[comma separated region specific options]) { @options[:region_options] = _1.split(',') }
-        o.on('-s', '--storage DIR', String, 'storage directory (default: "~/.aipp")') { @options[:storage] = Pathname(_1) }
-        o.on('-h', '--[no-]check-links', 'check all links with HEAD requests') { @options[:check_links] = _1 }
-        o.on('-f', '--[no-]force', 'ignore XML schema validation (default: false)') { @options[:force] = _1 }
-        o.on('-v', '--[no-]verbose', 'verbose output including unsevere warnings (default: false)') { @options[:verbose] = _1 }
-        o.on('-w', '--debug-on-warning [ID]', Integer, 'open debug session on warning with ID (default: false)') { @options[:debug_on_warning] = _1 || true }
-        o.on('-e', '--[no-]debug-on-error', 'open debug session on error (default: false)') { @options[:debug_on_error] = _1 }
-        o.on('-A', '--about', 'show author/license information and exit') { about }
-        o.on('-R', '--readme', 'show README and exit') { readme }
-        o.on('-L', '--list', 'list implemented regions and AIPs') { list }
-        o.on('-V', '--version', 'show version and exit') { version }
-      end.parse!
     end
 
     def run
-      with_debugger(**options.slice(:verbose, :debug_on_warning, :debug_on_error)) do
+      with_debugger do
+        String.disable_colorization = !STDOUT.tty?
         starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        AIPP::Parser.new(options: options).tap do |parser|
-          parser.read_config
-          parser.read_region
-          parser.parse_aip
-          parser.validate_aixm
-          parser.write_build
-          parser.write_aixm
-          parser.write_config
-        end
+        [:AIPP, AIPP.options.module, :Runner].constantize.new.run
         ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         info("finished after %s" % Time.at(ending - starting).utc.strftime("%H:%M:%S"))
       end
@@ -62,12 +30,28 @@ module AIPP
 
     private
 
-    def airac_for(argument)
-      if argument.match?(/^[+-]\d+$/)   # delta
-        AIRAC::Cycle.new + argument.to_i
-      else   # date
-        AIRAC::Cycle.new(argument)
+    def common_options(o)
+      o.on('-r', '--region STRING', String, 'region (e.g. "LF")') { AIPP.options.region = _1.upcase }
+      o.on('-s', '--section STRING', String, 'process this section only') { AIPP.options.section = _1.classify }
+      o.on('-d', '--storage DIR', String, 'storage directory (default: "~/.aipp")') { AIPP.options.storage = Pathname(_1) }
+      o.on('-o', '--output FILE', String, 'output file') { AIPP.options.output_file = _1 }
+    end
+
+    def developer_options(o)
+      if AIPP.options.schema == :ofmx
+        o.on('-m', '--[no-]mid', 'insert mid attributes into all Uid elements (default: false)') { AIPP.options.mid = _1 }
       end
+      o.on('-h', '--[no-]check-links', 'check all links with HEAD requests') { AIPP.options.check_links = _1 }
+      o.on('-c', '--[no-]clean', 'clean cache and download from sources anew (default: false)') { AIPP.options.clean = _1 }
+      o.on('-f', '--[no-]force', 'continue on non-fatal errors (default: false)') { AIPP.options.force = _1 }
+      o.on('-q', '--[no-]quiet', 'suppress all informational output (default: false)') { AIPP.options.quiet = _1 }
+      o.on('-v', '--[no-]verbose', 'verbose output including unsevere warnings (default: false)') { AIPP.options.verbose = _1 }
+      o.on('-w', '--debug-on-warning [ID]', Integer, 'open debug session on warning with ID (default: false)') { AIPP.options.debug_on_warning = _1 || true }
+      o.on('-e', '--[no-]debug-on-error', 'open debug session on error (default: false)') { AIPP.options.debug_on_error = _1 }
+      o.on('-A', '--about', 'show author/license information and exit') { about }
+      o.on('-R', '--readme', 'show README and exit') { readme }
+      o.on('-L', '--list', 'list implemented regions') { list }
+      o.on('-V', '--version', 'show version and exit') { version }
     end
 
     def about
@@ -76,19 +60,18 @@ module AIPP
     end
 
     def readme
-      readme_path = Pathname($0).dirname.join('..', 'gems', "aipp-#{AIPP::VERSION}", 'README.md')
-      puts IO.read(readme_path)
+      puts IO.read(Pathname(__dir__).join('README.md'))
       exit
     end
 
     def list
-      regions_path = Pathname($0).dirname.join('..', 'gems', "aipp-#{AIPP::VERSION}", 'lib', 'aipp', 'regions')
-      hash = Dir.each_child(regions_path).each.with_object({}) do |region, hash|
-        hash[region] = Dir.children(regions_path.join(region)).sort.map do |aip|
-          File.basename(aip, '.rb') if File.file?(regions_path.join(region, aip))
+      hash = Pathname(__dir__).join('regions').glob('*').each.with_object({}) do |dir, hash|
+        region = "Sections for region #{dir.basename}"
+        hash[region] = dir.join(AIPP.options.module.downcase).glob('*.rb').map do |file|
+          File.basename(file, '.rb')
         end.compact
       end
-      puts hash.to_yaml.sub(/\A\W*/, '')
+      puts hash.to_yaml.lines[1..]
       exit
     end
 

data/lib/aipp/notam/README.md

@@ -0,0 +1,25 @@
+# AIPP NOTAM Module
+
+## Cache Time Window
+
+The default time window for NOTAM is the hour of day. This means:
+
+* Source data is downloaded and cached based on the hour of the day.
+* The effective date and time is rounded down to the previous full hour.
+
+To force a rebuild within this time window, you have to clean the cache using the `-c` command line argument.
+
+### Soft Fail and Crossload
+
+Malformed NOTAM which cannot be processed normally cause the build to fail. The `-f` command line argument changes this behaviour to skip the malformed NOTAM, issue a warning and continue.
+
+To fix broken NOTAM, you can set a crossload directory with `-x`. The contents of this directory must adhere to the following convention:
+
+```
+/                          ⬅︎ custom crossload directory
+├── LS                     ⬅︎ region
+│   └── W2479_22.txt       ⬅︎ NOTAM ID (replace "/" with "_")
+└── ED                     ⬅︎ other region
+```
+
+If a matching file is found in the crossload directory, it will be used in place of the original, malformed NOTAM.

data/lib/aipp/notam/executable.rb

@@ -0,0 +1,27 @@
+module AIPP
+  module NOTAM
+
+    class Executable < AIPP::Executable
+
+      def initialize(exe_file)
+        super
+        now = Time.now.utc.round
+        AIPP.options.merge(
+          module: 'NOTAM',
+          effective_at: now - now.sec - (now.min * 60)   # previous full hour
+        )
+        OptionParser.new do |o|
+          o.banner = <<~END
+            Download online NOTAM and convert it to #{AIPP.options.schema.upcase}.
+            Usage: #{File.basename($0)} [options]
+          END
+          common_options(o)
+          o.on('-t', '--effective (TIME)', String, %Q[effective after this point in time (default: #{AIPP.options.effective_at})]) { AIPP.options.effective_at = Time.parse(_1) }
+          o.on('-x', '--crossload DIR', String, 'crossload directory') { AIPP.options.crossload = Pathname(_1) }
+          developer_options(o)
+        end.parse!
+      end
+
+    end
+  end
+end

data/lib/aipp/notam/parser.rb

@@ -0,0 +1,9 @@
+module AIPP
+  module NOTAM
+
+    # @abstract
+    class Parser < AIPP::Parser
+    end
+
+  end
+end

data/lib/aipp/notam/runner.rb

@@ -0,0 +1,28 @@
+module AIPP
+  module NOTAM
+
+    class Runner < AIPP::Runner
+
+      def effective_at
+        AIPP.options.effective_at
+      end
+
+      def expiration_at
+        effective_at.end_of_day.round - 1
+      end
+
+      def run
+        info("NOTAM effective #{effective_at}", color: :green)
+        read_config
+        read_region
+        read_parsers
+        parse_sections
+        validate_aixm
+        write_aixm(AIPP.options.output_file || output_file)
+        write_config
+      end
+
+    end
+
+  end
+end

data/lib/aipp/parser.rb

@@ -1,196 +1,169 @@
 module AIPP
 
-  # AIP parser infrastructure
+  # @abstract
   class Parser
-    extend Forwardable
     include AIPP::Debugger
-    using AIXM::Refinements
+    include AIPP::Patcher
 
-    # @return [Hash] passed command line arguments
-    attr_reader :options
+    # @return [AIXM::Document] AIXM document instance
+    attr_reader :aixm
 
-    # @return [Hash] configuration read from config.yml
-    attr_reader :config
+    class << self
+      # Declare a dependency
+      #
+      # @param dependencies [Array<String>] class names of other parsers this
+      #   parser depends on
+      def depends_on(*dependencies)
+        @dependencies = dependencies.map(&:to_s)
+      end
 
-    # @return [AIXM::Document] target document
-    attr_reader :aixm
+      # Declared dependencies
+      #
+      # @return [Array<String>] class names of other parsers this parser
+      #   depends on
+      def dependencies
+        @dependencies || []
+      end
+    end
 
-    # @return [Hash] map from AIP name to fixtures
-    attr_reader :fixtures
-
-    # @return [Hash] map from border names to border objects
-    attr_reader :borders
-
-    # @return [OpenStruct] object cache
-    attr_reader :cache
-
-    def initialize(options:)
-      @options = options
-      @options[:storage] = options[:storage].join(options[:region])
-      @options[:storage].mkpath
-      @config = {}
-      @aixm = AIXM.document(effective_at: @options[:airac].date)
-      @dependencies = THash.new
-      @fixtures = {}
-      @borders = {}
-      @cache = OpenStruct.new
-      AIXM.send("#{options[:schema]}!")
-      AIXM.config.region = options[:region]
+    def initialize(downloader:, aixm:)
+      @downloader, @aixm = downloader, aixm
+      setup if respond_to? :setup
     end
 
     # @return [String]
     def inspect
-      "#<AIPP::Parser>"
+      "#<AIPP::Parser #{section}>"
     end
 
-    # Read the configuration from config.yml.
-    def read_config
-      info("reading config.yml")
-      @config = YAML.load_file(config_file, symbolize_names: true, fallback: {}) if config_file.exist?
-      @config[:namespace] ||= SecureRandom.uuid
-      @aixm.namespace = @config[:namespace]
+    # @return [String]
+    def section
+      self.class.to_s.sectionize
     end
 
-    # Read the region directory and build the dependency list.
-    def read_region
-      info("reading region #{options[:region]}")
-      dir = Pathname(__FILE__).dirname.join('regions', options[:region])
-      fail("unknown region `#{options[:region]}'") unless dir.exist?
-      # Fixtures
-      dir.glob('fixtures/*.yml').each do |file|
-        verbose_info "reading fixture fixtures/#{file.basename}"
-        fixture = YAML.load_file(file)
-        @fixtures[file.basename('.yml').to_s] = fixture
-      end
-      # Borders
-      dir.glob('borders/*.geojson').each do |file|
-        verbose_info "reading border borders/#{file.basename}"
-        border = AIPP::Border.from_file(file)
-        @borders[file.basename] = border
-      end
-      # Helpers
-      dir.glob('helpers/*.rb').each do |file|
-        verbose_info "reading helper helpers/#{file.basename}"
-        require file
-      end
-      # Parsers
-      dir.glob('*.rb').each do |file|
-        verbose_info "requiring #{file.basename}"
-        require file
-        aip = file.basename('.*').to_s
-        @dependencies[aip] = ("AIPP::%s::%s::DEPENDS" % [options[:region], aip.remove(/\W/).camelcase]).constantize
-      end
+    # @abstract
+    def origin_for(*)
+      fail "origin_for method must be implemented in parser"
     end
 
-    # Parse AIP by invoking the parser classes for the current region.
-    def parse_aip
-      info("AIRAC #{options[:airac].id} effective #{options[:airac].date}", color: :green)
-      AIPP::Downloader.new(storage: options[:storage], source: options[:airac].date.xmlschema) do |downloader|
-        @dependencies.tsort(options[:aip]).each do |aip|
-          info("parsing #{aip}")
-          ("AIPP::%s::%s" % [options[:region], aip.remove(/\W/).camelcase]).constantize.new(
-            aip: aip,
-            downloader: downloader,
-            fixture: @fixtures[aip],
-            parser: self
-          ).attach_patches.tap(&:parse).detach_patches
-        end
-      end
-      if options[:grouped_obstacles]
-        info("grouping obstacles")
-        aixm.group_obstacles!
-      end
-      info("counting #{aixm.features.count} features")
+    # Read a source document
+    #
+    # Read the cached document if it exists in the source archive. Otherwise,
+    # download and cache it.
+    #
+    # An origin builder method +origin_for+ must be implemented by the parser
+    # definition.
+    #
+    # @param document [String] e.g. "ENR-2.1" or "aerodromes" (default: current
+    #   +section+)
+    # @return [Nokogiri::XML::Document, Nokogiri::HTML5::Document,
+    #   Roo::Spreadsheet, String] document
+    def read(document=section)
+      @downloader.read(
+        document: document,
+        origin: origin_for(document)
+      )
     end
 
-    # Validate the AIXM document.
+    # Add feature to AIXM
     #
-    # @raise [RuntimeError] if the document is not valid
-    def validate_aixm
-      info("detecting duplicates")
-      if (duplicates = aixm.features.duplicates).any?
-        message = "duplicates found:\n" + duplicates.map { "#{_1.inspect} from #{_1.source}" }.join("\n")
-        @options[:force] ? warn(message) : fail(message)
-      end
-      info("validating #{options[:schema].upcase}")
-      unless aixm.valid?
-        message = "invalid #{options[:schema].upcase} document:\n" + aixm.errors.map(&:message).join("\n")
-        @options[:force] ? warn(message) : fail(message)
-      end
+    # @param feature [AIXM::Feature] e.g. airport or airspace
+    # @return [AIXM::Feature] added feature
+    def add(feature)
+      verbose_info "adding #{feature.inspect}"
+      aixm.add_feature feature
+      feature
     end
 
-    # Write the AIXM document and context information.
-    def write_build
-      if @options[:aip]
-        info ("skipping build")
-      else
-        info("writing build")
-        builds_path.mkpath
-        build_file = builds_path.join("#{@options[:airac].date.xmlschema}.zip")
-        Dir.mktmpdir do |tmp_dir|
-          tmp_dir = Pathname(tmp_dir)
-          # AIXM/OFMX file
-          AIXM.config.mid = true
-          File.write(tmp_dir.join(aixm_file), aixm.to_xml)
-          # Build details
-          File.write(
-            tmp_dir.join('build.yaml'), {
-              version: AIPP::VERSION,
-              config: @config,
-              options: @options
-            }.to_yaml
-          )
-          # Manifest
-          manifest = ['AIP','Feature', 'Comment', 'Short Uid Hash', 'Short Feature Hash'].to_csv
-          manifest += aixm.features.map do |feature|
-            xml = feature.to_xml
-            element = xml.first_match(/<(\w{3})\s/)
-            [
-              feature.source.split('|')[2],
-              element,
-              xml.match(/<!-- (.*?) -->/)[1],
-              AIXM::PayloadHash.new(xml.match(%r(<#{element}Uid\s.*?</#{element}Uid>)m).to_s).to_uuid[0,8],
-              AIXM::PayloadHash.new(xml).to_uuid[0,8]
-            ].to_csv
-          end.sort.join
-          File.write(tmp_dir.join('manifest.csv'), manifest)
-          # Zip it
-          build_file.delete if build_file.exist?
-          Zip::File.open(build_file, Zip::File::CREATE) do |zip|
-            tmp_dir.children.each do |entry|
-              zip.add(entry.basename.to_s, entry) unless entry.basename.to_s[0] == '.'
-            end
-          end
-        end
+    # @!method find_by(klass, attributes={})
+    #   Find objects of the given class and optionally with the given attribute
+    #   values previously written to AIXM.
+    #
+    #   @note This method is delegated to +AIXM::Association::Array+.
+    #   @see https://www.rubydoc.info/gems/aixm/AIXM/Association/Array#find_by-instance_method
+    #
+    # @!method find(object)
+    #   Find equal objects previously written to AIXM.
+    #
+    #   @note This method is delegated to +AIXM::Association::Array+.
+    #   @see https://www.rubydoc.info/gems/aixm/AIXM/Association/Array#find-instance_method
+    %i(find_by find).each do |method|
+      define_method method do |*args|
+        aixm.features.send(method, *args)
       end
     end
 
-    # Write the AIXM document.
-    def write_aixm
-      info("writing #{aixm_file}")
-      AIXM.config.mid = options[:mid]
-      File.write(aixm_file, aixm.to_xml)
+    # @overload given(*objects)
+    #   Return +objects+ unless at least one of them equals nil
+    #
+    #   @example
+    #     # Instead of this:
+    #     first, last = unless ((first = expensive_first).nil? || (last = expensive_last).nil?)
+    #       [first, last]
+    #     end
+    #
+    #     # Use the following:
+    #     first, last = given(expensive_first, expensive_last)
+    #
+    #   @param *objects [Array<Object>] any objects really
+    #   @return [Object] nil if at least one of the objects is nil, given
+    #     objects otherwise
+    #
+    # @overload given(*objects)
+    #   Yield +objects+ unless at least one of them equals nil
+    #
+    #   @example
+    #     # Instead of this:
+    #     name = unless ((first = expensive_first.nil? || (last = expensive_last.nil?)
+    #       "#{first} #{last}"
+    #     end
+    #
+    #     # Use any of the following:
+    #     name = given(expensive_first, expensive_last) { |f, l| "#{f} #{l}" }
+    #     name = given(expensive_first, expensive_last) { "#{_1} #{_2}" }
+    #
+    #   @param *objects [Array<Object>] any objects really
+    #   @yield [Array<Object>] objects passed as parameter
+    #   @return [Object] nil if at least one of the objects is nil, return of
+    #     block otherwise
+    def given(*objects)
+      if objects.none?(&:nil?)
+        block_given? ? yield(*objects) : objects
+      end
     end
 
-    # Write the configuration to config.yml.
-    def write_config
-      info("writing config.yml")
-      File.write(config_file, config.to_yaml)
+    # Build and optionally check a Markdown link
+    #
+    # @example
+    #   AIPP.options.check_links = false
+    #   link_to('foo', 'https://bar.com/exists')      # => "[foo](https://bar.com/exists)"
+    #   link_to('foo', 'https://bar.com/not-found')   # => "[foo](https://bar.com/not-found)"
+    #   AIPP.options.check_links = true
+    #   link_to('foo', 'https://bar.com/exists')      # => "[foo](https://bar.com/exists)"
+    #   link_to('foo', 'https://bar.com/not-found')   # => nil
+    #
+    # @param body [String] body text of the link
+    # @param url [String] URL of the link
+    # @return [String, nil] Markdown link
+    def link_to(body, url)
+      "[#{body}](#{url})" if !AIPP.options.check_links || url_exists?(url)
     end
 
     private
 
-    def aixm_file
-      "#{options[:region]}_#{options[:airac].date.xmlschema}.#{options[:schema]}"
-    end
-
-    def builds_path
-      options[:storage].join('builds')
+    def url_exists?(url)
+      uri = URI.parse(url)
+      Net::HTTP.new(uri.host, uri.port).tap do |request|
+        request.use_ssl = (uri.scheme == 'https')
+        path = uri.path.present? ? uri.path : '/'
+        result = request.request_head(path)
+        if result.kind_of? Net::HTTPRedirection
+          url_exist?(result['location'])
+        else
+          result.code == '200'
+        end
+      end
     end
 
-    def config_file
-      options[:storage].join('config.yml')
-    end
   end
-
 end

data/lib/aipp/patcher.rb

@@ -1,9 +1,9 @@
 module AIPP
   module Patcher
 
-    def self.included(klass)
-      klass.extend(ClassMethods)
-      klass.class_variable_set(:@@patches, {})
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.class_variable_set(:@@patches, {})
     end
 
     module ClassMethods
@@ -17,14 +17,13 @@ module AIPP
     end
 
     def attach_patches
-      parser = self
       verbose_info_method = method(:verbose_info)
       self.class.patches[self.class]&.each do |(klass, attribute, block)|
         klass.instance_eval do
           alias_method :"original_#{attribute}=", :"#{attribute}="
           define_method(:"#{attribute}=") do |value|
             error = catch :abort do
-              value = block.call(parser, self, value)
+              value = block.call(self, value)
               verbose_info_method.call("Patching #{self.inspect} with #{attribute}=#{value.inspect}", color: :magenta)
             end
             fail "patching #{self.inspect} with #{attribute}=#{value.inspect} failed: #{error}" if error

data/lib/aipp/regions/LF/README.md

@@ -1,4 +1,4 @@
-# AIP LF – Mainland France
+# LF – France Mainland
 
 ## Prerequisites
 
@@ -7,7 +7,7 @@ This parser requires the XML data dump from SIA. It is available free of charge,
 1. Browse to the [SIA web shop](https://www.sia.aviation-civile.gouv.fr/produits-numeriques-en-libre-disposition/les-bases-de-donnees-sia.html).
 2. Shop the desired dump named «données aéronautiques XML AIRAC ii/yy».
 3. Browse to [your customer page](https://www.sia.aviation-civile.gouv.fr/customer/account/#orders-and-proposals).
-4. In section «mes produits téléchargeables» download the desired dump.
+4. On page «mes produits téléchargeables» download the desired dump.
 5. Unzip the downloaded ZIP archive.
 6. Move the file «XML_SIA_yyyy-mm-dd.xml» to the directory in which you will execute the parser.
 
@@ -41,6 +41,10 @@ grep "<Revetement>" XML_SIA_2021-12-02_UTF.xml | sort | uniq
 (...)
 ```
 
+## Error Reports
+
+Feedback and error reports should be sent to SIA. However, their [contact form](https://www.sia.aviation-civile.gouv.fr/contact) is sometimes broken. If you don't receive an automatic reception confirmation, you should send it directly to sia-qualite@aviation-civile.gouv.fr instead.
+
 ## References
 
 * [SIA – AIP publisher](https://www.sia.aviation-civile.gouv.fr)