aipp 1.0.0 → 2.0.0

data/exe/aip2aixm

@@ -1,11 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'bundler/inline'
+require 'aipp'
 
-gemfile do
-  source 'https://rubygems.org'
-  ruby '>= 3.0.0'
-  gem 'aipp', '~> 1'
-end
-
-AIPP::Executable.new(schema: File.basename($0)[4..-1].to_sym).run
+AIPP::AIP::Executable.new(File.basename($0)).run

data/exe/aip2ofmx

@@ -1,11 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'bundler/inline'
+require 'aipp'
 
-gemfile do
-  source 'https://rubygems.org'
-  ruby '>= 3.0.0'
-  gem 'aipp', '~> 1'
-end
-
-AIPP::Executable.new(schema: File.basename($0)[4..-1].to_sym).run
+AIPP::AIP::Executable.new(File.basename($0)).run

data/exe/notam2aixm

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'aipp'
+
+AIPP::NOTAM::Executable.new(File.basename($0)).run

data/exe/notam2ofmx

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'aipp'
+
+AIPP::NOTAM::Executable.new(File.basename($0)).run

data/lib/aipp/aip/README.md

@@ -0,0 +1,10 @@
+# AIPP AIP Module
+
+## Cache Time Window
+
+The default time window for AIP is the AIRAC cycle. This means:
+
+* Source data is downloaded and cached once for every AIRAC cycle.
+* The effective date and time is rounded down to the first day midnight of the AIRAC cycle.
+
+To force a rebuild within this time window, you have to clean the cache using the `-c` command line argument.

data/lib/aipp/aip/executable.rb

@@ -0,0 +1,40 @@
+module AIPP
+  module AIP
+
+    class Executable < AIPP::Executable
+
+      def initialize(exe_file)
+        super
+        AIPP.options.merge(
+          module: 'AIP',
+          airac: AIRAC::Cycle.new,
+          region_options: []
+        )
+        OptionParser.new do |o|
+          o.banner = <<~END
+            Download online AIP and convert it to #{AIPP.options.schema.upcase}.
+            Usage: #{File.basename($0)} [options]
+          END
+          common_options(o)
+          o.on('-a', '--airac (DATE|INTEGER)', String, %Q[AIRAC date or delta e.g. "+1" (default: "#{AIPP.options.airac.date.xmlschema}")]) { AIPP.options.airac = airac_for(_1) }
+          if AIPP.options.schema == :ofmx
+            o.on('-g', '--[no-]grouped-obstacles', 'group obstacles (default: false)') { AIPP.options.grouped_obstacles = _1 }
+          end
+          o.on('-O', '--region-options STRING', String, %Q[comma separated region specific options]) { AIPP.options.region_options = _1.split(',') }
+          developer_options(o)
+        end.parse!
+      end
+
+      private
+
+      def airac_for(argument)
+        if argument.match?(/^[+-]\d+$/)   # delta
+          AIRAC::Cycle.new + argument.to_i
+        else   # date
+          AIRAC::Cycle.new(argument)
+        end
+      end
+
+    end
+  end
+end

data/lib/aipp/aip/parser.rb

@@ -0,0 +1,9 @@
+module AIPP
+  module AIP
+
+    # @abstract
+    class Parser < AIPP::Parser
+    end
+
+  end
+end

data/lib/aipp/aip/runner.rb

@@ -0,0 +1,85 @@
+module AIPP
+  module AIP
+
+    class Runner < AIPP::Runner
+
+      def effective_at
+        AIPP.options.airac.effective.begin
+      end
+
+      def expiration_at
+        AIPP.options.airac.effective.end
+      end
+
+      def run
+        info("AIP AIRAC #{AIPP.options.airac.id} effective #{effective_at}", color: :green)
+        read_config
+        read_region
+        read_parsers
+        parse_sections
+        validate_aixm
+        write_build
+        write_aixm(AIPP.options.output_file || output_file)
+        write_config
+      end
+
+      private
+
+      # Parse AIP by invoking the parser classes for the current region.
+      def parse_sections
+        super
+        if AIPP.options.grouped_obstacles
+          info("grouping obstacles")
+          aixm.group_obstacles!
+        end
+      end
+
+      # Write the AIXM document and context information.
+      def write_build
+        if AIPP.options.section
+          super
+        else
+          info("writing build")
+          builds_dir.mkpath
+          build_file = builds_dir.join("#{AIPP.options.airac.date.xmlschema}.zip")
+          Dir.mktmpdir do |tmp_dir|
+            tmp_dir = Pathname(tmp_dir)
+            # AIXM/OFMX file
+            AIXM.config.mid = true
+            File.write(tmp_dir.join(output_file), aixm.to_xml)
+            # Build details
+            File.write(
+              tmp_dir.join('build.yaml'), {
+                version: AIPP::VERSION,
+                config: AIPP.config,
+                options: AIPP.options,
+              }.to_yaml
+            )
+            # Manifest
+            manifest = ['AIP','Feature', 'Comment', 'Short Uid Hash', 'Short Feature Hash'].to_csv
+            manifest += aixm.features.map do |feature|
+              xml = feature.to_xml
+              element = xml.first_match(/<(\w{3})\s/)
+              [
+                feature.source.split('|')[2],
+                element,
+                xml.match(/<!-- (.*?) -->/)[1],
+                AIXM::PayloadHash.new(xml.match(%r(<#{element}Uid\s.*?</#{element}Uid>)m).to_s).to_uuid[0,8],
+                AIXM::PayloadHash.new(xml).to_uuid[0,8]
+              ].to_csv
+            end.sort.join
+            File.write(tmp_dir.join('manifest.csv'), manifest)
+            # Zip it
+            build_file.delete if build_file.exist?
+            Zip::File.open(build_file, Zip::File::CREATE) do |zip|
+              tmp_dir.children.each do |entry|
+                zip.add(entry.basename.to_s, entry) unless entry.basename.to_s[0] == '.'
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/aipp/border.rb

@@ -35,7 +35,7 @@ module AIPP
       #   }
       #
       # Please note that GeoJSON orders coordinate tuples in mathematical order
-      # as `[longitude, latitude]`!
+      # as +[longitude, latitude]+!
       #
       # @param file [Pathname, String] GeoJSON file
       #
@@ -60,7 +60,7 @@ module AIPP
       # New border object from array of points
       #
       # The array must contain coordinate tuples in geographical order as
-      # `latitude longitude` separated by whitespace and/or commas.
+      # +latitude longitude+ separated by whitespace and/or commas.
       #
       # @param array [Array<Array<String>>] one or more arrays of coordinate pairs
       #

data/lib/aipp/debugger.rb

@@ -22,14 +22,14 @@ module AIPP
     #   @param verbose [Boolean] print verbose info, print unsevere warnings
     #     and re-raise rescued errors
     #   @yield Block the debugger is watching
-    def with_debugger(**options, &)
-      DEBUGGER__.instance_variable_set(:@options__, options.merge(counter: 0))
+    def with_debugger(&)
+      AIPP.cache.debug_counter = 0
       case
-      when id = debugger_options[:debug_on_warning]
+      when id = AIPP.options.debug_on_warning
         puts instructions_for(@id == true ? 'warning' : "warning #{id}")
         DEBUGGER__::start(no_sigint_hook: true, nonstop: true)
         call_with_rescue(&)
-      when debugger_options[:debug_on_error]
+      when AIPP.options.debug_on_error
         puts instructions_for('error')
         DEBUGGER__::start(no_sigint_hook: true, nonstop: true, postmortem: true)
         call_without_rescue(&)
@@ -39,20 +39,17 @@ module AIPP
       end
     end
 
-    def self.included(*)
-      DEBUGGER__.instance_variable_set(:@options__, {})
-    end
+    alias_method :original_warn, :warn
 
     # Issue a warning and maybe open a debug session.
     #
     # @param message [String] warning message
     # @param severe [Boolean] whether this problem must be fixed or not
-    alias_method :original_warn, :warn
     def warn(message, severe: true)
-      if severe || debugger_options[:verbose]
-        debugger_options[:counter] += 1
-        original_warn "WARNING #{debugger_options[:counter]}: #{message.upcase_first} #{'(unsevere)' unless severe}".red
-        debugger if debugger_options[:debug_on_warning] == true || debugger_options[:debug_on_warning] == debugger_options[:counter]
+      if severe || AIPP.options.verbose
+        AIPP.cache.debug_counter += 1
+        original_warn "WARNING #{AIPP.cache.debug_counter}: #{message.upcase_first} #{'(unsevere)' unless severe}".red
+        debugger if AIPP.options.debug_on_warning == true || AIPP.options.debug_on_warning == AIPP.cache.debug_counter
       end
     end
 
@@ -61,7 +58,9 @@ module AIPP
     # @param message [String] informational message
     # @param color [Symbol] message color
     def info(message, color: nil)
-      puts color ? message.upcase_first.send(color) : message.upcase_first
+      unless AIPP.options.quiet
+        puts color ? message.upcase_first.send(color) : message.upcase_first
+      end
     end
 
     # Issue a verbose informational message.
@@ -69,21 +68,17 @@ module AIPP
     # @param message [String] verbose informational message
     # @param color [Symbol] message color
     def verbose_info(message, color: :blue)
-      info(message, color: color) if debugger_options[:verbose]
+      info(message, color: color) if AIPP.options.verbose
     end
 
     private
 
-    def debugger_options
-      DEBUGGER__.instance_variable_get(:@options__)
-    end
-
     def call_with_rescue(&block)
       block.call
     rescue => error
       message = error.respond_to?(:original_message) ? error.original_message : error.message
       puts "ERROR: #{message}".magenta
-      raise if debugger_options[:verbose]
+      raise if AIPP.options.verbose
     end
 
     def call_without_rescue(&block)

data/lib/aipp/downloader/file.rb

@@ -0,0 +1,57 @@
+module AIPP
+  class Downloader
+
+    # Local file
+    class File
+      def initialize(archive: nil, file:, type: nil)
+        @archive = Pathname(archive) if archive
+        @file, @type = Pathname(file), type&.to_s
+      end
+
+      def fetch_to(path)
+        path.join(fetched_file).tap do |target|
+          if @archive
+            fail NotFoundError unless @archive.exist?
+            extract(@file, from: @archive, as: target)
+          else
+            fail NotFoundError unless @file.exist?
+            FileUtils.cp(@file, target)
+          end
+        end
+        self
+      end
+
+      def fetched_file
+        [name, type].join('.')
+      end
+
+      private
+
+      def name
+        @file.basename(@file.extname).to_s
+      end
+
+      def type
+        @type || @file.extname[1..] || fail("type must be declared")
+      end
+
+      def extract(file, from:, as:)
+        if respond_to?(extractor = 'un' + from.extname[1..], true)
+          send(extractor, file, from: from, as: as) or fail NotFoundError
+        else
+          fail "archive type not recognized"
+        end
+      end
+
+      # @return [Boolean] whether a file was extracted
+      def unzip(file, from:, as:)
+        Zip::File.open(from).inject(nil) do |_, entry|
+          if file.to_s == entry.name
+            break entry.extract(as)
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/aipp/downloader/graphql.rb

@@ -0,0 +1,29 @@
+module AIPP
+  class Downloader
+
+    # Remote file via HTTP
+    class GraphQL < File
+      def initialize(client:, query:, variables:)
+        @client, @query, @variables = client, query, variables
+      end
+
+      def fetch_to(path)
+        @client.query(@query, variables: @variables).tap do |result|
+          ::File.write(path.join(fetched_file), result.data.to_h.to_json)
+        end
+        self
+      end
+
+      private
+
+      def name
+        [@client, @query, @variables].map(&:to_s).join('|').to_digest
+      end
+
+      def type
+        :json
+      end
+    end
+
+  end
+end

data/lib/aipp/downloader/http.rb

@@ -0,0 +1,48 @@
+module AIPP
+  class Downloader
+
+    # Remote file via HTTP
+    class HTTP < File
+      ARCHIVE_MIME_TYPES = {
+        'application/zip' => :zip
+      }.freeze
+
+      def initialize(archive: nil, file:, type: nil, headers: {})
+        @archive = URI(archive) if archive
+        @file, @type, @headers = URI(file), type&.to_s, headers
+        @digest = (archive || file).to_digest
+      end
+
+      # @param path [Pathname] directory where to write the fetched file
+      # @return [File] fetched file
+      def fetch_to(path)
+        response = Excon.get((@archive || @file).to_s, headers: @headers)
+        fail NotFoundError if response.status == 404
+        mime_type = ARCHIVE_MIME_TYPES.fetch(response.headers['Content-Type'], :dat)
+        downloaded_file = path.join([@digest, mime_type].join('.'))
+        ::File.write(downloaded_file, response.body)
+        path.join(fetched_file).tap do |target|
+          if @archive
+            extract(@file, from: downloaded_file, as: target)
+            ::File.delete(downloaded_file)
+          else
+            ::File.rename(downloaded_file, target)
+          end
+        end
+        self
+      end
+
+      private
+
+      def name
+        path = Pathname(@file.path)
+        path.basename(path.extname).to_s.blank_to_nil || @digest
+      end
+
+      def type
+        @type || Pathname(@file.path).extname[1..].blank_to_nil || fail("type must be declared")
+      end
+    end
+
+  end
+end

data/lib/aipp/downloader.rb

@@ -4,28 +4,53 @@ module AIPP
   #
   # The downloader operates in the +storage+ directory where it creates two
   # subdirectories "sources" and "work". The initializer looks for the +source+
-  # archive in "sources" and (if found) unzips its contents into "work". When
+  # archive in "sources" and (if found) unpacks its contents into "work". When
   # reading a +document+, the downloader looks for the +document+ in "work" and
-  # (unless found) downloads it from +url+. HTML documents are parsed to
-  # +Nokogiri::HTML5::Document+, PDF documents are parsed to +AIPP::PDF+.
-  # Finally, the contents of "work" are written back to the +source+ archive.
+  # (if not found or the clean option is set) downloads it from +origin+.
+  # Finally, the contents of "work" are packed back into the +source+ archive.
+  #
+  # Origins are defined as instances of downloader origin objects:
+  #
+  # * {AIXM::Downloader::File} – local file or archive
+  # * {AIXM::Downloader::HTTP} – remote file or archive via HTTP
+  # * {AIXM::Downloader::GraphQL} – GraphQL query
+  #
+  # The following archives are recognized:
+  #
+  # [.zip] ZIP archive
+  #
+  # The following file types are recognised:
+  #
+  # [.ofmx] Parsed by Nokogiri returning an instance of {Nokogiri::XML::Document}[https://www.rubydoc.info/gems/nokogiri/Nokogiri/XML/Document]
+  # [.xml] Parsed by Nokogiri returning an instance of {Nokogiri::XML::Document}[https://www.rubydoc.info/gems/nokogiri/Nokogiri/XML/Document]
+  # [.html] Parsed by Nokogiri returning an instance of {Nokogiri::HTML5::Document}[https://www.rubydoc.info/gems/nokogiri/Nokogiri/HTML5/Document]
+  # [.pdf] Converted to text – see {AIPP::PDF}
+  # [.json] Deserialized JSON e.g. as response to a GraphQL query
+  # [.xlsx] Parsed by Roo returning an instance of {Roo::Excelx}[https://www.rubydoc.info/gems/roo/Roo/Excelx]
+  # [.ods] Parsed by Roo returning an instance of {Roo::OpenOffice}[https://www.rubydoc.info/gems/roo/Roo/OpenOffice]
+  # [.csv] Parsed by Roo returning an instance of {Roo::CSV}[https://www.rubydoc.info/gems/roo/Roo/CSV]
+  # [.txt] Instance of +String+
   #
   # @example
-  #   AIPP::Downloader.new(storage: options[:storage], source: "2018-11-08") do |downloader|
+  #   AIPP::Downloader.new(storage: AIPP.options.storage, source: "2018-11-08") do |downloader|
   #     html = downloader.read(
   #       document: 'ENR-5.1',
-  #       url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/FRANCE/AIRAC-2018-11-08/html/eAIP/FR-ENR-5.1-fr-FR.html'
+  #       origin: AIPP::Downloader::HTTP.new(
+  #         file: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/FRANCE/AIRAC-2018-11-08/html/eAIP/FR-ENR-5.1-fr-FR.html'
+  #       )
   #     )
   #     pdf = downloader.read(
   #       document: 'VAC-LFMV',
-  #       url: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/Atlas-VAC/PDF_AIPparSSection/VAC/AD/AD-2.LFMV.pdf'
+  #       origin: AIPP::Downloader::HTTP.new(
+  #         file: 'https://www.sia.aviation-civile.gouv.fr/dvd/eAIP_08_NOV_2018/Atlas-VAC/PDF_AIPparSSection/VAC/AD/AD-2.LFMV.pdf'
+  #       )
   #     )
   #   end
   class Downloader
-    extend Forwardable
     include AIPP::Debugger
 
-    # Error when URL results in "404 Not Found" HTTP status
+    # Error raised when any kind of downloader fails to find the resource e.g.
+    # because the local file does not exist or the remote file is unavailable.
     class NotFoundError < StandardError; end
 
     # @return [Pathname] directory to operate within
@@ -44,9 +69,15 @@ module AIPP
       fail(ArgumentError, 'bad storage directory') unless Dir.exist? storage
       @source_file = sources_path.join("#{@source}.zip")
       prepare
-      unzip if @source_file.exist?
+      if @source_file.exist?
+        if AIPP.options.clean
+          @source_file.delete
+        else
+          unpack
+        end
+      end
       yield self
-      zip
+      pack
     ensure
       teardown
     end
@@ -59,17 +90,14 @@ module AIPP
     # Download and read +document+
     #
     # @param document [String] document to read (without extension)
-    # @param url [String] URL to download the document from
-    # @param type [Symbol, nil] document type: +nil+ (default) to derive it from
-    #   the URL, :html, :pdf, :xlsx, :ods or :csv
-    # @return [Nokogiri::HTML5::Document, AIPP::PDF, Roo::Spreadsheet]
-    def read(document:, url:, type: nil)
-      type ||= Pathname(URI(url).path).extname[1..-1].to_sym
-      file = work_path.join([document, type].join('.'))
+    # @param origin [AIPP::Downloader::File, AIPP::Downloader::HTTP,
+    #   AIPP::Downloader::GraphQL] origin to download the document from
+    # @return [Object]
+    def read(document:, origin:)
+      file = work_path.join(origin.fetched_file)
       unless file.exist?
         verbose_info "downloading #{document}"
-        uri = URI.open(url)
-        IO.copy_stream(uri, file)
+        origin.fetch_to(work_path)
       end
       convert file
     end
@@ -97,13 +125,11 @@ module AIPP
       end
     end
 
-    def unzip
-      Zip::File.open(source_file).each do |entry|
-        entry.extract(work_path.join(entry.name))
-      end
+    def unpack
+      extract(source_file) or fail
     end
 
-    def zip
+    def pack
       backup_file = source_file.sub(/$/, '.old') if source_file.exist?
       source_file.rename(backup_file) if backup_file
       Zip::File.open(source_file, Zip::File::CREATE) do |zip|
@@ -114,15 +140,38 @@ module AIPP
       backup_file&.delete
     end
 
+    def extract(archive, only_entry: nil)
+      case archive.extname
+        when '.zip' then unzip(archive, only_entry: only_entry)
+        else fail(ArgumentError, "unrecognized archive type")
+      end
+    end
+
+    # @return [Boolean] whether at least one file was extracted
+    def unzip(archive, only_entry:)
+      Zip::File.open(archive).inject(false) do |_, entry|
+        case
+        when only_entry && only_entry == entry.name
+          break !!entry.extract(work_path.join(Pathname(entry.name).basename))
+        when !only_entry
+          !!entry.extract(work_path.join(entry.name))
+        else
+          false
+        end
+      end
+    end
+
     def convert(file)
       case file.extname
-        when '.xml' then Nokogiri.XML(File.open(file))
-        when '.html' then Nokogiri.HTML5(File.open(file))
+        when '.xml', '.ofmx' then Nokogiri.XML(::File.open(file), &:noblanks)
+        when '.html' then Nokogiri.HTML5(::File.open(file))
+        when '.json' then JSON.load_file(file)
         when '.pdf' then AIPP::PDF.new(file)
         when '.xlsx', '.ods', '.csv' then Roo::Spreadsheet.open(file.to_s)
-      else
-        fail(ArgumentError, "invalid document type")
+        when '.txt' then ::File.read(file)
+        else fail(ArgumentError, "unrecognized file type")
       end
     end
+
   end
 end

data/lib/aipp/environment.rb

@@ -0,0 +1,88 @@
+module AIPP
+
+  # Runtime environment
+  #
+  # Runtime environment objects inherit from OpenStruct but feature some
+  # extensions:
+  #
+  # * Use +replace+ to replace the current key/value table with the given hash.
+  # * Use +merge+ to merge the given hash into the current key/value hash.
+  # * When reading a value using square brackets, the key is implicitly
+  #   converted to Symbol.
+  #
+  # @example
+  #   AIPP.config                      # => AIPP::Environment::Config
+  #   AIPP.config.foo                  # => nil
+  #   AIPP.config.foo = :bar           # => :bar
+  #   AIPP.config.replace(fii: :bir)
+  #   AIPP.config.foo                  # => nil
+  #   AIPP.config.fii                  # => :bir
+  #   AIPP.config.read!                # method defined on Config class
+  class Environment
+    include Singleton
+
+    # Cache to store transient objects
+    class Cache < OpenStruct
+      def [](key)
+        super(key.to_s.to_sym)
+      end
+
+      def replace(hash)
+        @table = hash
+      end
+
+      def merge(hash)
+        @table.merge! hash
+      end
+    end
+
+    # Borders read from directory containing GeoJSON files
+    class Borders < Cache
+      def read!(dir)
+        @table.clear
+        dir.glob('*.geojson').each do |file|
+          @table[file.basename('.geojson').to_s.to_sym] = AIPP::Border.from_file(file)
+        end
+      end
+    end
+
+    # Fixtures read from directory containing YAML files
+    class Fixtures < Cache
+      def read!(dir)
+        @table.clear
+        dir.glob('*.yml').each do |file|
+          @table[file.basename('.yml').to_s.to_sym] = YAML.load_file(file)
+        end
+      end
+    end
+
+    # Options set via the CLI executable
+    class Options < Cache
+    end
+
+    # Config read from config.yml file
+    class Config < Cache
+      def read!(file)
+        @table = YAML.safe_load_file(file, symbolize_names: true, fallback: {}) if file.exist?
+        @table[:namespace] ||= SecureRandom.uuid
+      end
+
+      def write!(file)
+        File.write(file, @table.transform_keys(&:to_s).to_yaml)
+      end
+    end
+
+    def initialize
+      [Cache, Borders, Fixtures, Options, Config].each do |klass|
+        attribute = klass.to_s.split('::').last.downcase
+        instance_variable_set("@#{attribute}", klass.new)
+        AIPP.define_singleton_method(attribute) do
+          Environment.instance.instance_variable_get "@#{attribute}"
+        end
+      end
+    end
+
+  end
+end
+
+AIPP::Environment.instance