aipp 1.0.0 → 2.0.0

data/lib/aipp/regions/LS/helpers/base.rb

@@ -0,0 +1,111 @@
+module AIPP
+  module NewayAPI
+    HttpAdapter = GraphQL::Client::HTTP.new(ENV['NEWAY_API_URL']) do
+      def headers(context)
+        { "Authorization": "Bearer #{ENV['NEWAY_API_AUTHORIZATION']}" }
+      end
+    end
+    Schema = GraphQL::Client.load_schema(HttpAdapter)
+    Client = GraphQL::Client.new(schema: Schema, execute: HttpAdapter)
+
+    class Notam
+      Query = Client.parse <<~END
+        query ($region: String!, $series: [String!], $start: Int!, $end: Int!) {
+          queryNOTAMs(
+            filter: {region: $region, series: $series, start: $start, end: $end}
+          ) {
+            notamRaw
+          }
+        }
+      END
+    end
+  end
+
+  module LS
+    module Helpers
+      module Base
+
+        using AIXM::Refinements
+
+        # Mandatory Interface
+
+        def setup
+          AIPP.cache.aip = read('AIP').css('Ase')
+          AIPP.cache.dabs = read('DABS')
+        end
+
+        def origin_for(document)
+          case document
+          when 'ENR'
+            variables = {
+              region: 'LS',
+              series: %w(W B),
+              start: aixm.effective_at.beginning_of_day.to_i,
+              end: aixm.expiration_at.to_i
+            }
+            verbose_info("Querying API with #{variables}")
+            AIPP::Downloader::GraphQL.new(
+              client: AIPP::NewayAPI::Client,
+              query: AIPP::NewayAPI::Notam::Query,
+              variables: variables
+            )
+          when 'AD'
+            fail "not yet implemented"
+          when 'AIP'
+            AIPP::Downloader::HTTP.new(
+              archive: "https://snapshots.openflightmaps.org/live/#{AIRAC::Cycle.new.id}/ofmx/lsas/latest/ofmx_ls.zip",
+              file: "ofmx_ls/isolated/ofmx_ls.ofmx"
+            )
+          when 'DABS'
+            if aixm.effective_at.to_date == Date.today   # DABS cross check works reliably for today only
+              AIPP::Downloader::HTTP.new(
+                file: "https://www.skybriefing.com/o/dabs?today",
+                type: :pdf
+              )
+            end
+          else
+            fail "document not recognized"
+          end
+        end
+
+        # Templates
+
+        def organisation_lf
+          unless AIPP.cache.organisation_lf
+            AIPP.cache.organisation_lf = AIXM.organisation(
+              source: source(position: 1, document: "GEN-3.1"),
+              name: 'SWITZERLAND',
+              type: 'S'
+            ).tap do |organisation|
+              organisation.id = 'LS'
+            end
+            add AIPP.cache.organisation_ls
+          end
+          AIPP.cache.organisation_ls
+        end
+
+        # Parserettes
+
+        def timetable_from(schedules)
+          AIXM.timetable.tap do |timetable|
+            schedules&.each do |schedule|
+              schedule.actives.each do |actives|
+                schedule.times.each do |times|
+                  timesheet = AIXM.timesheet(
+                    adjust_to_dst: false,
+                    dates: (actives.instance_of?(Range) ? actives : (actives..actives))
+                    # TODO: transform to...
+                    # dates: actives
+                  )
+                  timesheet.times = times
+                  timetable.add_timesheet timesheet
+                end
+              end
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aipp/regions/LS/notam/ENR.rb

@@ -0,0 +1,173 @@
+using AIXM::Refinements
+
+module AIPP::LS::NOTAM
+  class ENR < AIPP::NOTAM::Parser
+
+    include AIPP::LS::Helpers::Base
+
+    def parse
+      json = read
+      fail "malformed JSON received from API" unless json.has_key?('queryNOTAMs')
+      added_notam_ids = []
+      json['queryNOTAMs'].each do |row|
+        next unless row['notamRaw'].match? /^Q\) LS/   # only parse national NOTAM
+
+# HACK: try to add missing commas to D-item of A- and B-series NOTAM
+if row['notamRaw'].match? /\A[AB]/
+  if row['notamRaw'].gsub!(/(#{NOTAM::Schedule::HOUR_RE.decapture}-#{NOTAM::Schedule::HOUR_RE.decapture})/, '\1,')
+    row['notamRaw'].gsub!(/,+/, ',')
+    row['notamRaw'].sub!(/,\n/, "\n")
+    warn("HACK: added missing commas to D item")
+  end
+end
+
+        (notam = notam_for(row['notamRaw'])) or next
+        if respect? notam
+          next if notam.data[:five_day_schedules] == []
+          added_notam_ids << notam.data[:id]
+          add(
+            case notam.data[:content]
+            when /\A[DR].AREA.+ACT/, /TMA.+ACT/
+              if fragment = fragment_for(notam)
+                AIXM.generic(fragment: fragment_for(notam)).tap do |airspace|
+                  element = airspace.fragment.children.first
+                  element.prepend_child(['<!--', notam.text ,'-->'].join("\n"))
+                  content = ["NOTAM #{notam.data[:id]}", element.at_css('txtName').content].join(": ").strip
+                  element.at_css('txtName').content = content
+                  content = [element.at_css('txtRmk')&.text, notam.data[:translated_content]].join("\n").strip
+                  element.find_or_add_child('txtRmk').content = content
+                  if schedule = notam.data[:five_day_schedules]
+                    timetable = timetable_from(schedule)
+                    element
+                      .find_or_add_child('Att', before_css: %w(codeSelAvbl txtRmk))
+                      .replace(timetable.to_xml(as: :Att).chomp)
+                  end
+                end
+              else
+                warn "no feature found for `#{notam.data[:content]}' - fallback to point and radius"
+                airspace_from(notam).tap do |airspace|
+                  airspace.geometry = geometry_from_q_item(notam)
+                end
+              end
+            when /\ATEMPO [DR].AREA.+(?:ACT|EST|ESTABLISHED) WI AREA/
+              airspace_from(notam).tap do |airspace|
+                airspace.geometry = geometry_from_content(notam)
+              end
+            else
+              airspace_from(notam).tap do |airspace|
+                airspace.geometry = geometry_from_q_item(notam)
+              end
+            end
+          )
+        else
+          verbose_info("Skipping NOTAM #{notam.data[:id]}")
+        end
+      end
+      dabs_cross_check(added_notam_ids)
+    end
+
+    private
+
+    def notam_for(raw_notam)
+      notam_id = raw_notam.strip.split(/\s+/, 2).first
+      if AIPP.options.crossload
+        crossload_file = AIPP.options.crossload.join('LS', "#{notam_id.sub('/', '_')}.txt")
+        if File.exist? crossload_file
+          info("crossloading #{crossload_file}")
+          return NOTAM.parse(crossload_file.read)
+        end
+      end
+      NOTAM.parse(raw_notam)
+    rescue
+      warn "cannot parse #{notam_id}"
+      raise if AIPP.options.force
+    end
+
+    # @return [Boolean] whether to respect this NOTAM or ignore it
+    def respect?(notam)
+      notam.data[:condition] != :checklist && (
+        notam.data[:scope].include?(:navigation_warning) ||
+        %i(terminal_control_area).include?(notam.data[:subject])   # TODO: include :obstacle as well
+      )
+    end
+
+    def fragment_for(notam)
+      case notam.data[:content]
+      when /(?<type>TMA) ((SECT )?(?<section>\d+) )?ACT/
+        'Ase:has(codeType:contains("%s") + codeId:contains("%s %s"))' % [$~['type'], notam.data[:locations].first, $~['section']]
+      when /[DR].AREA (?<name>LS-[DR]\d+[A-Z]?).+ACT/
+        'Ase:has(codeId:matches("^%s( .+)?$"))' % [$~['name']]
+      else
+        return
+      end.then do |selector|
+        AIPP.cache.aip.at_css(selector, Nokogiri::MATCHES)
+      end
+    end
+
+    def airspace_from(notam)
+      AIXM.airspace(
+        id: notam.data[:id],
+        type: :regulated_airspace,
+        name: "NOTAM #{notam.data[:id]}"
+      ).tap do |airspace|
+        airspace.add_layer(
+          AIXM.layer(
+            vertical_limit: AIXM.vertical_limit(
+              upper_z: notam.data[:upper_limit],
+              lower_z: notam.data[:lower_limit]
+            )
+          ).tap do |layer|
+            layer.selective = true
+            if schedule = notam.data[:five_day_schedules]
+              layer.timetable = timetable_from(schedule)
+            end
+            layer.remarks = notam.data[:translated_content]
+          end
+        )
+        airspace.comment = notam.text
+      end
+    end
+
+    def geometry_from_content(notam)
+      if notam.data[:content].squish.match(/WI AREA(?<coordinates>(?: \d{6}N\d{7}E)+)/)
+        AIXM.geometry.tap do |geometry|
+          $~['coordinates'].split.each do |coordinate|
+            xy = AIXM.xy(lat: coordinate[0, 7], long: coordinate[7, 8])
+            geometry.add_segment(AIXM.point(xy: xy))
+          end
+        end
+      else
+        warn "cannot parse WI AREA - fallback to point and radius"
+        geometry_from_q_item(notam)
+      end
+    end
+
+    def geometry_from_q_item(notam)
+      AIXM.geometry.tap do |geometry|
+        geometry.add_segment AIXM.circle(
+          center_xy: notam.data[:center_point],
+          radius: notam.data[:radius]
+        )
+      end
+    end
+
+    def obstacle_from(notam)
+      # TODO: implement obstacle
+    end
+
+    def dabs_cross_check(added_notam_ids)
+      dabs_date = aixm.effective_at.to_date.strftime("DABS Date: %Y %^b %d")
+      case
+      when AIPP.cache.dabs.nil?
+        warn("DABS not available - skipping cross check")
+      when !AIPP.cache.dabs.text.include?(dabs_date)
+        warn("DABS date mismatch - skippping cross check")
+      else
+        dabs_notam_ids = AIPP.cache.dabs.text.scan(NOTAM::Item::ID_RE.decapture).uniq
+        missing_notam_ids = dabs_notam_ids - added_notam_ids
+        warn("DABS disagrees: #{missing_notam_ids.join(', ')} missing") if missing_notam_ids.any?
+      end
+    end
+
+  end
+end

data/lib/aipp/runner.rb

@@ -0,0 +1,152 @@
+module AIPP
+
+  # @abstract
+  class Runner
+    include AIPP::Debugger
+    using AIXM::Refinements
+
+    # @return [AIXM::Document] target document
+    attr_reader :aixm
+
+    def initialize
+      AIPP.options.storage = AIPP.options.storage.join(AIPP.options.region, AIPP.options.module.downcase)
+      AIPP.options.storage.mkpath
+      @dependencies = THash.new
+      @aixm = AIXM.document(effective_at: effective_at, expiration_at: expiration_at)
+      AIXM.send("#{AIPP.options.schema}!")
+      AIXM.config.region = AIPP.options.region
+    end
+
+    # @return [String]
+    def inspect
+      "#<#{self.class}>"
+    end
+
+    # @abstract
+    def effective_at
+      fail "effective_at method must be implemented by module runner"
+    end
+
+    # @abstract
+    def expiration_at
+      nil
+    end
+
+    # @abstract
+    def run
+      fail "run method must be implemented by module runner"
+    end
+
+    # @return [Pathname] directory containing all files for the current region
+    def region_dir
+      Pathname(__FILE__).dirname.join('regions', AIPP.options.region)
+    end
+
+    # @return [String] sources file name (default: xmlschema representation
+    #   of effective_at date/time)
+    def sources_file
+      effective_at.xmlschema
+    end
+
+    def output_file
+      "#{AIPP.options.region}_#{AIPP.options.module}_#{effective_at.strftime('%F_%HZ')}.#{AIPP.options.schema}"
+    end
+
+    # @return [Pathname] directory containing the builds
+    def builds_dir
+      AIPP.options.storage.join('builds')
+    end
+
+    # @return [Pathname] config file for the current region
+    def config_file
+      AIPP.options.storage.join('config.yml')
+    end
+
+    private
+
+    # Read the configuration from config.yml.
+    def read_config
+      info("reading config.yml")
+      AIPP.config.read! config_file
+      @aixm.namespace = AIPP.config.namespace
+    end
+
+    # Read the region directory.
+    def read_region
+      info("reading region #{AIPP.options.region}")
+      fail("unknown region `#{AIPP.options.region}'") unless region_dir.exist?
+      verbose_info "reading fixtures"
+      AIPP.fixtures.read! region_dir.join('fixtures')
+      verbose_info "reading borders"
+      AIPP.borders.read! region_dir.join('borders')
+      verbose_info "reading helpers"
+      region_dir.glob('helpers/*.rb').each { |f| require f }
+    end
+
+    # Read parser files.
+    def read_parsers
+      verbose_info("reading parsers")
+      region_dir.join(AIPP.options.module.downcase).glob('*.rb').each do |file|
+        verbose_info "requiring #{file.basename}"
+        require file
+        section = file.basename('.*').to_s.classify
+        @dependencies[section] = class_for(section).dependencies
+      end
+    end
+
+    # Parse sections by invoking the parser classes.
+    def parse_sections
+      AIPP::Downloader.new(storage: AIPP.options.storage, source: sources_file) do |downloader|
+        @dependencies.tsort(AIPP.options.section).each do |section|
+          info("parsing #{section.sectionize}")
+          class_for(section).new(
+            downloader: downloader,
+            aixm: aixm
+          ).attach_patches.tap(&:parse).detach_patches
+        end
+      end
+    end
+
+    # Validate the AIXM document.
+    #
+    # @raise [RuntimeError] if the document is not valid
+    def validate_aixm
+      info("detecting duplicates")
+      if (duplicates = aixm.features.duplicates).any?
+        message = "duplicates found"
+        details = duplicates.map { "#{_1.inspect} from #{_1.source}" }.join("\n")
+        AIPP.options.force ? warn(message) : fail([message, details].join(":\n"))
+      end
+      info("validating #{AIPP.options.schema.upcase}")
+      unless aixm.valid?
+        message = "invalid #{AIPP.options.schema.upcase} document"
+        details = aixm.errors.map(&:message).join("\n")
+        AIPP.options.force ? warn(message) : fail([message, details].join(":\n"))
+      end
+      info("counting #{aixm.features.count} features")
+    end
+
+    # Write the AIXM document.
+    def write_aixm(file)
+      info("writing #{file}")
+      AIXM.config.mid = AIPP.options.mid
+      File.write(file, aixm.to_xml)
+    end
+
+    # Write build information.
+    def write_build
+      info ("skipping build")
+    end
+
+    # Write the configuration to config.yml.
+    def write_config
+      info("writing config.yml")
+      AIPP.config.write! config_file
+    end
+
+    def class_for(section)
+      [:AIPP, AIPP.options.region, AIPP.options.module.upcase, section.classify].constantize
+    end
+  end
+
+end

data/lib/aipp/version.rb

@@ -1,3 +1,3 @@
 module AIPP
-  VERSION = "1.0.0".freeze
+  VERSION = "2.0.0".freeze
 end

data/lib/aipp.rb

@@ -1,46 +1,65 @@
 require 'debug/session'
-require 'forwardable'
+require 'singleton'
 require 'colorize'
 require 'optparse'
 require 'yaml'
-require 'csv'
-require 'roo'
 require 'pathname'
+require 'fileutils'
 require 'tmpdir'
-require 'net/http'
-require 'open-uri'
 require 'securerandom'
 require 'tsort'
 require 'ostruct'
 require 'date'
+require 'excon'
+require 'graphql/client'
+require 'graphql/client/http'
 require 'nokogiri'
+require 'csv'
+require 'roo'
 require 'pdf-reader'
 require 'json'
 require 'zip'
 require 'airac'
 require 'aixm'
+require 'notam'
 
 require 'active_support'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/string'
+require 'active_support/core_ext/date_time'
 
+require_relative 'core_ext/nil_class'
 require_relative 'core_ext/integer'
 require_relative 'core_ext/string'
-require_relative 'core_ext/nil_class'
+require_relative 'core_ext/array'
 require_relative 'core_ext/enumerable'
 require_relative 'core_ext/hash'
 require_relative 'core_ext/nokogiri'
 
 require_relative 'aipp/version'
 require_relative 'aipp/debugger'
-require_relative 'aipp/pdf'
+require_relative 'aipp/downloader'
+require_relative 'aipp/downloader/file'
+require_relative 'aipp/downloader/http'
+require_relative 'aipp/downloader/graphql'
+require_relative 'aipp/patcher'
+require_relative 'aipp/parser'
+
+require_relative 'aipp/environment'
 require_relative 'aipp/border'
+require_relative 'aipp/pdf'
 require_relative 'aipp/t_hash'
+
 require_relative 'aipp/executable'
-require_relative 'aipp/patcher'
-require_relative 'aipp/aip'
-require_relative 'aipp/parser'
-require_relative 'aipp/downloader'
+require_relative 'aipp/runner'
+
+require_relative 'aipp/aip/executable'
+require_relative 'aipp/aip/runner'
+require_relative 'aipp/aip/parser'
+
+require_relative 'aipp/notam/executable'
+require_relative 'aipp/notam/runner'
+require_relative 'aipp/notam/parser'
 
 # Disable "did you mean?" suggestions
 #

data/lib/core_ext/array.rb

@@ -0,0 +1,13 @@
+class Array
+
+  # Convert array of namespaces to constant.
+  #
+  # @example
+  #   %i(AIPP AIP Base).constantize   # => AIPP::AIP::Base
+  #
+  # @return [Class, Module] converted array
+  def constantize
+    map(&:to_s).join('::').constantize
+  end
+
+end

data/lib/core_ext/nokogiri.rb

@@ -1,18 +1,26 @@
 module Nokogiri
+
+  module PseudoClasses
+    class Matches
+      def matches(node_set, regexp)
+        node_set.find_all { _1.content.match?(/#{regexp}/) }
+      end
+    end
+  end
+
+  # Pseudo class which matches the content of each node set member against the
+  # given regular expression.
+  #
+  # @example
+  #   node.css('title:matches("\w+")', Nokogiri::MATCHES)
+  MATCHES = PseudoClasses::Matches.new
+
   module XML
     class Element
 
       BOOLEANIZE_AS_TRUE_RE = /^(true|yes|oui|ja)$/i.freeze
       BOOLEANIZE_AS_FALSE_RE = /^(false|no|non|nein)$/i.freeze
 
-      # Traverse all child elements and build a hash mapping the symbolized
-      # child node name to the child content.
-      #
-      # @return [Hash]
-      def contents
-        @contents ||= elements.to_h { [_1.name.to_sym, _1.content] }
-      end
-
       # Shortcut to query +contents+ array which accepts both String or
       # Symbol queries as well as query postfixes.
       #
@@ -39,6 +47,46 @@ module Nokogiri
         end
       end
 
+      # Traverse all child elements and build a hash mapping the symbolized
+      # child node name to the child content.
+      #
+      # @return [Hash]
+      def contents
+        @contents ||= elements.to_h { [_1.name.to_sym, _1.content] }
+      end
+
+      # Find this child element or add a new such element if none is found.
+      #
+      # The position to add is determined as follows:
+      #
+      # * If +after_css+ is given, its rules are applied in reverse order and
+      #   the last matching rule defines the predecessor of the added child.
+      # * If only +before_css+ is given, its rules are applied in order and
+      #   the first matching rule defines the successor of the added child.
+      # * If none of the above are given, the child is added at the end.
+      #
+      # @param name [Array<String>] name of the child element
+      # @param after_css [Array<String>] array of CSS rules
+      # @param before_css [Array<String>] array of CSS rules
+      # @return [Nokogiri::XML::Element, nil] element or +nil+ if none found
+      #   and no position to add a new one could be determined
+      def find_or_add_child(name, after_css: nil, before_css: nil)
+        at_css(name) or begin
+          case
+          when after_css
+            at_css(*after_css.reverse).then do |predecessor|
+              predecessor&.add_next_sibling("<#{name}/>")
+            end&.first
+          when before_css
+            at_css(*before_css).then do |successor|
+              successor&.add_previous_sibling("<#{name}/>")
+            end&.first
+          else
+            add_child("<#{name}/>").first
+          end
+        end
+      end
+
       private
 
       def booleanize(content)

data/lib/core_ext/string.rb

@@ -1,6 +1,53 @@
 class String
+  remove_method :classify
 
-  # Convert blank strings to +nil+.
+  # Convert (underscored) file name to (camelcased) class name
+  #
+  # Similar to +classify+ from ActiveSupport, however, with a few differences:
+  #
+  # * Namespaces are ignored.
+  # * Plural strings are not singularized.
+  # * Characters other than A-Z, a-z, 0-9 and _ are removed.
+  #
+  # Use +sectionize+ to reverse this method.
+  #
+  # @example
+  #   "navigational_aids".classify     # => "NavigationalAids"
+  #   "ENR".classify                   # => "ENR"
+  #   "ENR-4.1".classify               # => "ENR41"
+  #   "AIPP/LF/AIP/ENR-4.1".classify   # => "ENR41"
+  #
+  # @return [String] converted string
+  def classify
+    split('/').last.remove(/\W/).camelcase
+  end
+
+  # Convert (camelcased) class name to (underscored) file name
+  #
+  # Similar to +underscore+ from ActiveSupport, however, with a few differences:
+  #
+  # * Namespaces are ignored.
+  # * AIP naming conventions are honored.
+  #
+  # Use +classify+ to reverse this method.
+  #
+  # @example
+  #   "NavigationalAids".sectionize       # => "navigational_aids"
+  #   "ENR".sectionize                    # => "ENR"
+  #   "ENR41".sectionize                  # => "ENR-4.1"
+  #   "AIPP::LF::AIP::ENR41".sectionize   # => "ENR-4.1"
+  #
+  # @return [String] converted string
+  def sectionize
+    case klass = self.split('::').last
+      when /\A([A-Z]{2,3})\z/ then $1
+      when /\A([A-Z]{2,3})(\d)\z/ then "#{$1}-#{$2}"
+      when /\A([A-Z]{2,3})(\d)(\d+)\z/ then "#{$1}-#{$2}.#{$3}"
+      else klass.underscore
+    end
+  end
+
+  # Convert blank strings to +nil+
   #
   # @example
   #   "foobar".blank_to_nil   # => "foobar"
@@ -77,10 +124,25 @@ class String
   end
 
   # Remove all XML/HTML tags and entities from the string
+  #
+  # @example
+  #   "this <em>is</em> a <br> test".strip_markup   # => "this is a  test"
+  #
+  # @return [String]
   def strip_markup
     self.gsub(/<.*?>|&[#\da-z]+;/i, '')
   end
 
+  # Builds the MD5 hash as hex and returns the first eight characters.
+  #
+  # @example
+  #   "this is a test".to_digest   # => "54b0c58c"
+  #
+  # @return [String]
+  def to_digest
+    Digest::MD5.hexdigest(self)[0,8]
+  end
+
   # Same as +to_f+ but accept both dot and comma as decimal separator
   #
   # @example