aipp 0.2.1 → 0.2.2

data/lib/aipp/regions/LF/helpers/common.rb

@@ -0,0 +1,186 @@
+module AIPP
+  module LF
+    module Helpers
+      module Common
+
+        using AIXM::Refinements
+
+        # Map border names to OFMX
+        BORDERS = {
+          'franco-allemande' => 'FRANCE_GERMANY',
+          'franco-espagnole' =>  'FRANCE_SPAIN',
+          'franco-italienne' => 'FRANCE_ITALY',
+          'franco-suisse' => 'FRANCE_SWITZERLAND',
+          'franco-luxembourgeoise' => 'FRANCE_LUXEMBOURG',
+          'franco-belge' => 'BELGIUM_FRANCE',
+          'germano-suisse' => 'GERMANY_SWITZERLAND',
+          'hispano-andorrane' => 'ANDORRA_SPAIN',
+          'la côte atlantique française' => 'FRANCE_ATLANTIC_COAST',   # TODO: handle internally
+          'côte méditérrannéenne' => 'FRANCE_MEDITERRANEAN_COAST',   # TODO: handle internally
+          'limite des eaux territoriales atlantique françaises' => 'FRANCE_ATLANTIC_TERRITORIAL_SEA',   # TODO: handle internally
+          'parc national des écrins' => 'FRANCE_ECRINS_NATIONAL_PARK'   # TODO: handle internally
+        }.freeze
+
+        # Intersection points between three countries
+        INTERSECTIONS = {
+          'FRANCE_SPAIN|ANDORRA_SPAIN' => AIXM.xy(lat: 42.502720, long: 1.725965),
+          'ANDORRA_SPAIN|FRANCE_SPAIN' => AIXM.xy(lat: 42.603571, long: 1.442681),
+          'FRANCE_SWITZERLAND|FRANCE_ITALY' => AIXM.xy(lat: 45.922701, long: 7.044125),
+          'BELGIUM_FRANCE|FRANCE_LUXEMBOURG' => AIXM.xy(lat: 49.546428, long: 5.818415),
+          'FRANCE_LUXEMBOURG|FRANCE_GERMANY' => AIXM.xy(lat: 49.469438, long: 6.367516),
+          'FRANCE_GERMANY|FRANCE_SWITZERLAND' => AIXM.xy(lat: 47.589831, long: 7.589049),
+          'GERMANY_SWITZERLAND|FRANCE_GERMANY' => AIXM.xy(lat: 47.589831, long: 7.589049)
+        }
+
+        # Map surface compositions to OFMX composition and preparation
+        COMPOSITIONS = {
+          'revêtue' => { preparation: :paved },
+          'non revêtue' => { preparation: :natural },
+          'macadam' => { composition: :macadam },
+          'béton' => { composition: :concrete, preparation: :paved },
+          'béton bitumineux' => { composition: :bitumen, preparation: :paved },
+          'enrobé bitumineux' => { composition: :bitumen },
+          'asphalte' => { composition: :asphalt, preparation: :paved },
+          'gazon' => { composition: :grass }
+        }
+
+        # Transform French text fragments to English
+        ANGLICISE_MAP = {
+          /[^A-Z0-9 .\-]/ => '',
+          /0(\d)/ => '\1',
+          /(\d)-(\d)/ => '\1.\2',
+          /PARTIE/ => '',
+          /DELEG\./ => 'DELEG ',
+          /FRANCAISE?/ => 'FR',
+          /ANGLAISE?/ => 'UK',
+          /BELGE/ => 'BE',
+          /LUXEMBOURGEOISE?/ => 'LU',
+          /ALLEMANDE?/ => 'DE',
+          /SUISSE/ => 'CH',
+          /ITALIEN(?:NE)?/ => 'IT',
+          /ESPAGNOLE?/ => 'ES',
+          /ANDORRANE?/ => 'AD',
+          /NORD/ => 'N',
+          /EST/ => 'E',
+          /SUD/ => 'S',
+          /OEST/ => 'W',
+          /ANGLO NORMANDES/ => 'ANGLO-NORMANDES',
+          / +/ => ' '
+        }.freeze
+
+        # Templates
+
+        def organisation_lf
+          @organisation_lf ||= AIXM.organisation(
+            name: 'FRANCE',
+            type: 'S'
+          ).tap do |organisation|
+            organisation.id = 'LF'
+          end
+        end
+
+        # Transformations
+
+        def prepare(html:)
+          html.tap do |node|
+            node.css('del, tr[class*="AmdtDeletedAIRAC"]').each(&:remove)   # remove deleted entries
+          end
+        end
+
+        def anglicise(name:)
+          name&.uptrans&.tap do |string|
+            ANGLICISE_MAP.each do |regexp, replacement|
+              string.gsub!(regexp, replacement)
+            end
+          end
+        end
+
+        # Parsers
+
+        def source(position:, aip_file: nil)
+          aip_file ||= @aip
+          [
+            options[:region],
+            aip_file.split('-').first,
+            aip_file,
+            options[:airac].date.xmlschema,
+            position
+          ].join('|')
+        end
+
+        def xy_from(text)
+          parts = text.strip.split(/\s+/)
+          AIXM.xy(lat: parts[0], long: parts[1])
+        end
+
+        def z_from(limit)
+          case limit
+            when nil then nil
+            when 'SFC' then AIXM::GROUND
+            when 'UNL' then AIXM::UNLIMITED
+            when /(\d+)ftASFC/ then AIXM.z($1.to_i, :qfe)
+            when /(\d+)ftAMSL/ then AIXM.z($1.to_i, :qnh)
+            when /FL(\d+)/ then AIXM.z($1.to_i, :qne)
+            else fail "z `#{limit}' not recognized"
+          end
+        end
+
+        def layer_from(text_for_limits, text_for_class=nil)
+          above, below = text_for_limits.gsub(/ /, '').split(/\n+/).select(&:blank_to_nil).split { |e| e.match? '---+' }
+          above.reverse!
+          AIXM.layer(
+            class: text_for_class,
+            vertical_limits: AIXM.vertical_limits(
+              max_z: z_from(above[1]),
+              upper_z: z_from(above[0]),
+              lower_z: z_from(below[0]),
+              min_z: z_from(below[1])
+            )
+          )
+        end
+
+        def geometry_from(text)
+          AIXM.geometry.tap do |geometry|
+            buffer = {}
+            text.gsub(/\s+/, ' ').strip.split(/ - /).append('end').each do |element|
+              case element
+              when /arc (anti-)?horaire .+ sur (\S+) , (\S+)/i
+                geometry << AIXM.arc(
+                  xy: buffer.delete(:xy),
+                  center_xy: AIXM.xy(lat: $2, long: $3),
+                  clockwise: $1.nil?
+                )
+              when /cercle de ([\d\.]+) (NM|km|m) .+ sur (\S+) , (\S+)/i
+                geometry << AIXM.circle(
+                  center_xy: AIXM.xy(lat: $3, long: $4),
+                  radius: AIXM.d($1.to_f, $2)
+                )
+              when /end|(\S+) , (\S+)/
+                geometry << AIXM.point(xy: buffer[:xy]) if buffer.has_key?(:xy)
+                buffer[:xy] = AIXM.xy(lat: $1, long: $2) if $1
+              when /^frontière ([\w-]+)/i, /^(\D[^(]+)/i
+                border_name = BORDERS.fetch($1.downcase.strip)
+                buffer[:xy] ||= INTERSECTIONS.fetch("#{buffer[:border_name]}|#{border_name}")
+                buffer[:border_name] = border_name
+                if border_name == 'FRANCE_SPAIN'   # specify which part of this split border
+                  border_name += buffer[:xy].lat < 42.55 ? '_EAST' : '_WEST'
+                end
+                geometry << AIXM.border(
+                  xy: buffer.delete(:xy),
+                  name: border_name
+                )
+              else
+                fail "geometry `#{element}' not recognized"
+              end
+            end
+          end
+        end
+
+        def timetable_from(text)
+          AIXM::H24 if text.gsub(/\W/, '') == 'H24'
+        end
+
+      end
+    end
+  end
+end

data/lib/aipp/version.rb

@@ -1,3 +1,3 @@
 module AIPP
-  VERSION = "0.2.1".freeze
+  VERSION = "0.2.2".freeze
 end

data/lib/core_ext/enumerable.rb

@@ -0,0 +1,52 @@
+module Enumerable
+
+  # !method split(object=nil, &block)
+  #   Divides an enumerable into sub-enumerables based on a delimiter,
+  #   returning an array of these sub-enumerables.
+  #
+  #   @example
+  #     [1, 2, 0, 3, 4].split { |e| e == 0 }   # => [[1, 2], [3, 4]]
+  #     [1, 2, 0, 3, 4].split(0)               # => [[1, 2], [3, 4]]
+  #     [0, 0, 1, 0, 2].split(0)               # => [[], [] [1], [2]]
+  #     [1, 0, 0, 2, 3].split(0)               # => [[1], [], [2], [3]]
+  #     [1, 0, 2, 0, 0].split(0)               # => [[1], [2]]
+  #
+  #   @note While similar to +Array#split+ from ActiveSupport, this core
+  #     extension works for all enumerables and therefore works fine with.
+  #     Nokogiri. Also, it behaves more like +String#split+ by ignoring any
+  #     trailing zero-length sub-enumerators.
+  #
+  #   @param object [Object] element at which to split
+  #   @yield [Object] element to analyze
+  #   @yieldreturn [Boolean] whether to split at this element or not
+  #   @return [Array]
+  def split(*args, &block)
+    [].tap do |array|
+      while index = slice((start ||= 0)...length).find_index(*args, &block)
+        array << slice(start...start+index)
+        start += index + 1
+      end
+      array << slice(start..-1) if start < length
+    end
+  end
+
+  # !method group_by_chunks(&block)
+  #   Build a hash which maps elements matching the chunk condition to
+  #   an array of subsequent elements which don't match the chunk condition.
+  #
+  #   @example
+  #     [1, 10, 11, 12, 2, 20, 21, 3, 30, 31, 32].group_by_chunks { |i| i < 10 }
+  #     # => { 1 => [10, 11, 12], 2 => [20, 21], 3 => [30, 31, 32] }
+  #
+  #   @note The first element must match the chunk condition.
+  #
+  #   @yield [Object]  object to analyze
+  #   @yieldreturn [Boolean] chunk condition: begin a new chunk with this
+  #     object as key if the condition returns true
+  #   @return [Hash]
+  def group_by_chunks
+    fail(ArgumentError, "first element must match chunk condition") unless yield(first)
+    slice_when { |_, e| yield(e) }.map { |e| [e.first, e[1..]] }.to_h
+  end
+
+end

data/lib/core_ext/nil_class.rb

@@ -0,0 +1,10 @@
+class NilClass
+
+  # Always returns +nil+, companion to +String#blank_to_nil+.
+  #
+  # @return [nil]
+  def blank_to_nil
+    nil
+  end
+
+end

data/lib/core_ext/object.rb

@@ -0,0 +1,42 @@
+class Object
+
+  # Issue a warning and maybe open a Pry session attached to the error or
+  # binding passed.
+  #
+  # @example with error context
+  #   begin
+  #     (...)
+  #   rescue => error
+  #     warn("oops", pry: error)
+  #   end
+  # @example with binding context
+  #   warn("oops", pry: binding)
+  # @param message [String] warning message
+  # @param pry [Exception, Binding, nil] attach the Pry session to this error
+  #   or binding
+  def warn(message, pry:)
+    $WARN_COUNTER = $WARN_COUNTER.to_i + 1
+    Kernel.warn "WARNING #{$WARN_COUNTER}: #{message}".red
+    if $PRY_ON_WARN == true || $PRY_ON_WARN == $WARN_COUNTER
+      case pry
+        when Exception then Pry::rescued(pry)
+        when Binding then pry.pry
+      end
+    end
+  end
+
+  # Issue an informational message.
+  #
+  # @param message [String] informational message
+  def info(message, color: :black)
+    puts message.send(color)
+  end
+
+  # Issue a debug message.
+  #
+  # @param message [String] debug message
+  def debug(message, color: :blue)
+    info(message, color: color) if $DEBUG
+  end
+
+end

data/lib/core_ext/string.rb

@@ -0,0 +1,105 @@
+class String
+
+  # Convert blank strings to +nil+.
+  #
+  # @example
+  #   "foobar".blank_to_nil   # => "foobar"
+  #   " ".blank_to_nil        # => nil
+  #   "".blank_to_nil         # => nil
+  #   nil.blank_to_nil        # => nil
+  #
+  # @return [String, nil] converted string
+  def blank_to_nil
+    self if present?
+  end
+
+  # Strip and collapse unnecessary whitespace
+  #
+  # @note While similar to +String#squish+ from ActiveSupport, newlines +\n+
+  #   are preserved and not collapsed into one space.
+  #
+  # @example
+  #   "  foo\n\nbar \r".copact   # => "foo\nbar"
+  #
+  # @return [String] compacted string
+  def compact
+    split("\n").map { |s| s.squish.blank_to_nil }.compact.join("\n")
+  end
+
+  # Fix messy oddities such as the use of two apostrophes instead of a quote
+  #
+  # @example
+  #   "the ''Terror'' was a fine ship".cleanup   # => "the \"Terror\" was a fine ship"
+  #
+  # @return [String] cleaned string
+  def cleanup
+    gsub(/[#{AIXM::MIN}]{2}|[#{AIXM::SEC}]/, '"').   # unify quotes
+      gsub(/[#{AIXM::MIN}]/, "'").   # unify apostrophes
+      gsub(/"[[:blank:]]*(.*?)[[:blank:]]*"/m, '"\1"').   # remove whitespace within quotes
+      split(/\r?\n/).map { |s| s.strip.blank_to_nil }.compact.join("\n")   # remove blank lines
+  end
+
+  # Add spaces between obviously glued words:
+  # * camel glued words
+  # * three-or-more-letter and number-only words
+  #
+  # @example
+  #   "thisString has spaceProblems".unglue   # => "this String has space problems"
+  #   "the first123meters of D25".unglue      # => "the first 123 meters of D25"
+  #
+  # @return [String] unglued string
+  def unglue
+    self.dup.tap do |string|
+      [/([[:lower:]])([[:upper:]])/, /([[:alpha:]]{3,})(\d)/, /(\d)([[:alpha:]]{3,})/].freeze.each do |regexp|
+        string.gsub!(regexp, '\1 \2')
+      end
+    end
+  end
+
+  # Calculate the correlation of two strings by counting mutual words
+  #
+  # Both strings are normalized as follows:
+  # * remove accents, umlauts etc
+  # * remove everything but members of the +\w+ class
+  # * downcase
+  #
+  # The normalized strings are split into words. Only words fulfilling either
+  # of the following conditions are taken into consideration:
+  # * words present in and translated by the +synonyms+ map
+  # * words of at least 5 characters length
+  # * words consisting of exactly one letter followed by any number of digits
+  #   (an optional whitespace between the two is ignored, e.g. "D 25" is the
+  #   same as "D25")
+  #
+  # The +synonyms+ map is an array where terms in even positions map to their
+  # synonym in the following (odd) position:
+  #
+  #   SYNONYMS = ['term1', 'synonym1', 'term2', 'synonym2']
+  #
+  # @example
+  #   subject = "Truck en route on N 3 sud"
+  #   subject.correlate("my car is on D25")          # => 0
+  #   subject.correlate("my truck is on D25")        # => 1
+  #   subject.correlate("my truck is on N3")         # => 2
+  #   subject.correlate("south", ['sud', 'south'])   # => 1
+  #
+  # @param other [String] string to compare with
+  # @param synonyms [Array<String>] array of synonym pairs
+  # @return [Integer] 0 for unrelated strings and positive integers for related
+  #   strings with higher numbers indicating tighter correlation
+  def correlate(other, synonyms=[])
+    self_words, other_words = [self, other].map do |string|
+      string.
+        unicode_normalize(:nfd).
+        downcase.gsub(/[-\u2013]/, ' ').
+        remove(/[^\w\s]/).
+        gsub(/\b(\w)\s?(\d+)\b/, '\1\2').
+        compact.
+        split(/\W+/).
+        map { |w| (i = synonyms.index(w)).nil? ? w : (i.odd? ? w : synonyms[i + 1]).upcase }.
+        keep_if { |w| w.match?(/\w{5,}|\w\d+|[[:upper:]]/) }.
+        uniq
+    end
+    (self_words & other_words).count
+  end
+end

data/spec/fixtures/document.pdf.json

@@ -0,0 +1 @@
+["page 1, line 1\npage 1, line 2\npage 1, line 3\npage 1, line 4\npage 1, line 5\fpage 2, line 1\npage 2, line 2\npage 2, line 3\npage 2, line 4\npage 2, line 5\fpage 3, line 1\npage 3, line 2\npage 3, line 3\npage 3, line 4\npage 3, line 5",[74,149,225]]

data/spec/fixtures/new.html

@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+  <body>
+    <p>fixture-html-new</p>
+  </body>
+</html>

data/spec/fixtures/new.txt

@@ -0,0 +1 @@
+fixture-txt-new

data/spec/lib/aipp/downloader_spec.rb

@@ -0,0 +1,81 @@
+require_relative '../../spec_helper'
+
+describe AIPP::Downloader do
+  let :fixtures_dir do
+    Pathname(__FILE__).join('..', '..', '..', 'fixtures')
+  end
+
+  let :tmp_dir do
+    Pathname(Dir.mktmpdir).tap do |tmp_dir|
+      (archives_dir = tmp_dir.join('archives')).mkpath
+      FileUtils.cp(fixtures_dir.join('archive.zip'), archives_dir)
+    end
+  end
+
+  after do
+    FileUtils.rm_rf(tmp_dir)
+  end
+
+  describe :read do
+    context "archive does not exist" do
+      it "creates the archive" do
+        Spy.on(Kernel, open: File.open(fixtures_dir.join('new.html')))
+        subject = AIPP::Downloader.new(storage: tmp_dir, archive: 'new-archive') do |downloader|
+          File.exist?(tmp_dir.join('work')).must_equal true
+          downloader.read(document: 'new', url: 'http://localhost/new.html')
+        end
+        zip_entries(subject.archive_file).must_equal %w(new.html)
+        subject.send(:archives_path).children.count.must_equal 2
+      end
+    end
+
+    context "archive does exist" do
+      it "unzips and uses the archive" do
+        Spy.on(Kernel, open: File.open(fixtures_dir.join('new.html')))
+        subject = AIPP::Downloader.new(storage: tmp_dir, archive: 'archive') do |downloader|
+          File.exist?(tmp_dir.join('work')).must_equal true
+          downloader.read(document: 'new', url: 'http://localhost/new.html').tap do |content|
+            content.must_be_instance_of Nokogiri::HTML5::Document
+            content.text.must_match /fixture-html-new/
+          end
+        end
+        zip_entries(subject.archive_file).must_equal %w(new.html one.html two.html)
+        subject.send(:archives_path).children.count.must_equal 1
+      end
+
+      it "downloads HTML documents to Nokogiri::HTML5::Document" do
+        Spy.on(Kernel, open: File.open(fixtures_dir.join('new.html')))
+        AIPP::Downloader.new(storage: tmp_dir, archive: 'archive') do |downloader|
+          downloader.read(document: 'new', url: 'http://localhost/new.html').tap do |content|
+            content.must_be_instance_of Nokogiri::HTML5::Document
+            content.text.must_match /fixture-html-new/
+          end
+        end
+      end
+
+      it "downloads and caches PDF documents to AIPP::PDF" do
+        Spy.on(Kernel, open: File.open(fixtures_dir.join('new.pdf')))
+        AIPP::Downloader.new(storage: tmp_dir, archive: 'archive') do |downloader|
+          downloader.read(document: 'new', url: 'http://localhost/new.pdf').tap do |content|
+            content.must_be_instance_of AIPP::PDF
+            content.text.must_match /fixture-pdf-new/
+          end
+        end
+      end
+
+      it "downloads explicitly specified type" do
+        Spy.on(Kernel, open: File.open(fixtures_dir.join('new.pdf')))
+        AIPP::Downloader.new(storage: tmp_dir, archive: 'archive') do |downloader|
+          downloader.read(document: 'new', url: 'http://localhost/new', type: :pdf).tap do |content|
+            content.must_be_instance_of AIPP::PDF
+            content.text.must_match /fixture-pdf-new/
+          end
+        end
+      end
+    end
+  end
+
+  def zip_entries(zip_file)
+    Zip::File.open(zip_file).entries.map(&:name).sort
+  end
+end