ul-wukong 4.1.0

data/examples/munging/airline_flights/indexable_spec.rb

@@ -0,0 +1,90 @@
+require 'spec_helper'
+require 'gorillib/model'
+require 'gorillib/pathname'
+#
+require 'gorillib/model/serialization'
+require 'gorillib/model/serialization/tsv'
+require 'gorillib/array/hashify'
+#
+require 'wu/model/indexable'
+
+describe Gorillib::Model::Indexable, :model_spec, :only do
+  let(:mock_array){ mock('array') }
+
+  let(:country_code_class) do
+    module Gorillib::Test
+      remove_const(:CountryCode) if defined?(CountryCode)
+
+      class CountryCode
+        include Gorillib::Model
+        include Gorillib::Model::Indexable
+        field :alpha_2_code,     String, position: 0
+        field :name,             String, position: 1
+        def self.load
+          self.values << new('dj', 'Djibouti')
+          self.values << new('us', 'United States of America')
+          values
+        end
+      end
+
+    end
+    Gorillib::Test::CountryCode
+  end
+
+  let(:djibouti){ country_code_class.new('dj', 'Djibouti') }
+  let(:usa     ){ country_code_class.new('us', 'United States of America') }
+
+  context 'test setup' do
+    subject{ country_code_class.load }
+    it{ should == [djibouti, usa] }
+  end
+
+  context '.values' do
+    # before{ country_code_class.send(:remove_instance_variable, '@values') }
+    it 'gets its values from .load' do
+      country_code_class.should_receive(:load).once.and_return mock_array
+      country_code_class.values.should equal(mock_array)
+    end
+    it 'memoizes once it is called' do
+      country_code_class.should_receive(:load).once.and_return mock_array
+      country_code_class.values.should equal(mock_array)
+      country_code_class.values.should equal(mock_array)
+    end
+  end
+
+  context '.index_on' do
+    it 'defines a .for_foo method' do
+      country_code_class.should_not respond_to(:for_name)
+      country_code_class.index_on(:name)
+      country_code_class.should respond_to(:for_name)
+      country_code_class.protected_methods.should include(:name_index)
+    end
+  end
+
+  context '.for_foo' do
+    before{ country_code_class.index_on :name }
+    context 'behaves like Hash#fetch:' do
+      context 'when key is not present' do
+        it 'retrieves a value if in the index' do
+          country_code_class.for_name('Djibouti').should == djibouti
+        end
+      end
+      context 'when key is not present' do
+        it 'and no default it raises KeyError' do
+          expect{ country_code_class.for_name('Yo Mama') }.to raise_error(KeyError, 'key not found: "Yo Mama"')
+        end
+        it 'returns default value if given' do
+          yo_mama = country_code_class.for_name('Yo Mama', 'wears combat boots')
+          yo_mama.should == 'wears combat boots'
+        end
+        it 'calls block if given' do
+          she = nil
+          so_fat = country_code_class.for_name('Yo Mama'){ she = 'sits around the house' ; 'when she sits' }
+          so_fat.should == 'when she sits'
+          she.should    == 'sits around the house'
+        end
+      end
+    end
+  end
+
+end

data/examples/munging/airline_flights/reconcile_airports.rb

@@ -0,0 +1,142 @@
+require_relative './models'
+require 'gorillib/model/reconcilable'
+
+class Airport
+  include Gorillib::Model::Reconcilable
+  attr_accessor :_origin # source of the record
+
+  def conflicting_attribute!(attr, this_val, that_val)
+    case attr
+    when :name, :city, :airport_ofid then return :pass
+    when :latitude, :longitude       then return true if (this_val - that_val).abs < 3
+    when :altitude                   then return true if (this_val - that_val).abs < 5
+    end
+    super
+  end
+
+  def ids
+    [:icao, :iata, :faa].hashify{|attr| public_send(attr) }.compact
+  end
+end
+
+#
+# Loads the Airport identifier tables scraped from Wikipedia
+#
+class RawAirportIdentifier < Airport
+  include RawAirport
+  include Gorillib::Model::LoadFromTsv
+
+  def self.from_tuple(icao, iata, faa, name, city=nil, *_)
+    self.new({icao: icao, iata: iata, faa: faa, name: name, city: city}.compact_blank)
+  end
+
+  def self.load_airports(filename, &block)
+    load_tsv(filename, num_fields: 4..6, &block)
+  end
+end
+
+class Airport
+  #
+  # Reconciler for Airports
+  #
+  # For each airport in turn across openflights, dataexpo and the two scraped
+  # identifier sets,
+  # 
+  #
+  class IdReconciler
+    include Gorillib::Model
+    include Gorillib::Model::LoadFromCsv
+    include Gorillib::Model::Reconcilable
+    self.csv_options = { col_sep: "\t", num_fields: 3..6 }
+
+    # Map the reconcilers to each ID they have anything to say about
+    ID_MAP = { icao: {}, iata: {}, faa: {} }
+    
+    field :opinions, Array, default: Array.new, doc: "every record having an id in common with the other records in this field"
+
+    def ids
+      opinions.flat_map{|op| op.ids.to_a }.uniq.compact
+    end
+
+    def self.load_all
+      Log.info "Loading all Airports and reconciling"
+      @airports = Array.new
+      RawDataexpoAirport  .load_airports(:dataexpo_raw_airports   ){|airport| register(:dataexpo, airport) }
+      RawOpenflightAirport.load_airports(:openflights_raw_airports){|airport| register(:openflights, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_icao          ){|airport| register(:wp_icao, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_iata          ){|airport| register(:wp_iata, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_us_abroad     ){|airport| register(:wp_us_abroad, airport) }
+
+      recs = ID_MAP.map{|attr, hsh| hsh.sort.map(&:last) }.flatten.uniq
+      recs.each do |rec|
+        consensus = rec.reconcile
+        # lint = consensus.lint
+        # puts "%-79s\t%s" % [lint, consensus.to_s[0..100]] if lint.present?
+        @airports << consensus
+      end
+    end
+
+    def self.airports
+      @airports
+    end
+
+    def self.exemplars
+      Airport::EXEMPLARS.map do |iata|
+        ID_MAP[:iata][iata].reconcile
+      end
+    end
+
+    def reconcile
+      consensus = Airport.new
+      clean = opinions.all?{|op| consensus.adopt(op) }
+      # puts "\t#{consensus.inspect}"
+      puts "confl\t#{self.inspect}" if not clean
+      consensus
+    end
+
+    def adopt_opinions(vals, _)
+      self.opinions = vals + self.opinions
+      self.opinions.uniq!
+    end
+
+    # * find all existing reconcilers that share an ID with that record
+    # * unify them into one reconciler
+    # * store it back under all the IDs
+    #
+    # Suppose our dataset has 3 identifiers, which look like
+    #
+    #     a    S
+    #          S    88
+    #     a    Z
+    #     b
+    #          Q
+    #     b    Q    77
+    #
+    # We will wind up with these two reconcilers:
+    #
+    #     <a S 88 opinions: [a,S, ],[S, ,88],[a,Z,  ]>
+    #     <b Q 77 opinions: [b, , ],[ ,Q,  ],[b,Q,77]>
+    #
+    def self.register(origin, obj)
+      obj._origin = origin
+      # get the existing reconcilers
+      existing   = obj.ids.map{|attr, id| ID_MAP[attr][id] }.compact.uniq
+      # push the new object in, and pull the most senior one out
+      existing.unshift(self.new(opinions: [obj]))
+      reconciler = existing.shift
+      # unite them into the reconciler
+      existing.each{|that| reconciler.adopt(that) }
+      # save the reconciler under each of the ids.
+      reconciler.ids.each{|attr, id| ID_MAP[attr][id] = reconciler }
+    end
+
+    def inspect
+      str = "#<#{self.class.name} #{ids}"
+      opinions.each do |op|
+        str << "\n\t  #{op._origin}\t#{op}"
+      end
+      str << ">"
+    end
+  end
+
+end

data/examples/munging/airline_flights/tasks.rake

@@ -0,0 +1,83 @@
+require_relative('../../rake_helper')
+require_relative('./models')
+
+Pathname.register_paths(
+  af_data:                  [:data, 'airline_flights'],
+  af_work:                  [:work, 'airline_flights'],
+  af_code:                  File.dirname(__FILE__),
+  #
+  openflights_raw_airports: [:af_data, "openflights_airports-raw#{Settings[:mini_slug]}.csv"   ],
+  openflights_raw_airlines: [:af_data, "openflights_airlines-raw.csv"   ],
+  dataexpo_raw_airports:    [:af_data, "dataexpo_airports-raw#{Settings[:mini_slug]}.csv"      ],
+  wikipedia_icao:           [:af_data, "wikipedia_icao.tsv" ],
+  wikipedia_iata:           [:af_data, "wikipedia_iata.tsv" ],
+  wikipedia_us_abroad:      [:af_data, "wikipedia_us_abroad.tsv" ],
+  #
+  openflights_airports:     [:af_work, "openflights_airports-parsed#{Settings[:mini_slug]}.tsv"],
+  openflights_airlines:     [:af_work, "openflights_airlines-parsed#{Settings[:mini_slug]}.tsv"],
+  dataexpo_airports:        [:af_work, "dataexpo_airports-parsed#{Settings[:mini_slug]}.tsv"   ],
+  airport_identifiers:      [:af_work, "airport_identifiers.tsv"   ],
+  airport_identifiers_mini: [:af_work, "airport_identifiers-sample.tsv"   ],
+  # helpers
+  country_name_lookup:       [:work, 'geo', "country_name_lookup.tsv"],
+  )
+
+chain :airline_flights do
+  code_files = FileList[Pathname.of(:af_code, '*.rb').to_s]
+  chain(:parse) do
+
+    # desc 'parse the dataexpo airports'
+    # create_file(:dataexpo_airports, after: code_files) do |dest|
+    #   RawDataexpoAirport.load_airports(:dataexpo_raw_airports) do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+
+    desc 'parse the openflights airports'
+    create_file(:openflights_airports, after: [code_files, :force]) do |dest|
+      require_relative('../geo/geo_models')
+      Geo::CountryNameLookup.load
+      RawOpenflightAirport.load_airports(:openflights_raw_airports) do |airport|
+        dest << airport.to_tsv << "\n"
+        # puts airport.country
+      end
+    end
+
+    # task :reconcile_airports => [:dataexpo_airports, :openflights_airports] do
+    #   require_relative 'reconcile_airports'
+    #   Airport::IdReconciler.load_all
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.airports.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers_mini, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.exemplars.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'parse the openflights airlines'
+    # create_file(:openflights_airlines, after: code_files) do |dest|
+    #   RawOpenflightAirline.load_airlines(:openflights_raw_airlines) do |airline|
+    #     dest << airline.to_tsv << "\n"
+    #     puts airline.to_tsv
+    #   end
+    # end
+
+  end
+end
+
+task :default => [
+  'airline_flights',
+  # 'airline_flights:parse:dataexpo_airports',
+  # 'airline_flights:parse:openflights_airports',
+  # 'airline_flights:parse:airport_identifiers',
+  # 'airline_flights:parse:airport_identifiers_mini',
+  # 'airline_flights:parse:openflights_airlines',
+]

data/examples/munging/airline_flights/topcities.rb

@@ -0,0 +1,167 @@
+#!/usr/bin/env ruby
+require('rake')
+require_relative('../../rake_helper')
+require_relative './models'
+
+Pathname.register_paths(
+  af_data:                  [:data, 'airline_flights'],
+  af_work:                  [:work, 'airline_flights'],
+  af_code:                  File.dirname(__FILE__),
+  airport_identifiers:      [:af_work, "airport_identifiers.tsv"   ],
+  )
+
+AIRPORTS_TO_MATCH = [
+  [ 'Tokyo',              1, "HND", ],
+  [ 'Guangzhou',          2, "CAN", ],
+  [ 'Seoul',              3, "ICN", ],
+  [ 'Shanghai',           4, "PVG", ],
+  [ 'Mexico.*City',       5, "MEX", ],
+  [ 'Delhi',              6, "DEL", ],
+  [ 'New.*York',          7, "JFK", ],
+  [ 'S.*o.*Paulo',        8, "GRU", ],
+  [ 'Mumbai|Bombay',      9, "BOM", ],
+  [ 'Manila',            10, "MNL", ],
+  [ 'Jakarta',           11, "CGK", ],
+  [ 'Los.*Angeles',      12, "LAX", ],
+  [ 'Karachi',           13, "KHI", ],
+  [ 'Osaka',             14, "KIX", ],
+  [ 'Beijing',           15, "PEK", ],
+  [ 'Moscow',            16, "SVO", ],
+  [ 'Cairo',             17, "CAI", ],
+  [ 'Kolkata|Calcutta',  18, "CCU", ],
+  [ 'Buenos.*Aires',     19, "EZE", ],
+  [ 'Dhaka',             20, "DAC", ],
+  [ 'Bangkok',           21, "BKK", ],
+  [ 'Tehran|Abyek',      22, "IKA", ],
+  [ 'Istanbul',          23, "IST", ],
+  [ 'Janeiro',           24, "GIG", ],
+  [ 'London',            25, "LHR", ],
+  [ 'Lagos',             26, "LOS", ],
+  [ 'Paris',             27, "CDG", ],
+  [ 'Chicago',           28, "ORD", ],
+  [ 'Kinshasa',          29, "FIH", ],
+  [ 'Lima',              30, "LIM", ],
+  [ 'Wuhan',             31, "WUH", ],
+  [ 'Bangalore',         32, "BLR", ],
+  [ 'Bogot.*',           33, "BOG", ],
+  [ 'Taipei',            34, "TSA", ],
+  [ 'Washington|Arling', 35, "DCA", ],
+  [ 'Johannesburg',      36, "JNB", ],
+  [ 'Saigon|Ho.Chi.M',   37, "SGN", ],
+  [ 'San.*Francisco',    38, "SFO", ],
+  [ 'Boston',            39, "BOS", ],
+  [ 'Hong.*Kong',        40, "HKG", ],
+  [ 'Baghdad',           41, "SDA", ],
+  [ 'Madrid',            42, "MAD", ],
+  [ 'Singapore',         43, "SIN", ],
+  [ 'Kuala.*Lumpur',     44, "KUL", ],
+  [ 'Chongqing|Chung.*', 45, "CKG", ],
+  [ 'Santiago',          46, "SCL", ],
+  [ 'Toronto',           47, "YYZ", ],
+  [ 'Riyadh',            48, "RUH", ],
+  [ 'Atlanta',           49, "ATL", ],
+  [ 'Miami',             50, "MIA", ],
+  [ 'Detroit',           51, "DTW", ],
+  [ 'St..*Petersburg',   52, "LED", ],
+  [ 'Khartoum',          53, "KRT", ],
+  [ 'Sydney',            54, "SYD", ],
+  [ 'Milan',             55, "MXP", ],
+  [ 'Abidjan',           56, "ABJ", ],
+  [ 'Barcelona',         57, "BCN", ],
+  [ 'Nairobi',           58, "NBO", ],
+  [ 'Caracas',           59, "CCS", ],
+  [ 'Monterrey',         60, "MTY", ],
+  [ 'Phoenix',           61, "PHX", ],
+  [ 'Berlin',            62, "TXL", ],
+  [ 'Melbourne',         63, "MEL", ],
+  [ 'Casablanca',        64, "CMN", ],
+  [ 'Montreal',          65, "YUL", ],
+  [ 'Salvador',          66, "SSA", ],
+  [ 'Rome',              67, "FCO", ],
+  [ 'Kiev',              68, "KBP", ],
+  [ 'Ad+is.*Ab.ba',      69, "ADD", ],
+  [ 'Denver',            70, "DEN", ],
+  [ 'St.*Louis',         71, "STL", ],
+  [ 'Dakar',             72, "DKR", ],
+  [ 'San.*Juan',         73, "SJU", ],
+  [ 'Vancouver',         74, "YVR", ],
+  [ 'Tel.*Aviv',         75, "TLV", ],
+  [ 'Tunis',             76, "TUN", ],
+  [ 'Portland',          77, "PDX", ],
+  [ 'Manaus',            78, "MAO", ],
+  [ 'Calgary',           79, "YYC", ],
+  [ 'Halifax',           80, "YHZ", ],
+  [ 'Prague',            81, "PRG", ],
+  [ 'Copenhagen',        82, "CPH", ],
+  [ 'Djibouti',          83, "JIB", ],
+  [ 'Quito',             84, "UIO", ],
+  [ 'Helsinki',          85, "HEL", ],
+  [ 'Papeete|Tahiti',    86, "PPT", ],
+  [ 'Frankfurt',         87, "FRA", ],
+  [ 'Reykjavik',         88, "RKV", ],
+  [ 'Riga',              89, "RIX", ],
+  [ 'Antananarivo',      90, "TNR", ],
+  [ 'Amsterdam',         91, "AMS", ],
+  [ 'Bucharest',         92, "OTP", ],
+  [ 'Novosibirsk',       93, "OVB", ],
+  [ 'Kigali',            94, "KGL", ],
+  [ 'Dushanbe',          95, "DYU", ],
+  [ 'Dubai',             96, "DXB", ],
+  [ 'Bermuda',           97, "BDA", ],
+  [ 'Anchorage',         98, "ANC", ],
+  [ 'Austin',            99, "AUS", ],
+  [ 'Honolulu',         100, "HNL", ],
+  [ 'Apia',             101, "FGI", ],
+  [ 'Vienna',           102, "VIE", ],
+  [ 'Brussels',         103, "BRU", ],
+  [ 'Munich',           104, "MUC", ],
+  [ 'Dublin',           105, "DUB", ],
+  [ 'Doha',             106, "DOH", ],
+  [ 'Taipei',           107, "TPE", ],
+  [ 'Yakutsk',          108, "YKS", ],
+  [ 'Z.rich',           109, "ZRH", ],
+  [ 'Manchester',       110, "MAN", ],
+  [ 'Houston',          111, "IAH", ],
+  [ 'Charlotte',        112, "CLT", ],
+  [ 'Dallas',           113, "DFW", ],
+  [ 'Las.*Vegas',       114, "LAS", ],
+  [ 'Antalya',          115, "AYT", ],
+  [ 'Auckland',         116, "AKL", ],
+]
+
+MATCHED_AIRPORTS = {}
+MATCH_ON_IATA = {}
+MATCH_ON_CITY = {}
+match_on_city_names = []
+
+AIRPORTS_TO_MATCH.each do |name, idx, iata|
+  hsh = {iata: iata, re: Regexp.new(name, 'i'), name: name, idx: idx}
+  if iata.present?
+    MATCH_ON_IATA[iata] = hsh
+  else
+    match_on_city_names << name
+    MATCH_ON_CITY[hsh[:re]] = hsh
+  end
+end
+match_on_city_re = Regexp.new(match_on_city_names.join('|'))
+
+Airport.load_tsv(:airport_identifiers) do |airport|
+  airport.name = airport.name[0..30]
+  if   MATCH_ON_IATA.include?(airport.iata)
+    hsh = MATCH_ON_IATA[airport.iata]
+    warn [hsh.values, airport.to_tsv].flatten.join("\t") unless hsh[:re] =~ airport.city
+    MATCHED_AIRPORTS[hsh[:idx]] = airport
+  # elsif (airport.city =~ match_on_city_re)
+  #   MATCH_ON_CITY.each do |re, hsh|
+  #     if (airport.city =~ re)
+  #       puts [airport.to_tsv, hsh[:name], hsh[:idx]].join("\t")
+  #     end
+  #   end
+  end
+end
+
+AIRPORTS_TO_MATCH.each do |name, idx, iata|
+  # next if MATCHED_AIRPORTS[idx]
+  airport_str = MATCHED_AIRPORTS[idx] ? MATCHED_AIRPORTS[idx].to_tsv : "\t\t\t\t\t\t\t\t\t\t\t\t"
+  puts [airport_str, name, "", idx].join("\t")
+end