wukong 3.0.0.pre → 3.0.0.pre2

data/examples/jabberwocky.txt

@@ -0,0 +1,36 @@
+"Jabberwocky"
+
+'Twas brillig, and the slithy toves
+Did gyre and gimble in the wabe;
+All mimsy were the borogoves,
+And the mome raths outgrabe.
+
+"Beware the Jabberwock, my son!
+The jaws that bite, the claws that catch!
+Beware the Jubjub bird, and shun
+The frumious Bandersnatch!"
+
+He took his vorpal sword in hand:
+Long time the manxome foe he sought—
+So rested he by the Tumtum tree,
+And stood awhile in thought.
+
+And as in uffish thought he stood,
+The Jabberwock, with eyes of flame,
+Came whiffling through the tulgey wood,
+And burbled as it came!
+
+One, two! One, two! and through and through
+The vorpal blade went snicker-snack!
+He left it dead, and with its head
+He went galumphing back.
+
+"And hast thou slain the Jabberwock?
+Come to my arms, my beamish boy!
+O frabjous day! Callooh! Callay!"
+He chortled in his joy.
+
+'Twas brillig, and the slithy toves
+Did gyre and gimble in the wabe;
+All mimsy were the borogoves,
+And the mome raths outgrabe.

data/examples/models/wikipedia.rb

@@ -0,0 +1,20 @@
+
+module Wu
+  module Data
+    module Wikipedia
+
+      class Article
+        include Gorillib::Model
+        field :title,        String
+        field :namespace,    Integer
+        field :id,           Integer
+        field :restrictions, String
+        field :revision_id,  String
+        field :timestamp,    String
+        field :sha1,         String
+        field :redirect,     String
+        field :xml_text,     String
+
+    end
+  end
+end

data/examples/munging/Gemfile

@@ -0,0 +1,8 @@
+source 'http://rubygems.org'
+  
+gem 'gorillib', :path => '/home/dlaw/dev/gorillib'
+gem 'ruby-progressbar'
+gem 'crack'
+gem 'rake'
+gem 'wukong', :path => '/home/dlaw/dev/wukong_og'
+gem 'json'

data/examples/munging/airline_flights/airline.rb

@@ -0,0 +1,57 @@
+class Airline
+  include Gorillib::Model
+  field :icao_id,      String,   doc: "3-letter ICAO code, if available", identifier: true, length: 2
+  field :iata_id,      String,   doc: "2-letter IATA code, if available", identifier: true, length: 2
+  field :airline_ofid, Integer,  doc: "Unique OpenFlights identifier for this airline.", identifier: true
+  field :active,       :boolean, doc: 'true if the airline is or has until recently been operational, false if it is defunct. (This is only a rough indication and should not be taken as 100% accurate)'
+  field :country,      String,   doc: "Country or territory where airline is incorporated"
+  field :name,         String,   doc: "Airline name."
+  field :callsign,     String,   doc: "Airline callsign", identifier: true
+  field :alias,        String,   doc: "Alias of the airline. For example, 'All Nippon Airways' is commonly known as 'ANA'"
+end
+
+#
+# As of January 2012, the OpenFlights Airlines Database contains 5888
+# airlines. If you enjoy this data, please consider [visiting their page and
+# donating](http://openflights.org/data.html)
+#
+# > Notes: Airlines with null codes/callsigns/countries generally represent
+# > user-added airlines. Since the data is intended primarily for current
+# > flights, defunct IATA codes are generally not included. For example,
+# > "Sabena" is not listed with a SN IATA code, since "SN" is presently used by
+# > its successor Brussels Airlines.
+#
+# Sample entries
+#
+#     324,"All Nippon Airways","ANA All Nippon Airways","NH","ANA","ALL NIPPON","Japan","Y"
+#     412,"Aerolineas Argentinas",\N,"AR","ARG","ARGENTINA","Argentina","Y"
+#     413,"Arrowhead Airways",\N,"","ARH","ARROWHEAD","United States","N"
+#
+class RawOpenflightAirline
+  include Gorillib::Model
+  include Gorillib::Model::LoadFromCsv
+  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
+
+  field :airline_ofid, Integer,  blankish: BLANKISH_STRINGS, doc: "Unique OpenFlights identifier for this airline.", identifier: true
+  field :name,         String,   blankish: BLANKISH_STRINGS, doc: "Airline name."
+  field :alias,        String,   blankish: BLANKISH_STRINGS, doc: "Alias of the airline. For example, 'All Nippon Airways' is commonly known as 'ANA'"
+  field :iata_id,      String,   blankish: BLANKISH_STRINGS, doc: "2-letter IATA code, if available", identifier: true, length: 2
+  field :icao_id,      String,   blankish: BLANKISH_STRINGS, doc: "3-letter ICAO code, if available", identifier: true, length: 2
+  field :callsign,     String,   blankish: BLANKISH_STRINGS, doc: "Airline callsign"
+  field :country,      String,   blankish: BLANKISH_STRINGS, doc: "Country or territory where airline is incorporated"
+  field :active,       :boolean, blankish: BLANKISH_STRINGS, doc: 'true if the airline is or has until recently been operational, false if it is defunct. (This is only a rough indication and should not be taken as 100% accurate)'
+
+  def receive_iata_id(val) super if val =~ /\A\w+\z/ ; end
+  def receive_icao_id(val) super if val =~ /\A\w+\z/ ; end
+  def receive_active(val)
+    super(case val.to_s when "Y" then true when "N" then false else val ; end)
+  end
+
+  def to_airline
+    Airline.receive(self.compact_attributes)
+  end
+
+  def self.load_airlines(filename)
+    load_csv(filename){|raw_airline| yield(raw_airline.to_airline) }
+  end
+end

data/examples/munging/airline_flights/airline_flights.rake

@@ -0,0 +1,83 @@
+require_relative('../../rake_helper')
+require_relative('./models')
+
+Pathname.register_paths(
+  af_data:                  [:data, 'airline_flights'],
+  af_work:                  [:work, 'airline_flights'],
+  af_code:                  File.dirname(__FILE__),
+  #
+  openflights_raw_airports: [:af_data, "openflights_airports-raw#{Settings[:mini_slug]}.csv"   ],
+  openflights_raw_airlines: [:af_data, "openflights_airlines-raw.csv"   ],
+  dataexpo_raw_airports:    [:af_data, "dataexpo_airports-raw#{Settings[:mini_slug]}.csv"      ],
+  wikipedia_icao:           [:af_data, "wikipedia_icao.tsv" ],
+  wikipedia_iata:           [:af_data, "wikipedia_iata.tsv" ],
+  wikipedia_us_abroad:      [:af_data, "wikipedia_us_abroad.tsv" ],
+  #
+  openflights_airports:     [:af_work, "openflights_airports-parsed#{Settings[:mini_slug]}.tsv"],
+  openflights_airlines:     [:af_work, "openflights_airlines-parsed#{Settings[:mini_slug]}.tsv"],
+  dataexpo_airports:        [:af_work, "dataexpo_airports-parsed#{Settings[:mini_slug]}.tsv"   ],
+  airport_identifiers:      [:af_work, "airport_identifiers.tsv"   ],
+  airport_identifiers_mini: [:af_work, "airport_identifiers-sample.tsv"   ],
+  # helpers
+  country_name_lookup:       [:work, 'geo', "country_name_lookup.tsv"],
+  )
+
+chain :airline_flights do
+  code_files = FileList[Pathname.of(:af_code, '*.rb').to_s]
+  chain(:parse) do
+
+    # desc 'parse the dataexpo airports'
+    # create_file(:dataexpo_airports, after: code_files) do |dest|
+    #   RawDataexpoAirport.load_airports(:dataexpo_raw_airports) do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+
+    desc 'parse the openflights airports'
+    create_file(:openflights_airports, after: [code_files, :force]) do |dest|
+      require_relative('../geo/geo_models')
+      Geo::CountryNameLookup.load
+      RawOpenflightAirport.load_airports(:openflights_raw_airports) do |airport|
+        dest << airport.to_tsv << "\n"
+        # puts airport.country
+      end
+    end
+
+    # task :reconcile_airports => [:dataexpo_airports, :openflights_airports] do
+    #   require_relative 'reconcile_airports'
+    #   Airport::IdReconciler.load_all
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.airports.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers_mini, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.exemplars.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'parse the openflights airlines'
+    # create_file(:openflights_airlines, after: code_files) do |dest|
+    #   RawOpenflightAirline.load_airlines(:openflights_raw_airlines) do |airline|
+    #     dest << airline.to_tsv << "\n"
+    #     puts airline.to_tsv
+    #   end
+    # end
+
+  end
+end
+
+task :default => [
+  'airline_flights',
+  # 'airline_flights:parse:dataexpo_airports',
+  # 'airline_flights:parse:openflights_airports',
+  # 'airline_flights:parse:airport_identifiers',
+  # 'airline_flights:parse:airport_identifiers_mini',
+  # 'airline_flights:parse:openflights_airlines',
+]

data/examples/munging/airline_flights/airport.rb

@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+
+### @export "airport_model"
+class Airport
+  include Gorillib::Model
+
+  field :icao,         String, doc: "4-letter ICAO code, or blank if not assigned.", length: 4, identifier: true, :blankish => ["", nil]
+  field :iata,         String, doc: "3-letter IATA code, or blank if not assigned.", length: 3, identifier: true, :blankish => ["", nil]
+  field :faa,          String, doc: "3-letter FAA code, or blank if not assigned.",  length: 3, identifier: true, :blankish => ["", nil]
+  field :utc_offset,   Float,  doc: "Hours offset from UTC. Fractional hours are expressed as decimals, eg. India is 5.5.",      validates: {  inclusion: (-12...12) }
+  field :dst_rule,     String, doc: "Daylight savings time rule. One of E (Europe), A (US/Canada), S (South America), O (Australia), Z (New Zealand), N (None) or U (Unknown). See the readme for more.", validates: {  inclusion: %w[E A S O Z N U] }
+  field :longitude,    Float,  doc: "Decimal degrees, usually to six significant digits. Negative is West,  positive is East.",  validates: {  inclusion: (-180...180) }
+  field :latitude,     Float,  doc: "Decimal degrees, usually to six significant digits. Negative is South, positive is North.", validates: {  inclusion: (-90.0...90.0) }
+  field :altitude,     Float,  doc: "Elevation in meters."
+  field :name,         String, doc: "Name of airport."
+  field :country,      String, doc: "Country or territory where airport is located.", length: 2
+  field :state,        String, doc: "State in which the airport is located",          length: 2
+  field :city,         String, doc: "Main city served by airport. This is the logical city it serves; so, for example SFO gets 'San Francisco', not 'San Bruno'"
+  field :airport_ofid, String, doc: "OpenFlights identifier for this airport.", identifier: true
+end
+### @export "nil"
+class Airport
+  EXEMPLARS = %w[
+    ANC ATL AUS BDL BNA BOI BOS BWI CLE CLT
+    CMH DCA DEN DFW DTW EWR FLL HNL IAD IAH
+    IND JAX JFK LAS LAX LGA MCI MCO MDW MIA
+    MSP MSY OAK ORD PDX PHL PHX PIT PVD RDU
+    SAN SEA SFO SJC SJU SLC SMF STL TPA YYZ
+  ]
+
+  def utc_time_for(tm)
+    utc_time  = tm.get_utc + utc_offset
+    utc_time += (60*60) if TimezoneFixup.dst?(tm)
+    utc_time
+  end
+
+  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
+  OK_CHARS_RE      = /[^a-zA-Z0-9\:\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÓÖØÚÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýĀāăĆćČčēėęěğīİıŁłńņňŌōőřŞşŠšţťūůųźŽžơț]/
+
+  def lint
+    errors = {}
+    errors["ICAO is wrong length"] = icao if icao.present? && icao.length != 4
+    if (icao && faa && (icao =~ /^K.../))
+      errors["ICAO != K+FAA yet ICAO is a K..."] = [icao, faa] if (icao != "K#{faa}")
+    end
+    # errors["ICAO present for piddlyshit airport"] = icao if icao.present? && ((faa.to_s.length == 4) || (faa.to_s =~ /\d/))
+    errors[:spaces] ||= []
+    errors[:funny]  ||= []
+    attributes.each do |attr, val|
+      next if val.blank?
+      errors["#{attr} looks blankish"] = val if BLANKISH_STRINGS.include?(val)
+      if (val.is_a?(String))
+        errors[:spaces] << [attr, val] if  (val.strip != val)
+        errors[:funny]  << [attr, val]  if val =~ OK_CHARS_RE
+      end
+    end
+    errors.compact_blank
+  end
+
+  def to_s
+    str = "#<Airport "
+    str << [icao, iata, faa,
+      (latitude && "%4.1f" % latitude), (longitude && "%5.1f" % longitude), state, country,
+      "%-30s" % name, country, city].join("\t")
+    str << ">"
+  end
+
+  def faa_controlled?
+    icao =~ /^(?:K|P[ABFGHJKMOPW]|T[IJ]|NS(AS|FQ|TU))/
+  end
+end
+### @export "airport_load"
+class Airport
+  include Gorillib::Model::LoadFromTsv
+  self.tsv_options.merge!(num_fields: 10..20)
+  def self.load_airports(filename)
+    load_tsv(filename){|airport| yield(airport) }
+  end
+
+end
+### @export "nil"
+
+#
+# As of January 2012, the OpenFlights Airports Database contains 6977 airports
+# [spanning the globe](http://openflights.org/demo/openflights-apdb-2048.png).
+# If you enjoy this data, please consider [visiting their page and
+# donating](http://openflights.org/data.html)
+#
+# > Note: Rules for daylight savings time change from year to year and from
+# > country to country. The current data is an approximation for 2009, built on
+# > a country level. Most airports in DST-less regions in countries that
+# > generally observe DST (eg. AL, HI in the USA, NT, QL in Australia, parts of
+# > Canada) are marked incorrectly.
+#
+# Sample entries
+#
+#     507,"Heathrow","London","United Kingdom","LHR","EGLL",51.4775,-0.461389,83,0,"E"
+#     26,"Kugaaruk","Pelly Bay","Canada","YBB","CYBB",68.534444,-89.808056,56,-6,"A"
+#     3127,"Pokhara","Pokhara","Nepal","PKR","VNPK",28.200881,83.982056,2712,5.75,"N"
+#
+
+### @export "raw_openflight_airport"
+
+module RawAirport
+  COUNTRIES        = { 'Puerto Rico' => 'us', 'Canada' => 'ca', 'USA' => 'us', 'United States' => 'us',
+    'Northern Mariana Islands' => 'us', 'N Mariana Islands' => 'us',
+    'Federated States of Micronesia' => 'fm',
+    'Thailand' => 'th', 'Palau' => 'pw',
+    'American Samoa' => 'as', 'Wake Island' => 'us', 'Virgin Islands' => 'vi', 'Guam' => 'gu'
+  }
+  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
+  OK_CHARS_RE      = /[^a-zA-Z0-9\:\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÓÖØÚÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýĀāăĆćČčēėęěğīİıŁłńņňŌōőřŞşŠšţťūůųźŽžơț]/
+
+  def receive_city(val)
+    super.tap{|val| if val then val.strip! ; val.gsub!(/\\+/, '') ; end }
+  end
+
+  def receive_country(val)
+    super(COUNTRIES[val] || val)
+  end
+
+  def receive_name(val)
+    super.tap do |val|
+      if val
+        val.strip!
+        val.gsub!(/\\+/, '')
+        val.gsub!(/\s*\[(military|private)\]/, '')
+        val.gsub!(/\b(Int\'l|International)\b/, 'Intl')
+        val.gsub!(/\b(Intercontinental)\b/,     'Intcntl')
+        val.gsub!(/\b(Airpt)\b/,                'Airport')
+        val.gsub!(/ Airport$/, '')
+      end
+    end
+  end
+end
+
+#
+class RawOpenflightAirport
+  include Gorillib::Model
+  include Gorillib::Model::LoadFromCsv
+  include RawAirport
+  #
+  field :airport_ofid, String, doc: "Unique OpenFlights identifier for this airport."
+  field :name,        String, doc: "Name of airport. May or may not contain the City name."
+  field :city,        String, blankish: BLANKISH_STRINGS, doc: "Main city served by airport. May be spelled differently from Name."
+  field :country,     String, doc: "Country or territory where airport is located."
+  field :iata_faa,    String, blankish: BLANKISH_STRINGS, doc: "3-letter FAA code, for airports located in the USA. For all other airports, 3-letter IATA code, or blank if not assigned."
+  field :icao,        String, blankish: BLANKISH_STRINGS, doc: "4-letter ICAO code; Blank if not assigned."
+  field :latitude,    Float,  doc: "Decimal degrees, usually to six significant digits. Negative is South, positive is North."
+  field :longitude,   Float,  doc: "Decimal degrees, usually to six significant digits. Negative is West,  positive is East."
+  field :altitude_ft, Float,  blankish: ['', nil, 0, '0'], doc: "In feet."
+  field :utc_offset,  Float,  doc: "Hours offset from UTC. Fractional hours are expressed as decimals, eg. India is 5.5."
+  field :dst_rule,    String, doc: "Daylight savings time rule. One of E (Europe), A (US/Canada), S (South America), O (Australia), Z (New Zealand), N (None) or U (Unknown). See the readme for more."
+
+  UNRELIABLE_OPENFLIGHTS_IATA_VALUES = /^(7AK|AGA|AUQ|BDJ|BGW|BME|BPM|BXH|BZY|CAT|CEE|CEJ|CFS|CGU|CIO|CLV|CNN|DEE|DIB|DNM|DUH|DUR|FKI|GES|GSM|HKV|HOJ|HYD|IEO|IFN|IKA|IZA|JCU|JGS|KMW|KNC|LGQ|LUM|MCU|MCY|MDO|MOH|MON|MPH|MVF|NAY|NMA|NOE|NQY|OTU|OUI|PBV|PCA|PCB|PGK|PHO|PIF|PKN|PKY|PMK|PTG|PZO|QAS|QKT|QVY|RCM|RJL|RTG|SBG|SDZ|SFG|SIC|SIQ|SJI|SRI|STP|STU|SWQ|TJQ|TJS|TMC|TYA|UKC|VIY|VQS|VTS|WDH|WKM|WPR|WPU|ZQF)$/
+
+  def id_is_faa?
+    (icao =~ /^(?:K)/) || (icao.blank? && country == 'us')
+  end
+
+  def iata ;  (id_is_faa? ? nil      : iata_faa) unless iata_faa =~ UNRELIABLE_OPENFLIGHTS_IATA_VALUES end
+  def faa  ;  (id_is_faa? ? iata_faa : nil     ) end
+  def altitude
+    altitude_ft && (0.3048 * altitude_ft).round(1)
+  end
+
+  def receive_country(val)
+    country = Geo::CountryNameLookup.for_alt_name(val, nil)
+    p val unless country
+    super(country ? country.country_id : val)
+  end
+
+  def to_airport
+    attrs = self.compact_attributes.except(:altitude_ft)
+    attrs[:altitude] = altitude
+    attrs[:iata]     = iata unless iata.to_s =~ UNRELIABLE_OPENFLIGHTS_IATA_VALUES
+    attrs[:faa]      = faa
+    Airport.receive(attrs)
+  end
+
+  def self.load_airports(filename)
+    load_csv(filename){|raw_airport| yield(raw_airport.to_airport) }
+  end
+end
+
+### @export "raw_dataexpo_airport"
+class RawDataexpoAirport
+  include Gorillib::Model
+  include Gorillib::Model::LoadFromCsv
+  include RawAirport
+  self.csv_options = self.csv_options.merge(pop_headers: true)
+
+  field :faa,          String, doc: "the international airport abbreviation code"
+  field :name,         String, doc: "Airport name"
+  field :city,         String, blankish: ["NA"], doc: "city in which the airport is located"
+  field :state,        String, blankish: ["NA"], doc: "state in which the airport is located"
+  field :country,      String, doc: "country in which airport is located"
+  field :latitude,     Float,  doc: "latitude of the airport"
+  field :longitude,    Float,  doc: "longitude of the airport"
+
+  def to_airport
+    attrs = self.compact_attributes
+    attrs[:icao] = "K#{faa}" if faa =~ /[A-Z]{3}/ && (not ['PR', 'AK', 'CQ', 'HI', 'AS', 'GU', 'VI'].include?(state)) && (country == 'us')
+    Airport.receive(attrs)
+  end
+
+  def self.load_airports(filename)
+    load_csv(filename){|raw_airport| yield(raw_airport.to_airport) }
+  end
+end
+### @export "nil"

data/examples/munging/airline_flights/airport_id_unification.rb

@@ -0,0 +1,129 @@
+class Airport
+
+  # [Hash] all options passed to the field not recognized by one of its own current fields
+  attr_reader :_extra_attributes
+
+    # # Airports whose IATA and FAA codes differ; all are in the US, so their ICAO is "K"+the FAA id
+    # FAA_ICAO_FIXUP = {
+    #   "GRM" => "CKC", "CLD" => "CRQ", "SDX" => "SEZ", "AZA" => "IWA", "SCE" => "UNV", "BLD" => "BVU",
+    #   "LKE" => "W55", "HSH" => "HND", "BKG" => "BBG", "UST" => "SGJ", "LYU" => "ELO", "WFK" => "FVE",
+    #   "FRD" => "FHR", "ESD" => "ORS", "RKH" => "UZA", "NZC" => "VQQ", "SCF" => "SDL", "JCI" => "IXD",
+    #   "AVW" => "AVQ", "UTM" => "UTA", "ONP" => "NOP", }
+    #
+    # [:iata, :icao, :latitude, :longitude, :country, :city, :name].each do |attr|
+    #   define_method("of_#{attr}"){ @_extra_attributes[:"of_#{attr}"] }
+    #   define_method("de_#{attr}"){ @_extra_attributes[:"de_#{attr}"] }
+    # end
+    #
+    # def lint_differences
+    #   errors = {}
+    #   return errors unless de_name.present? && of_name.present?
+    #   [
+    #     [:iata, of_iata, de_iata], [:icao, of_icao, de_icao], [:country, of_country, de_country],
+    #     [:city, of_city, de_city],
+    #     [:name, of_name, de_name],
+    #   ].each{|attr, of, de| next unless of && de ; errors[attr] = [of, de] if of != de }
+    #
+    #   if (of_latitude && of_longitude && de_latitude && de_longitude)
+    #     lat_diff = (of_latitude  - de_latitude ).abs
+    #     lng_diff = (of_longitude - de_longitude).abs
+    #     unless (lat_diff < 0.015) && (lng_diff < 0.015)
+    #       msg = [of_latitude, de_latitude, of_longitude, de_longitude, lat_diff, lng_diff].map{|val| "%9.4f" % val }.join(" ")
+    #       errors["distance"] = ([msg, of_city, de_city, of_name, de_name])
+    #     end
+    #   end
+    #
+    #   errors
+    # end
+    #
+    # AIRPORTS      = Hash.new # unless defined?(AIRPORTS)
+    # def self.load(of_filename, de_filename)
+    #   RawOpenflightAirport.load_csv(of_filename) do |raw_airport|
+    #     airport = raw_airport.to_airport
+    #     AIRPORTS[airport.iata_icao] = airport
+    #   end
+    #   RawDataexpoAirport.load_csv(de_filename) do |raw_airport|
+    #     airport = (AIRPORTS[raw_airport.iata_icao] ||= self.new)
+    #     if airport.de_name
+    #       warn "duplicate data for #{[iata, de_iata, icao, de_icao]}: #{raw_airport.to_tsv} #{airport.to_tsv}"
+    #     end
+    #     airport.receive!(raw_airport.airport_attrs)
+    #   end
+    #   AIRPORTS
+    # end
+
+    def self.load(dirname)
+      load_csv(File.join(dirname, 'wikipedia_icao.tsv')) do |id_mapping|
+        [:icao, :iata, :faa ].each do |attr|
+          val = id_mapping.read_attribute(attr) or next
+          next if (val == '.') || (val == '_')
+          if that = ID_MAPPINGS[attr][val]
+            lint = that.disagreements(id_mapping)
+            puts [attr, val, "%-25s" % lint.inspect, id_mapping, that, "%-60s" % id_mapping.name, "%-25s" % that.name].join("\t") if lint.present?
+          else
+            ID_MAPPINGS[attr][val] = id_mapping
+          end
+        end
+        # [:icao, :iata, :faa ].each do |attr|
+        #   val = id_mapping.read_attribute(attr)
+        #   ID_MAPPINGS[attr][val] = id_mapping
+        # end
+      end
+      load_csv(File.join(dirname, 'wikipedia_iata.tsv')) do |id_mapping|
+        # if not ID_MAPPINGS[:icao].has_key?(id_mapping.icao)
+        #   puts [:badicao, "%-25s" % "", id_mapping, " "*24, "%-60s" % id_mapping.name].join("\t")
+        # end
+        [:icao, :iata, :faa ].each do |attr|
+          val = id_mapping.read_attribute(attr) or next
+          next if (val == '.') || (val == '_')
+          if that = ID_MAPPINGS[attr][val]
+            lint = that.disagreements(id_mapping)
+            puts [attr, val, "%-25s" % lint.inspect, id_mapping, that, "%-60s" % id_mapping.name, "%-25s" % that.name].join("\t") if lint.present?
+          else
+            ID_MAPPINGS[attr][val] = id_mapping
+          end
+        end
+      end
+
+      # def adopt_field(that, attr)
+      #   this_val = self.read_attribute(attr)
+      #   that_val = that.read_attribute(attr)
+      #   if name =~ /Bogus|Austin/i
+      #     puts [attr, this_val, that_val, attribute_set?(attr), that.attribute_set?(attr), to_tsv, that.to_tsv].join("\t")
+      #   end
+      #   if    this_val && that_val
+      #     if (this_val != that_val) then warn [attr, this_val, that_val, name].join("\t") ; end
+      #   elsif that_val
+      #     write_attribute(that_val)
+      #   end
+      # end
+
+      def to_s
+        attributes.values[0..2].join("\t")
+      end
+
+      def disagreements(that)
+        errors = {}
+        [:icao, :iata, :faa ].each do |attr|
+          this_val = self.read_attribute(attr) or next
+          that_val = that.read_attribute(attr) or next
+          next if that_val == '.' || that_val == '_'
+          errors[attr] = [this_val, that_val] if this_val != that_val
+        end
+        errors
+      end
+
+    def self.dump_ids(ids)
+      "%s\t%s\t%s" % [icao, iata, faa]
+    end
+    def self.dump_mapping
+      [:icao, :iata, :faa].map do |attr|
+        "%-50s" % ID_MAP[attr].to_a.sort.map{|id, val| "#{id}:#{val.icao||'    '}|#{val.iata||'   '}|#{val.faa||'   '}"}.join(";")
+      end
+    end
+
+    def self.dump_info(kind, ids, reconciler, existing, *args)
+      ex_str = [existing.map{|el| dump_ids(el.ids) }, "\t\t","\t\t","\t\t"].flatten[0..2]
+      puts [kind, dump_ids(ids), dump_ids(reconciler.ids), ex_str, *args, dump_mapping.join("//") ].flatten.join("\t| ")
+    end
+end