wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/airline_flights/airport_ok_chars.rb

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+
+OK_CHARS_RE      = /[^a-zA-Z0-9\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÖØÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýāăčėęěğİıŁłńōőřŞşŠšţťūźŽžơț]/

data/examples/munging/airline_flights/flight.rb

@@ -0,0 +1,156 @@
+# Raw data:
+#   Year,Month,DayofMonth,DayOfWeek,DepTime,CRSDepTime,ArrTime,CRSArrTime,UniqueCarrier,FlightNum,TailNum,ActualElapsedTime,CRSElapsedTime,AirTime,ArrDelay,DepDelay,Origin,Dest,Distance,TaxiIn,TaxiOut,Can
+#   2007,1,1,1,1232,1225,1341,1340,WN,2891,N351,69,75,54,1,7,SMF,ONT,389,4,11,0,,0,0,0,0,0,0
+
+class RawAirlineFlight
+  include Gorillib::Model
+
+  field :date_year,           Integer,     position:  1, doc: "Year (1987-2008)"
+  field :date_month,          Integer,     position:  2, doc: "Month (1-12)"
+  field :date_day,            Integer,     position:  3, doc: "Day of month (1-31)"
+  field :day_of_week,         Integer,     position:  4, doc: "Day of week -- 1 (Monday) - 7 (Sunday)"
+  #
+  field :act_dep_tod,         String,      position:  5, doc: "time of day for actual departure (local, hhmm)", blankish: [nil, '', 'NA']
+  field :crs_dep_tod,         String,      position:  6, doc: "time of day for scheduled departure (local, hhmm)"
+  field :act_arr_tod,         String,      position:  7, doc: "time of day for actual arrival (local, hhmm). Not adjusted for wrap-around.",   blankish: [nil, '', 'NA']
+  field :crs_arr_tod,         String,      position:  8, doc: "time of day for scheduled arrival (local, hhmm). Not adjusted for wrap-around."
+  #
+  field :unique_carrier,      String,     position:  9, doc: "unique carrier code",                 validates: { length: { in: 0..5 } }
+  field :flight_num,          Integer,     position: 10, doc: "flight number"
+  field :tail_num,            String,      position: 11, doc: "plane tail number",                   validates: { length: { in: 0..8 } }
+  #
+  field :act_duration,        Integer,     position: 12, doc: "actual flight time, in minutes",      blankish: [nil, '', 'NA']
+  field :crs_duration,        Integer,     position: 13, doc: "CRS flight time, in minutes"
+  field :air_duration,        Integer,     position: 14, doc: "Air time, in minutes",                blankish: [nil, '', 'NA']
+  field :arr_delay,           Integer,     position: 15, doc: "arrival delay, in minutes",           blankish: [nil, '', 'NA']
+  field :dep_delay,           Integer,     position: 16, doc: "departure delay, in minutes",         blankish: [nil, '', 'NA']
+  field :from_airport,        String,      position: 17, doc: "Origin IATA airport code",            validates: { length: { in: 0..3 } }
+  field :into_airport,        String,      position: 18, doc: "Destination IATA airport code",       validates: { length: { in: 0..3 } }
+  field :distance_mi,         Integer,     position: 19, doc: "Flight distance, in miles"
+  field :taxi_in_duration,    Integer,     position: 20, doc: "taxi in time, in minutes",            blankish: [nil, '', 'NA']
+  field :taxi_out_duration,   Integer,     position: 21, doc: "taxi out time in minutes",            blankish: [nil, '', 'NA']
+  #
+  field :is_cancelled,        :boolean_10, position: 22, doc: "was the flight cancelled?"
+  field :cancellation_code,   String,      position: 23, doc: "Reason for cancellation (A = carrier, B = weather, C = NAS, D = security, Z = no cancellation)"
+  field :is_diverted,         :boolean_10, position: 24, doc: "Was the plane diverted?"
+  field :carrier_delay,       Integer,     position: 25, doc: "in minutes"
+  field :weather_delay,       Integer,     position: 26, doc: "in minutes"
+  field :nas_delay,           Integer,     position: 27, doc: "in minutes"
+  field :security_delay,      Integer,     position: 28, doc: "in minutes"
+  field :late_aircraft_delay, Integer,     position: 29, doc: "in minutes"
+
+  def flight_date
+    Time.new(date_year, date_month, date_day)
+  end
+
+  # uses the year / month / day, along with an "hhmm" string, to
+  def inttime_from_hhmm(val, fencepost=nil)
+    hour, minutes = [val.to_i / 100, val.to_i % 100]
+    res = Time.utc(date_year, date_month, date_day, hour, minutes)
+    # if before fencepost, we wrapped around in time
+    res += (24 * 60 * 60) if fencepost && (res.to_i < fencepost)
+    res.to_i
+  end
+
+  def act_dep_itime ; @act_dep_itime = inttime_from_hhmm(act_dep_tod)                if act_dep_tod ; end
+  def crs_dep_itime ; @crs_dep_itime = inttime_from_hhmm(crs_dep_tod)                               ; end
+  def act_arr_itime ; @act_arr_itime = inttime_from_hhmm(act_arr_tod, act_dep_itime) if act_arr_tod ; end
+  def crs_arr_itime ; @crs_arr_itime = inttime_from_hhmm(crs_arr_tod, crs_dep_itime)                ; end
+
+  def receive_tail_num(val) ; val = nil if val.to_s == "0" ; super(val) ; end
+  def arr_delay(val) val = nil if val.to_s == 0 ; super(val) ; end
+
+  def receive_cancellation_code(val) ; if val == "" then super("Z") else super(val) ; end ; end
+
+  def to_airline_flight
+    attrs = self.attributes.reject{|attr,val| [:year, :month, :day, :distance_mi].include?(attr) }
+    attrs[:flight_datestr] = flight_date.strftime("%Y%m%d")
+    attrs[:distance_km]    = (distance_mi * 1.609_344).to_i
+
+    attrs[:act_dep_tod] = "%04d" % act_dep_tod.to_i if act_dep_tod
+    attrs[:crs_dep_tod] = "%04d" % crs_dep_tod.to_i if crs_dep_tod
+    attrs[:act_arr_tod] = "%04d" % act_arr_tod.to_i if act_arr_tod
+    attrs[:crs_arr_tod] = "%04d" % crs_arr_tod.to_i if crs_arr_tod
+
+    attrs[:act_dep_itime] = act_dep_itime
+    attrs[:crs_dep_itime] = crs_dep_itime
+    attrs[:act_arr_itime] = act_arr_itime
+    attrs[:crs_arr_itime] = crs_arr_itime
+
+    AirlineFlight.receive(attrs)
+  end
+end
+
+class AirlineFlight
+  include Gorillib::Model
+
+  # Identifier
+  field :flight_datestr,      String,      position:  0, doc: "Date, YYYYMMDD. Use flight_date method if you want a date"
+  field :unique_carrier,      String,      position:  1, doc: "Unique Carrier Code. When the same code has been used by multiple carriers, a numeric suffix is used for earlier users, for example, PA, PA(1), PA(2).",           validates: { length: { in: 0..5 } }
+  field :flight_num,          Integer,     position:  2, doc: "flight number"
+  # Flight
+  field :from_airport,        String,      position:  3, doc: "Origin IATA airport code",      validates: { length: { in: 0..3 } }
+  field :into_airport,        String,      position:  4, doc: "Destination IATA airport code", validates: { length: { in: 0..3 } }
+  field :tail_num,            String,      position:  5, doc: "Plane tail number",             validates: { length: { in: 0..8 } }
+  field :distance_km,         Integer,     position:  6, doc: "Flight distance, in kilometers"
+  field :day_of_week,         Integer,     position:  7, doc: "Day of week -- 1 (Monday) - 7 (Sunday)"
+  # Departure and Arrival Absolute Time
+  field :crs_dep_itime,       IntTime,     position:  8, doc: "scheduled departure time (utc epoch seconds)"
+  field :crs_arr_itime,       IntTime,     position:  9, doc: "scheduled arrival time (utc epoch seconds)"
+  field :act_dep_itime,       IntTime,     position: 10, doc: "actual departure time (utc epoch seconds)"
+  field :act_arr_itime,       IntTime,     position: 11, doc: "actual arrival time (utc epoch seconds)"
+  # Departure and Arrival Local Time of Day
+  field :crs_dep_tod,         String,     position:  12, doc: "time of day for scheduled departure (local, hhmm)"
+  field :crs_arr_tod,         String,     position:  13, doc: "time of day for scheduled arrival (local, hhmm). Not adjusted for wrap-around."
+  field :act_dep_tod,         String,     position:  14, doc: "time of day for actual departure (local, hhmm)"
+  field :act_arr_tod,         String,     position:  15, doc: "time of day for actual arrival (local, hhmm). Not adjusted for wrap-around."
+  # Duration
+  field :crs_duration,        Integer,     position: 16, doc: "CRS flight time, in minutes"
+  field :act_duration,        Integer,     position: 17, doc: "Actual flight time, in minutes"
+  field :air_duration,        Integer,     position: 18, doc: "Air time, in minutes"
+  field :taxi_in_duration,    Integer,     position: 19, doc: "taxi in time, in minutes"
+  field :taxi_out_duration,   Integer,     position: 20, doc: "taxi out time in minutes"
+  # Delay
+  field :is_diverted,         :boolean_10, position: 21, doc: "Was the plane diverted? The actual_duration column remains NULL for all diverted flights."
+  field :is_cancelled,        :boolean_10, position: 22, doc: "was the flight cancelled?"
+  field :cancellation_code,   String,      position: 23, doc: "Reason for cancellation (A = carrier, B = weather, C = NAS, D = security, Z = no cancellation)"
+  field :dep_delay,           Integer,     position: 24, doc: "Difference in minutes between scheduled and actual departure time. Early departures show negative numbers. "
+  field :arr_delay,           Integer,     position: 25, doc: "Difference in minutes between scheduled and actual arrival time. Early arrivals show negative numbers."
+  field :carrier_delay,       Integer,     position: 26, doc: "Carrier delay, in minutes"
+  field :weather_delay,       Integer,     position: 27, doc: "Weather delay, in minutes"
+  field :nas_delay,           Integer,     position: 28, doc: "National Air System delay, in minutes"
+  field :security_delay,      Integer,     position: 29, doc: "Security delay, in minutes"
+  field :late_aircraft_delay, Integer,     position: 30, doc: "Late Aircraft delay, in minutes"
+
+  def to_tsv
+    attrs = attributes
+    attrs[:is_cancelled] = is_cancelled ? 1 : 0
+    attrs[:is_diverted]  = is_diverted  ? 1 : 0
+    attrs[:act_dep_itime] ||= '         '
+    attrs[:act_arr_itime] ||= '         '
+
+    # FIXME
+    attrs[:act_duration]  = ((crs_arr_itime - crs_dep_itime) / 60.0).to_i
+    attrs[:air_duration]  = attrs[:act_duration] - attrs[:crs_duration]
+    attrs.each{|key, val| attrs[key] = val.to_s[-7..-1] if val.to_s.length > 7 } # FIXME: for testing
+
+    attrs.values.join("\t")
+  end
+
+  def flight_date
+    @flight_date ||= Gorillib::Factory::DateFactory.receive(flight_datestr)
+  end
+
+  # checks that the record is sane
+  def lint
+    {
+      act_duration:       (!act_arr_itime) || (act_arr_itime - act_dep_itime == act_duration * 60),
+      crs_duration:       (!crs_arr_itime) || (crs_arr_itime - crs_dep_itime == crs_duration * 60),
+      cancelled_has_code: (is_cancelled == (cancellation_code != "Z")),
+      cancellation_code:  (%w[A B C D Z].include?(cancellation_code)),
+      act_duration:       (!act_duration) || (act_duration == (air_duration + taxi_in_duration + taxi_out_duration)),
+      dep_delay:          (!act_dep_itime) || (dep_delay == (act_dep_itime - crs_dep_itime)/60.0),
+      arr_delay:          (!act_arr_itime) || (arr_delay == (act_arr_itime - crs_arr_itime)/60.0),
+    }
+  end
+end

data/examples/munging/airline_flights/models.rb

@@ -0,0 +1,4 @@
+require_relative './airline'
+require_relative './airport'
+require_relative './route'
+require_relative './flight'

data/examples/munging/airline_flights/parse.rb

@@ -0,0 +1,26 @@
+
+# see alsospec/examples/munging/airline_flights_spec.rb
+
+      puts described_class.field_names.map{|fn| fn[0..6] }.join("\t")
+      raw_airports = RawDataexpoAirport.load_csv(de_airports_filename)
+      raw_airports.each do |airport|
+        puts airport.to_tsv
+      end
+
+      puts described_class.field_names.join("\t") # .map{|fn| fn[0..6] }.join("\t")
+      raw_airports = described_class.load_csv(raw_airports_filename)
+      raw_airports.each do |airport|
+        # puts airport.to_tsv
+        linted = airport.lint
+        puts [airport.iata, airport.icao, linted.inspect, airport.to_tsv, ].join("\t") if linted.present?
+      end
+
+      Airport.load(raw_airports_filename, de_airports_filename)
+      Airport::AIRPORTS.each{|id,airport|
+        #puts airport.to_tsv
+        linted = airport.lint
+        warn [airport.iata, airport.icao, airport.de_iata, "%-25s" % airport.name, linted.inspect].join("\t") if linted.present?
+      }
+
+
+# Model.from_tuple(...)

data/examples/munging/airline_flights/reconcile_airports.rb

@@ -0,0 +1,142 @@
+require_relative './models'
+require 'gorillib/model/reconcilable'
+
+class Airport
+  include Gorillib::Model::Reconcilable
+  attr_accessor :_origin # source of the record
+
+  def conflicting_attribute!(attr, this_val, that_val)
+    case attr
+    when :name, :city, :airport_ofid then return :pass
+    when :latitude, :longitude       then return true if (this_val - that_val).abs < 3
+    when :altitude                   then return true if (this_val - that_val).abs < 5
+    end
+    super
+  end
+
+  def ids
+    [:icao, :iata, :faa].hashify{|attr| public_send(attr) }.compact
+  end
+end
+
+#
+# Loads the Airport identifier tables scraped from Wikipedia
+#
+class RawAirportIdentifier < Airport
+  include RawAirport
+  include Gorillib::Model::LoadFromTsv
+
+  def self.from_tuple(icao, iata, faa, name, city=nil, *_)
+    self.new({icao: icao, iata: iata, faa: faa, name: name, city: city}.compact_blank)
+  end
+
+  def self.load_airports(filename, &block)
+    load_tsv(filename, num_fields: 4..6, &block)
+  end
+end
+
+class Airport
+  #
+  # Reconciler for Airports
+  #
+  # For each airport in turn across openflights, dataexpo and the two scraped
+  # identifier sets,
+  # 
+  #
+  class IdReconciler
+    include Gorillib::Model
+    include Gorillib::Model::LoadFromCsv
+    include Gorillib::Model::Reconcilable
+    self.csv_options = { col_sep: "\t", num_fields: 3..6 }
+
+    # Map the reconcilers to each ID they have anything to say about
+    ID_MAP = { icao: {}, iata: {}, faa: {} }
+    
+    field :opinions, Array, default: Array.new, doc: "every record having an id in common with the other records in this field"
+
+    def ids
+      opinions.flat_map{|op| op.ids.to_a }.uniq.compact
+    end
+
+    def self.load_all
+      Log.info "Loading all Airports and reconciling"
+      @airports = Array.new
+      RawDataexpoAirport  .load_airports(:dataexpo_raw_airports   ){|airport| register(:dataexpo, airport) }
+      RawOpenflightAirport.load_airports(:openflights_raw_airports){|airport| register(:openflights, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_icao          ){|airport| register(:wp_icao, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_iata          ){|airport| register(:wp_iata, airport) }
+      RawAirportIdentifier.load_airports(:wikipedia_us_abroad     ){|airport| register(:wp_us_abroad, airport) }
+
+      recs = ID_MAP.map{|attr, hsh| hsh.sort.map(&:last) }.flatten.uniq
+      recs.each do |rec|
+        consensus = rec.reconcile
+        # lint = consensus.lint
+        # puts "%-79s\t%s" % [lint, consensus.to_s[0..100]] if lint.present?
+        @airports << consensus
+      end
+    end
+
+    def self.airports
+      @airports
+    end
+
+    def self.exemplars
+      Airport::EXEMPLARS.map do |iata|
+        ID_MAP[:iata][iata].reconcile
+      end
+    end
+
+    def reconcile
+      consensus = Airport.new
+      clean = opinions.all?{|op| consensus.adopt(op) }
+      # puts "\t#{consensus.inspect}"
+      puts "confl\t#{self.inspect}" if not clean
+      consensus
+    end
+
+    def adopt_opinions(vals, _)
+      self.opinions = vals + self.opinions
+      self.opinions.uniq!
+    end
+
+    # * find all existing reconcilers that share an ID with that record
+    # * unify them into one reconciler
+    # * store it back under all the IDs
+    #
+    # Suppose our dataset has 3 identifiers, which look like
+    #
+    #     a    S
+    #          S    88
+    #     a    Z
+    #     b
+    #          Q
+    #     b    Q    77
+    #
+    # We will wind up with these two reconcilers:
+    #
+    #     <a S 88 opinions: [a,S, ],[S, ,88],[a,Z,  ]>
+    #     <b Q 77 opinions: [b, , ],[ ,Q,  ],[b,Q,77]>
+    #
+    def self.register(origin, obj)
+      obj._origin = origin
+      # get the existing reconcilers
+      existing   = obj.ids.map{|attr, id| ID_MAP[attr][id] }.compact.uniq
+      # push the new object in, and pull the most senior one out
+      existing.unshift(self.new(opinions: [obj]))
+      reconciler = existing.shift
+      # unite them into the reconciler
+      existing.each{|that| reconciler.adopt(that) }
+      # save the reconciler under each of the ids.
+      reconciler.ids.each{|attr, id| ID_MAP[attr][id] = reconciler }
+    end
+
+    def inspect
+      str = "#<#{self.class.name} #{ids}"
+      opinions.each do |op|
+        str << "\n\t  #{op._origin}\t#{op}"
+      end
+      str << ">"
+    end
+  end
+
+end

data/examples/munging/airline_flights/route.rb

@@ -0,0 +1,35 @@
+
+
+# As of January 2012, the OpenFlights/Airline Route Mapper Route Database
+# contains 59036 routes between 3209 airports on 531 airlines [spanning the
+# globe](http://openflights.org/demo/openflights-routedb-2048.png).  If you
+# enjoy this data, please consider [visiting their page and
+# donating](http://openflights.org/data.html)
+#
+# > Notes: Routes are directional: if an airline operates services from A to B
+# > and from B to A, both A-B and B-A are listed separately. Routes where one
+# > carrier operates both its own and codeshare flights are listed only once.
+#
+# Sample entries
+#
+#     BA,1355,SIN,3316,LHR,507,,0,744 777
+#     BA,1355,SIN,3316,MEL,3339,Y,0,744
+#     TOM,5013,ACE,1055,BFS,465,,0,320
+#
+class RawOpenflightRoute
+  include Gorillib::Model
+
+  field :iataicao,              String,   doc: "2-letter (IATA) or 3-letter (ICAO) code of the airline."
+  field :airline_ofid,          Integer,  doc: "Unique OpenFlights identifier for airline (see Airline)."
+  field :from_airport_iataicao, String,   doc: "3-letter (IATA) or 4-letter (ICAO) code of the source airport."
+  field :from_airport_ofid,     Integer,  doc: "Unique OpenFlights identifier for source airport (see Airport)"
+  field :into_airport_iataicao, String,   doc: "3-letter (IATA) or 4-letter (ICAO) code of the destination airport."
+  field :into_airport_ofid,     Integer,  doc: "Unique OpenFlights identifier for destination airport (see Airport)"
+  field :codeshare,             :boolean, doc: "true if this flight is a codeshare (that is, not operated by Airline, but another carrier); empty otherwise."
+  field :stops,                 Integer,  doc: "Number of stops on this flight, or '0' for direct"
+  field :equipment_list,        String,   doc: "3-letter codes for plane type(s) generally used on this flight, separated by spaces"
+
+  def receive_codeshare(val)
+    super(case val when "Y" then true when "N" then false else val ; end)
+  end
+end

data/examples/munging/airline_flights/tasks.rake

@@ -0,0 +1,83 @@
+require_relative('../../rake_helper')
+require_relative('./models')
+
+Pathname.register_paths(
+  af_data:                  [:data, 'airline_flights'],
+  af_work:                  [:work, 'airline_flights'],
+  af_code:                  File.dirname(__FILE__),
+  #
+  openflights_raw_airports: [:af_data, "openflights_airports-raw#{Settings[:mini_slug]}.csv"   ],
+  openflights_raw_airlines: [:af_data, "openflights_airlines-raw.csv"   ],
+  dataexpo_raw_airports:    [:af_data, "dataexpo_airports-raw#{Settings[:mini_slug]}.csv"      ],
+  wikipedia_icao:           [:af_data, "wikipedia_icao.tsv" ],
+  wikipedia_iata:           [:af_data, "wikipedia_iata.tsv" ],
+  wikipedia_us_abroad:      [:af_data, "wikipedia_us_abroad.tsv" ],
+  #
+  openflights_airports:     [:af_work, "openflights_airports-parsed#{Settings[:mini_slug]}.tsv"],
+  openflights_airlines:     [:af_work, "openflights_airlines-parsed#{Settings[:mini_slug]}.tsv"],
+  dataexpo_airports:        [:af_work, "dataexpo_airports-parsed#{Settings[:mini_slug]}.tsv"   ],
+  airport_identifiers:      [:af_work, "airport_identifiers.tsv"   ],
+  airport_identifiers_mini: [:af_work, "airport_identifiers-sample.tsv"   ],
+  # helpers
+  country_name_lookup:       [:work, 'geo', "country_name_lookup.tsv"],
+  )
+
+chain :airline_flights do
+  code_files = FileList[Pathname.of(:af_code, '*.rb').to_s]
+  chain(:parse) do
+
+    # desc 'parse the dataexpo airports'
+    # create_file(:dataexpo_airports, after: code_files) do |dest|
+    #   RawDataexpoAirport.load_airports(:dataexpo_raw_airports) do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+
+    desc 'parse the openflights airports'
+    create_file(:openflights_airports, after: [code_files, :force]) do |dest|
+      require_relative('../geo/geo_models')
+      Geo::CountryNameLookup.load
+      RawOpenflightAirport.load_airports(:openflights_raw_airports) do |airport|
+        dest << airport.to_tsv << "\n"
+        # puts airport.country
+      end
+    end
+
+    # task :reconcile_airports => [:dataexpo_airports, :openflights_airports] do
+    #   require_relative 'reconcile_airports'
+    #   Airport::IdReconciler.load_all
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.airports.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'run the identifier reconciler'
+    # create_file(:airport_identifiers_mini, after: code_files, invoke: 'airline_flights:parse:reconcile_airports') do |dest|
+    #   Airport::IdReconciler.exemplars.each do |airport|
+    #     dest << airport.to_tsv << "\n"
+    #   end
+    # end
+    #
+    # desc 'parse the openflights airlines'
+    # create_file(:openflights_airlines, after: code_files) do |dest|
+    #   RawOpenflightAirline.load_airlines(:openflights_raw_airlines) do |airline|
+    #     dest << airline.to_tsv << "\n"
+    #     puts airline.to_tsv
+    #   end
+    # end
+
+  end
+end
+
+task :default => [
+  'airline_flights',
+  # 'airline_flights:parse:dataexpo_airports',
+  # 'airline_flights:parse:openflights_airports',
+  # 'airline_flights:parse:airport_identifiers',
+  # 'airline_flights:parse:airport_identifiers_mini',
+  # 'airline_flights:parse:openflights_airlines',
+]