wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/dataflow/apache_log_line.rb

@@ -1,100 +0,0 @@
-#
-# Parses logs in either the [Apache Common Log Format](http://en.wikipedia.org/wiki/Common_Log_Format)
-# or [Apache Combined Log Format](http://httpd.apache.org/docs/2.2/logs.html#combined)
-#
-# Common:   `%h %l %u %t "%r" %>s %b`
-# Combined: `%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"`
-#
-class ApacheLogLine
-  include Gorillib::Model
-
-  field  :client,        Hostname
-  field  :rfc_1413,      String
-  field  :userid,        String
-  field  :log_timestamp, Time
-  field  :http_method,   String
-  field  :rsrc,          String
-  field  :protocol,      String
-  field  :response_code, Integer
-  field  :size,          Integer
-  # field  :referer,       String
-  # field  :user_agent,    String
-
-  def page_type
-    case
-    when rsrc =~ /\.(css|js)$/                  then :asset
-    when rsrc =~ /\.(png|gif|ico)$/             then :image
-    when rsrc =~ /\.(pl|s?html?|asp|jsp|cgi)$/  then :page
-    else                                             :other
-    end
-  end
-
-  #
-  # Regular expression to parse an apache log line.
-  #
-  # local - - [24/Oct/1994:13:43:13 -0600] "GET index.html HTTP/1.0" 200 3185
-  # 83.240.154.3 - - [07/Jun/2008:20:37:11 +0000] "GET /faq HTTP/1.1" 200 569 "http://infochimps.org/search?query=CAC" "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
-  # whidbey.whidbey.com    - - [04/Sep/1995:00:30:18 -0400] "GET /pub/sshay/images/btthumb.jpg"      200 4624
-  # jgbustam-ppp.clark.net - - [04/Sep/1995:00:00:28 -0400] "GET /pub/jgbustam/famosos/alpha.html HTTP/1.0" 304 -
-  #
-  COMMON_LOG_RE = Regexp.compile(%r{\A
-           (\S+)                        # client              83.240.154.3
-         \s(\S+)                        # rfc_1413            -
-         \s(\S+)                        # userid              -
-      \s\[([\w\:\+\-\ \/]+)\]           # date part           [07/Jun/2008:20:37:11 +0000]
-    \s\"(?:(\S+)                        # http_method         "GET
-         \s(.+?)                        # rsrc                /faq
-      (?:\s(HTTP/\d+\.\d+))?|-)\"       # protocol            HTTP/1.1"
-         \s(\d+|-)                      # response_code       200
-         \s(\d+|-)                      # size                569
-      \z
-      }x)
-
-  COMBINED_LOG_RE = Regexp.compile(%r{\A
-           (\S+)                        # client              83.240.154.3
-         \s(\S+)                        # rfc_1413            -
-         \s(\S+)                        # userid              -
-      \s\[([\w\:\+\-\ \/]+)\]           # date part           [07/Jun/2008:20:37:11 +0000]
-    \s\"(?:(\S+)                        # http_method         "GET
-         \s([^\"]+?)                    # rsrc                /faq
-      (?:\s(HTTP/\d+\.\d+))?|-)\"       # protocol            HTTP/1.1"
-         \s(\d+|-)                      # response_code       200
-         \s(\d+|-)                      # size                569
-    (?:\s\"([^\"]*)\")                  # referer             "http://infochimps.org/search?query=CAC"
-    (?:\s\"([^\"]*)\")                  # ua                  "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr; rv:1.9.0.16) Gecko/2009120208 Firefox/3.0.16"
-      \z
-      }x)
-
-  # LOG_RE = Regexp.compile(%r{\A(\S+)\s})
-
-  MONTHS = { 'Jan' => 1, 'Feb' => 2, 'Mar' => 3, 'Apr' => 4, 'May' => 5, 'Jun' => 6, 'Jul' => 7, 'Aug' => 8, 'Sep' => 9, 'Oct' => 10, 'Nov' => 11, 'Dec' => 12, }
-
-  # Converts a time like `10/Apr/2007:10:58:27 +0300` to something parseable
-  def receive_log_timestamp(raw_ts)
-    return super(nil) if raw_ts.nil?
-    match = %r{(\d+)/(\w+)/(\d+):(\d+):(\d+):(\d+)\s([\+\-\w]+)}.match(raw_ts)
-    unless match then warn "Can't parse date #{raw_ts}" ; return super(nil) ; end
-    #
-    day, month_name, year, hour, min, sec, tz = match.captures
-    month = MONTHS[month_name]
-    tz.insert(3, ':')  # -0600 to -06:00
-    #
-    # super "#{year}-#{month}-#{day} #{hour}:#{min}:#{sec} #{tz}"
-    super Time.new(year.to_i, month, day.to_i, hour.to_i, min.to_i, sec.to_i, tz)
-  end
-
-  # @returns the log_timestamp in the common log format
-  def unparsed_log_timestamp
-    return if log_timestamp.blank?
-    log_timestamp.strftime("%d/%b/%Y:%H:%M:%S %z")
-  end
-
-  # Use the regex to break line into fields
-  # Emit each record as flat line
-  def self.make(line)
-    m = COMMON_LOG_RE.match(line) or return
-    from_tuple *m.captures
-  rescue ArgumentError => err
-    raise unless err.message =~ /invalid byte sequence in UTF-8/
-  end
-end

data/examples/jabberwocky.txt

@@ -1,36 +0,0 @@
-"Jabberwocky"
-
-'Twas brillig, and the slithy toves
-Did gyre and gimble in the wabe;
-All mimsy were the borogoves,
-And the mome raths outgrabe.
-
-"Beware the Jabberwock, my son!
-The jaws that bite, the claws that catch!
-Beware the Jubjub bird, and shun
-The frumious Bandersnatch!"
-
-He took his vorpal sword in hand:
-Long time the manxome foe he sought—
-So rested he by the Tumtum tree,
-And stood awhile in thought.
-
-And as in uffish thought he stood,
-The Jabberwock, with eyes of flame,
-Came whiffling through the tulgey wood,
-And burbled as it came!
-
-One, two! One, two! and through and through
-The vorpal blade went snicker-snack!
-He left it dead, and with its head
-He went galumphing back.
-
-"And hast thou slain the Jabberwock?
-Come to my arms, my beamish boy!
-O frabjous day! Callooh! Callay!"
-He chortled in his joy.
-
-'Twas brillig, and the slithy toves
-Did gyre and gimble in the wabe;
-All mimsy were the borogoves,
-And the mome raths outgrabe.

data/examples/munging/Gemfile

@@ -1,8 +0,0 @@
-source 'http://rubygems.org'
-  
-gem 'gorillib', :path => '/home/dlaw/dev/gorillib'
-gem 'ruby-progressbar'
-gem 'crack'
-gem 'rake'
-gem 'wukong', :path => '/home/dlaw/dev/wukong_og'
-gem 'json'

data/examples/munging/airline_flights/airline.rb

@@ -1,57 +0,0 @@
-class Airline
-  include Gorillib::Model
-  field :icao_id,      String,   doc: "3-letter ICAO code, if available", identifier: true, length: 2
-  field :iata_id,      String,   doc: "2-letter IATA code, if available", identifier: true, length: 2
-  field :airline_ofid, Integer,  doc: "Unique OpenFlights identifier for this airline.", identifier: true
-  field :active,       :boolean, doc: 'true if the airline is or has until recently been operational, false if it is defunct. (This is only a rough indication and should not be taken as 100% accurate)'
-  field :country,      String,   doc: "Country or territory where airline is incorporated"
-  field :name,         String,   doc: "Airline name."
-  field :callsign,     String,   doc: "Airline callsign", identifier: true
-  field :alias,        String,   doc: "Alias of the airline. For example, 'All Nippon Airways' is commonly known as 'ANA'"
-end
-
-#
-# As of January 2012, the OpenFlights Airlines Database contains 5888
-# airlines. If you enjoy this data, please consider [visiting their page and
-# donating](http://openflights.org/data.html)
-#
-# > Notes: Airlines with null codes/callsigns/countries generally represent
-# > user-added airlines. Since the data is intended primarily for current
-# > flights, defunct IATA codes are generally not included. For example,
-# > "Sabena" is not listed with a SN IATA code, since "SN" is presently used by
-# > its successor Brussels Airlines.
-#
-# Sample entries
-#
-#     324,"All Nippon Airways","ANA All Nippon Airways","NH","ANA","ALL NIPPON","Japan","Y"
-#     412,"Aerolineas Argentinas",\N,"AR","ARG","ARGENTINA","Argentina","Y"
-#     413,"Arrowhead Airways",\N,"","ARH","ARROWHEAD","United States","N"
-#
-class RawOpenflightAirline
-  include Gorillib::Model
-  include Gorillib::Model::LoadFromCsv
-  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
-
-  field :airline_ofid, Integer,  blankish: BLANKISH_STRINGS, doc: "Unique OpenFlights identifier for this airline.", identifier: true
-  field :name,         String,   blankish: BLANKISH_STRINGS, doc: "Airline name."
-  field :alias,        String,   blankish: BLANKISH_STRINGS, doc: "Alias of the airline. For example, 'All Nippon Airways' is commonly known as 'ANA'"
-  field :iata_id,      String,   blankish: BLANKISH_STRINGS, doc: "2-letter IATA code, if available", identifier: true, length: 2
-  field :icao_id,      String,   blankish: BLANKISH_STRINGS, doc: "3-letter ICAO code, if available", identifier: true, length: 2
-  field :callsign,     String,   blankish: BLANKISH_STRINGS, doc: "Airline callsign"
-  field :country,      String,   blankish: BLANKISH_STRINGS, doc: "Country or territory where airline is incorporated"
-  field :active,       :boolean, blankish: BLANKISH_STRINGS, doc: 'true if the airline is or has until recently been operational, false if it is defunct. (This is only a rough indication and should not be taken as 100% accurate)'
-
-  def receive_iata_id(val) super if val =~ /\A\w+\z/ ; end
-  def receive_icao_id(val) super if val =~ /\A\w+\z/ ; end
-  def receive_active(val)
-    super(case val.to_s when "Y" then true when "N" then false else val ; end)
-  end
-
-  def to_airline
-    Airline.receive(self.compact_attributes)
-  end
-
-  def self.load_airlines(filename)
-    load_csv(filename){|raw_airline| yield(raw_airline.to_airline) }
-  end
-end

data/examples/munging/airline_flights/airport.rb

@@ -1,211 +0,0 @@
-# -*- coding: utf-8 -*-
-
-### @export "airport_model"
-class Airport
-  include Gorillib::Model
-
-  field :icao,         String, doc: "4-letter ICAO code, or blank if not assigned.", length: 4, identifier: true, :blankish => ["", nil]
-  field :iata,         String, doc: "3-letter IATA code, or blank if not assigned.", length: 3, identifier: true, :blankish => ["", nil]
-  field :faa,          String, doc: "3-letter FAA code, or blank if not assigned.",  length: 3, identifier: true, :blankish => ["", nil]
-  field :utc_offset,   Float,  doc: "Hours offset from UTC. Fractional hours are expressed as decimals, eg. India is 5.5.",      validates: {  inclusion: (-12...12) }
-  field :dst_rule,     String, doc: "Daylight savings time rule. One of E (Europe), A (US/Canada), S (South America), O (Australia), Z (New Zealand), N (None) or U (Unknown). See the readme for more.", validates: {  inclusion: %w[E A S O Z N U] }
-  field :longitude,    Float,  doc: "Decimal degrees, usually to six significant digits. Negative is West,  positive is East.",  validates: {  inclusion: (-180...180) }
-  field :latitude,     Float,  doc: "Decimal degrees, usually to six significant digits. Negative is South, positive is North.", validates: {  inclusion: (-90.0...90.0) }
-  field :altitude,     Float,  doc: "Elevation in meters."
-  field :name,         String, doc: "Name of airport."
-  field :country,      String, doc: "Country or territory where airport is located.", length: 2
-  field :state,        String, doc: "State in which the airport is located",          length: 2
-  field :city,         String, doc: "Main city served by airport. This is the logical city it serves; so, for example SFO gets 'San Francisco', not 'San Bruno'"
-  field :airport_ofid, String, doc: "OpenFlights identifier for this airport.", identifier: true
-end
-### @export "nil"
-class Airport
-  EXEMPLARS = %w[
-    ANC ATL AUS BDL BNA BOI BOS BWI CLE CLT
-    CMH DCA DEN DFW DTW EWR FLL HNL IAD IAH
-    IND JAX JFK LAS LAX LGA MCI MCO MDW MIA
-    MSP MSY OAK ORD PDX PHL PHX PIT PVD RDU
-    SAN SEA SFO SJC SJU SLC SMF STL TPA YYZ
-  ]
-
-  def utc_time_for(tm)
-    utc_time  = tm.get_utc + utc_offset
-    utc_time += (60*60) if TimezoneFixup.dst?(tm)
-    utc_time
-  end
-
-  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
-  OK_CHARS_RE      = /[^a-zA-Z0-9\:\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÓÖØÚÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýĀāăĆćČčēėęěğīİıŁłńņňŌōőřŞşŠšţťūůųźŽžơț]/
-
-  def lint
-    errors = {}
-    errors["ICAO is wrong length"] = icao if icao.present? && icao.length != 4
-    if (icao && faa && (icao =~ /^K.../))
-      errors["ICAO != K+FAA yet ICAO is a K..."] = [icao, faa] if (icao != "K#{faa}")
-    end
-    # errors["ICAO present for piddlyshit airport"] = icao if icao.present? && ((faa.to_s.length == 4) || (faa.to_s =~ /\d/))
-    errors[:spaces] ||= []
-    errors[:funny]  ||= []
-    attributes.each do |attr, val|
-      next if val.blank?
-      errors["#{attr} looks blankish"] = val if BLANKISH_STRINGS.include?(val)
-      if (val.is_a?(String))
-        errors[:spaces] << [attr, val] if  (val.strip != val)
-        errors[:funny]  << [attr, val]  if val =~ OK_CHARS_RE
-      end
-    end
-    errors.compact_blank
-  end
-
-  def to_s
-    str = "#<Airport "
-    str << [icao, iata, faa,
-      (latitude && "%4.1f" % latitude), (longitude && "%5.1f" % longitude), state, country,
-      "%-30s" % name, country, city].join("\t")
-    str << ">"
-  end
-
-  def faa_controlled?
-    icao =~ /^(?:K|P[ABFGHJKMOPW]|T[IJ]|NS(AS|FQ|TU))/
-  end
-end
-### @export "airport_load"
-class Airport
-  include Gorillib::Model::LoadFromTsv
-  self.tsv_options.merge!(num_fields: 10..20)
-  def self.load_airports(filename)
-    load_tsv(filename){|airport| yield(airport) }
-  end
-
-end
-### @export "nil"
-
-#
-# As of January 2012, the OpenFlights Airports Database contains 6977 airports
-# [spanning the globe](http://openflights.org/demo/openflights-apdb-2048.png).
-# If you enjoy this data, please consider [visiting their page and
-# donating](http://openflights.org/data.html)
-#
-# > Note: Rules for daylight savings time change from year to year and from
-# > country to country. The current data is an approximation for 2009, built on
-# > a country level. Most airports in DST-less regions in countries that
-# > generally observe DST (eg. AL, HI in the USA, NT, QL in Australia, parts of
-# > Canada) are marked incorrectly.
-#
-# Sample entries
-#
-#     507,"Heathrow","London","United Kingdom","LHR","EGLL",51.4775,-0.461389,83,0,"E"
-#     26,"Kugaaruk","Pelly Bay","Canada","YBB","CYBB",68.534444,-89.808056,56,-6,"A"
-#     3127,"Pokhara","Pokhara","Nepal","PKR","VNPK",28.200881,83.982056,2712,5.75,"N"
-#
-
-### @export "raw_openflight_airport"
-
-module RawAirport
-  COUNTRIES        = { 'Puerto Rico' => 'us', 'Canada' => 'ca', 'USA' => 'us', 'United States' => 'us',
-    'Northern Mariana Islands' => 'us', 'N Mariana Islands' => 'us',
-    'Federated States of Micronesia' => 'fm',
-    'Thailand' => 'th', 'Palau' => 'pw',
-    'American Samoa' => 'as', 'Wake Island' => 'us', 'Virgin Islands' => 'vi', 'Guam' => 'gu'
-  }
-  BLANKISH_STRINGS = ["", nil, "NULL", '\\N', "NONE", "NA", "Null", "..."]
-  OK_CHARS_RE      = /[^a-zA-Z0-9\:\ \/\.\,\-\(\)\'ÁÂÄÅÇÉÍÎÑÓÖØÚÜÞàáâãäåæçèéêëìíîïðñóôõöøúüýĀāăĆćČčēėęěğīİıŁłńņňŌōőřŞşŠšţťūůųźŽžơț]/
-
-  def receive_city(val)
-    super.tap{|val| if val then val.strip! ; val.gsub!(/\\+/, '') ; end }
-  end
-
-  def receive_country(val)
-    super(COUNTRIES[val] || val)
-  end
-
-  def receive_name(val)
-    super.tap do |val|
-      if val
-        val.strip!
-        val.gsub!(/\\+/, '')
-        val.gsub!(/\s*\[(military|private)\]/, '')
-        val.gsub!(/\b(Int\'l|International)\b/, 'Intl')
-        val.gsub!(/\b(Intercontinental)\b/,     'Intcntl')
-        val.gsub!(/\b(Airpt)\b/,                'Airport')
-        val.gsub!(/ Airport$/, '')
-      end
-    end
-  end
-end
-
-#
-class RawOpenflightAirport
-  include Gorillib::Model
-  include Gorillib::Model::LoadFromCsv
-  include RawAirport
-  #
-  field :airport_ofid, String, doc: "Unique OpenFlights identifier for this airport."
-  field :name,        String, doc: "Name of airport. May or may not contain the City name."
-  field :city,        String, blankish: BLANKISH_STRINGS, doc: "Main city served by airport. May be spelled differently from Name."
-  field :country,     String, doc: "Country or territory where airport is located."
-  field :iata_faa,    String, blankish: BLANKISH_STRINGS, doc: "3-letter FAA code, for airports located in the USA. For all other airports, 3-letter IATA code, or blank if not assigned."
-  field :icao,        String, blankish: BLANKISH_STRINGS, doc: "4-letter ICAO code; Blank if not assigned."
-  field :latitude,    Float,  doc: "Decimal degrees, usually to six significant digits. Negative is South, positive is North."
-  field :longitude,   Float,  doc: "Decimal degrees, usually to six significant digits. Negative is West,  positive is East."
-  field :altitude_ft, Float,  blankish: ['', nil, 0, '0'], doc: "In feet."
-  field :utc_offset,  Float,  doc: "Hours offset from UTC. Fractional hours are expressed as decimals, eg. India is 5.5."
-  field :dst_rule,    String, doc: "Daylight savings time rule. One of E (Europe), A (US/Canada), S (South America), O (Australia), Z (New Zealand), N (None) or U (Unknown). See the readme for more."
-
-  UNRELIABLE_OPENFLIGHTS_IATA_VALUES = /^(7AK|AGA|AUQ|BDJ|BGW|BME|BPM|BXH|BZY|CAT|CEE|CEJ|CFS|CGU|CIO|CLV|CNN|DEE|DIB|DNM|DUH|DUR|FKI|GES|GSM|HKV|HOJ|HYD|IEO|IFN|IKA|IZA|JCU|JGS|KMW|KNC|LGQ|LUM|MCU|MCY|MDO|MOH|MON|MPH|MVF|NAY|NMA|NOE|NQY|OTU|OUI|PBV|PCA|PCB|PGK|PHO|PIF|PKN|PKY|PMK|PTG|PZO|QAS|QKT|QVY|RCM|RJL|RTG|SBG|SDZ|SFG|SIC|SIQ|SJI|SRI|STP|STU|SWQ|TJQ|TJS|TMC|TYA|UKC|VIY|VQS|VTS|WDH|WKM|WPR|WPU|ZQF)$/
-
-  def id_is_faa?
-    (icao =~ /^(?:K)/) || (icao.blank? && country == 'us')
-  end
-
-  def iata ;  (id_is_faa? ? nil      : iata_faa) unless iata_faa =~ UNRELIABLE_OPENFLIGHTS_IATA_VALUES end
-  def faa  ;  (id_is_faa? ? iata_faa : nil     ) end
-  def altitude
-    altitude_ft && (0.3048 * altitude_ft).round(1)
-  end
-
-  def receive_country(val)
-    country = Geo::CountryNameLookup.for_alt_name(val, nil)
-    p val unless country
-    super(country ? country.country_id : val)
-  end
-
-  def to_airport
-    attrs = self.compact_attributes.except(:altitude_ft)
-    attrs[:altitude] = altitude
-    attrs[:iata]     = iata unless iata.to_s =~ UNRELIABLE_OPENFLIGHTS_IATA_VALUES
-    attrs[:faa]      = faa
-    Airport.receive(attrs)
-  end
-
-  def self.load_airports(filename)
-    load_csv(filename){|raw_airport| yield(raw_airport.to_airport) }
-  end
-end
-
-### @export "raw_dataexpo_airport"
-class RawDataexpoAirport
-  include Gorillib::Model
-  include Gorillib::Model::LoadFromCsv
-  include RawAirport
-  self.csv_options = self.csv_options.merge(pop_headers: true)
-
-  field :faa,          String, doc: "the international airport abbreviation code"
-  field :name,         String, doc: "Airport name"
-  field :city,         String, blankish: ["NA"], doc: "city in which the airport is located"
-  field :state,        String, blankish: ["NA"], doc: "state in which the airport is located"
-  field :country,      String, doc: "country in which airport is located"
-  field :latitude,     Float,  doc: "latitude of the airport"
-  field :longitude,    Float,  doc: "longitude of the airport"
-
-  def to_airport
-    attrs = self.compact_attributes
-    attrs[:icao] = "K#{faa}" if faa =~ /[A-Z]{3}/ && (not ['PR', 'AK', 'CQ', 'HI', 'AS', 'GU', 'VI'].include?(state)) && (country == 'us')
-    Airport.receive(attrs)
-  end
-
-  def self.load_airports(filename)
-    load_csv(filename){|raw_airport| yield(raw_airport.to_airport) }
-  end
-end
-### @export "nil"

data/examples/munging/airline_flights/flight.rb

@@ -1,156 +0,0 @@
-# Raw data:
-#   Year,Month,DayofMonth,DayOfWeek,DepTime,CRSDepTime,ArrTime,CRSArrTime,UniqueCarrier,FlightNum,TailNum,ActualElapsedTime,CRSElapsedTime,AirTime,ArrDelay,DepDelay,Origin,Dest,Distance,TaxiIn,TaxiOut,Can
-#   2007,1,1,1,1232,1225,1341,1340,WN,2891,N351,69,75,54,1,7,SMF,ONT,389,4,11,0,,0,0,0,0,0,0
-
-class RawAirlineFlight
-  include Gorillib::Model
-
-  field :date_year,           Integer,     position:  1, doc: "Year (1987-2008)"
-  field :date_month,          Integer,     position:  2, doc: "Month (1-12)"
-  field :date_day,            Integer,     position:  3, doc: "Day of month (1-31)"
-  field :day_of_week,         Integer,     position:  4, doc: "Day of week -- 1 (Monday) - 7 (Sunday)"
-  #
-  field :act_dep_tod,         String,      position:  5, doc: "time of day for actual departure (local, hhmm)", blankish: [nil, '', 'NA']
-  field :crs_dep_tod,         String,      position:  6, doc: "time of day for scheduled departure (local, hhmm)"
-  field :act_arr_tod,         String,      position:  7, doc: "time of day for actual arrival (local, hhmm). Not adjusted for wrap-around.",   blankish: [nil, '', 'NA']
-  field :crs_arr_tod,         String,      position:  8, doc: "time of day for scheduled arrival (local, hhmm). Not adjusted for wrap-around."
-  #
-  field :unique_carrier,      String,     position:  9, doc: "unique carrier code",                 validates: { length: { in: 0..5 } }
-  field :flight_num,          Integer,     position: 10, doc: "flight number"
-  field :tail_num,            String,      position: 11, doc: "plane tail number",                   validates: { length: { in: 0..8 } }
-  #
-  field :act_duration,        Integer,     position: 12, doc: "actual flight time, in minutes",      blankish: [nil, '', 'NA']
-  field :crs_duration,        Integer,     position: 13, doc: "CRS flight time, in minutes"
-  field :air_duration,        Integer,     position: 14, doc: "Air time, in minutes",                blankish: [nil, '', 'NA']
-  field :arr_delay,           Integer,     position: 15, doc: "arrival delay, in minutes",           blankish: [nil, '', 'NA']
-  field :dep_delay,           Integer,     position: 16, doc: "departure delay, in minutes",         blankish: [nil, '', 'NA']
-  field :from_airport,        String,      position: 17, doc: "Origin IATA airport code",            validates: { length: { in: 0..3 } }
-  field :into_airport,        String,      position: 18, doc: "Destination IATA airport code",       validates: { length: { in: 0..3 } }
-  field :distance_mi,         Integer,     position: 19, doc: "Flight distance, in miles"
-  field :taxi_in_duration,    Integer,     position: 20, doc: "taxi in time, in minutes",            blankish: [nil, '', 'NA']
-  field :taxi_out_duration,   Integer,     position: 21, doc: "taxi out time in minutes",            blankish: [nil, '', 'NA']
-  #
-  field :is_cancelled,        :boolean_10, position: 22, doc: "was the flight cancelled?"
-  field :cancellation_code,   String,      position: 23, doc: "Reason for cancellation (A = carrier, B = weather, C = NAS, D = security, Z = no cancellation)"
-  field :is_diverted,         :boolean_10, position: 24, doc: "Was the plane diverted?"
-  field :carrier_delay,       Integer,     position: 25, doc: "in minutes"
-  field :weather_delay,       Integer,     position: 26, doc: "in minutes"
-  field :nas_delay,           Integer,     position: 27, doc: "in minutes"
-  field :security_delay,      Integer,     position: 28, doc: "in minutes"
-  field :late_aircraft_delay, Integer,     position: 29, doc: "in minutes"
-
-  def flight_date
-    Time.new(date_year, date_month, date_day)
-  end
-
-  # uses the year / month / day, along with an "hhmm" string, to
-  def inttime_from_hhmm(val, fencepost=nil)
-    hour, minutes = [val.to_i / 100, val.to_i % 100]
-    res = Time.utc(date_year, date_month, date_day, hour, minutes)
-    # if before fencepost, we wrapped around in time
-    res += (24 * 60 * 60) if fencepost && (res.to_i < fencepost)
-    res.to_i
-  end
-
-  def act_dep_itime ; @act_dep_itime = inttime_from_hhmm(act_dep_tod)                if act_dep_tod ; end
-  def crs_dep_itime ; @crs_dep_itime = inttime_from_hhmm(crs_dep_tod)                               ; end
-  def act_arr_itime ; @act_arr_itime = inttime_from_hhmm(act_arr_tod, act_dep_itime) if act_arr_tod ; end
-  def crs_arr_itime ; @crs_arr_itime = inttime_from_hhmm(crs_arr_tod, crs_dep_itime)                ; end
-
-  def receive_tail_num(val) ; val = nil if val.to_s == "0" ; super(val) ; end
-  def arr_delay(val) val = nil if val.to_s == 0 ; super(val) ; end
-
-  def receive_cancellation_code(val) ; if val == "" then super("Z") else super(val) ; end ; end
-
-  def to_airline_flight
-    attrs = self.attributes.reject{|attr,val| [:year, :month, :day, :distance_mi].include?(attr) }
-    attrs[:flight_datestr] = flight_date.strftime("%Y%m%d")
-    attrs[:distance_km]    = (distance_mi * 1.609_344).to_i
-
-    attrs[:act_dep_tod] = "%04d" % act_dep_tod.to_i if act_dep_tod
-    attrs[:crs_dep_tod] = "%04d" % crs_dep_tod.to_i if crs_dep_tod
-    attrs[:act_arr_tod] = "%04d" % act_arr_tod.to_i if act_arr_tod
-    attrs[:crs_arr_tod] = "%04d" % crs_arr_tod.to_i if crs_arr_tod
-
-    attrs[:act_dep_itime] = act_dep_itime
-    attrs[:crs_dep_itime] = crs_dep_itime
-    attrs[:act_arr_itime] = act_arr_itime
-    attrs[:crs_arr_itime] = crs_arr_itime
-
-    AirlineFlight.receive(attrs)
-  end
-end
-
-class AirlineFlight
-  include Gorillib::Model
-
-  # Identifier
-  field :flight_datestr,      String,      position:  0, doc: "Date, YYYYMMDD. Use flight_date method if you want a date"
-  field :unique_carrier,      String,      position:  1, doc: "Unique Carrier Code. When the same code has been used by multiple carriers, a numeric suffix is used for earlier users, for example, PA, PA(1), PA(2).",           validates: { length: { in: 0..5 } }
-  field :flight_num,          Integer,     position:  2, doc: "flight number"
-  # Flight
-  field :from_airport,        String,      position:  3, doc: "Origin IATA airport code",      validates: { length: { in: 0..3 } }
-  field :into_airport,        String,      position:  4, doc: "Destination IATA airport code", validates: { length: { in: 0..3 } }
-  field :tail_num,            String,      position:  5, doc: "Plane tail number",             validates: { length: { in: 0..8 } }
-  field :distance_km,         Integer,     position:  6, doc: "Flight distance, in kilometers"
-  field :day_of_week,         Integer,     position:  7, doc: "Day of week -- 1 (Monday) - 7 (Sunday)"
-  # Departure and Arrival Absolute Time
-  field :crs_dep_itime,       IntTime,     position:  8, doc: "scheduled departure time (utc epoch seconds)"
-  field :crs_arr_itime,       IntTime,     position:  9, doc: "scheduled arrival time (utc epoch seconds)"
-  field :act_dep_itime,       IntTime,     position: 10, doc: "actual departure time (utc epoch seconds)"
-  field :act_arr_itime,       IntTime,     position: 11, doc: "actual arrival time (utc epoch seconds)"
-  # Departure and Arrival Local Time of Day
-  field :crs_dep_tod,         String,     position:  12, doc: "time of day for scheduled departure (local, hhmm)"
-  field :crs_arr_tod,         String,     position:  13, doc: "time of day for scheduled arrival (local, hhmm). Not adjusted for wrap-around."
-  field :act_dep_tod,         String,     position:  14, doc: "time of day for actual departure (local, hhmm)"
-  field :act_arr_tod,         String,     position:  15, doc: "time of day for actual arrival (local, hhmm). Not adjusted for wrap-around."
-  # Duration
-  field :crs_duration,        Integer,     position: 16, doc: "CRS flight time, in minutes"
-  field :act_duration,        Integer,     position: 17, doc: "Actual flight time, in minutes"
-  field :air_duration,        Integer,     position: 18, doc: "Air time, in minutes"
-  field :taxi_in_duration,    Integer,     position: 19, doc: "taxi in time, in minutes"
-  field :taxi_out_duration,   Integer,     position: 20, doc: "taxi out time in minutes"
-  # Delay
-  field :is_diverted,         :boolean_10, position: 21, doc: "Was the plane diverted? The actual_duration column remains NULL for all diverted flights."
-  field :is_cancelled,        :boolean_10, position: 22, doc: "was the flight cancelled?"
-  field :cancellation_code,   String,      position: 23, doc: "Reason for cancellation (A = carrier, B = weather, C = NAS, D = security, Z = no cancellation)"
-  field :dep_delay,           Integer,     position: 24, doc: "Difference in minutes between scheduled and actual departure time. Early departures show negative numbers. "
-  field :arr_delay,           Integer,     position: 25, doc: "Difference in minutes between scheduled and actual arrival time. Early arrivals show negative numbers."
-  field :carrier_delay,       Integer,     position: 26, doc: "Carrier delay, in minutes"
-  field :weather_delay,       Integer,     position: 27, doc: "Weather delay, in minutes"
-  field :nas_delay,           Integer,     position: 28, doc: "National Air System delay, in minutes"
-  field :security_delay,      Integer,     position: 29, doc: "Security delay, in minutes"
-  field :late_aircraft_delay, Integer,     position: 30, doc: "Late Aircraft delay, in minutes"
-
-  def to_tsv
-    attrs = attributes
-    attrs[:is_cancelled] = is_cancelled ? 1 : 0
-    attrs[:is_diverted]  = is_diverted  ? 1 : 0
-    attrs[:act_dep_itime] ||= '         '
-    attrs[:act_arr_itime] ||= '         '
-
-    # FIXME
-    attrs[:act_duration]  = ((crs_arr_itime - crs_dep_itime) / 60.0).to_i
-    attrs[:air_duration]  = attrs[:act_duration] - attrs[:crs_duration]
-    attrs.each{|key, val| attrs[key] = val.to_s[-7..-1] if val.to_s.length > 7 } # FIXME: for testing
-
-    attrs.values.join("\t")
-  end
-
-  def flight_date
-    @flight_date ||= Gorillib::Factory::DateFactory.receive(flight_datestr)
-  end
-
-  # checks that the record is sane
-  def lint
-    {
-      act_duration:       (!act_arr_itime) || (act_arr_itime - act_dep_itime == act_duration * 60),
-      crs_duration:       (!crs_arr_itime) || (crs_arr_itime - crs_dep_itime == crs_duration * 60),
-      cancelled_has_code: (is_cancelled == (cancellation_code != "Z")),
-      cancellation_code:  (%w[A B C D Z].include?(cancellation_code)),
-      act_duration:       (!act_duration) || (act_duration == (air_duration + taxi_in_duration + taxi_out_duration)),
-      dep_delay:          (!act_dep_itime) || (dep_delay == (act_dep_itime - crs_dep_itime)/60.0),
-      arr_delay:          (!act_arr_itime) || (arr_delay == (act_arr_itime - crs_arr_itime)/60.0),
-    }
-  end
-end