wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/munging/airline_flights/models.rb

@@ -1,4 +0,0 @@
-require_relative './airline'
-require_relative './airport'
-require_relative './route'
-require_relative './flight'

data/examples/munging/airline_flights/parse.rb

@@ -1,26 +0,0 @@
-
-# see alsospec/examples/munging/airline_flights_spec.rb
-
-      puts described_class.field_names.map{|fn| fn[0..6] }.join("\t")
-      raw_airports = RawDataexpoAirport.load_csv(de_airports_filename)
-      raw_airports.each do |airport|
-        puts airport.to_tsv
-      end
-
-      puts described_class.field_names.join("\t") # .map{|fn| fn[0..6] }.join("\t")
-      raw_airports = described_class.load_csv(raw_airports_filename)
-      raw_airports.each do |airport|
-        # puts airport.to_tsv
-        linted = airport.lint
-        puts [airport.iata, airport.icao, linted.inspect, airport.to_tsv, ].join("\t") if linted.present?
-      end
-
-      Airport.load(raw_airports_filename, de_airports_filename)
-      Airport::AIRPORTS.each{|id,airport|
-        #puts airport.to_tsv
-        linted = airport.lint
-        warn [airport.iata, airport.icao, airport.de_iata, "%-25s" % airport.name, linted.inspect].join("\t") if linted.present?
-      }
-
-
-# Model.from_tuple(...)

data/examples/munging/airline_flights/route.rb

@@ -1,35 +0,0 @@
-
-
-# As of January 2012, the OpenFlights/Airline Route Mapper Route Database
-# contains 59036 routes between 3209 airports on 531 airlines [spanning the
-# globe](http://openflights.org/demo/openflights-routedb-2048.png).  If you
-# enjoy this data, please consider [visiting their page and
-# donating](http://openflights.org/data.html)
-#
-# > Notes: Routes are directional: if an airline operates services from A to B
-# > and from B to A, both A-B and B-A are listed separately. Routes where one
-# > carrier operates both its own and codeshare flights are listed only once.
-#
-# Sample entries
-#
-#     BA,1355,SIN,3316,LHR,507,,0,744 777
-#     BA,1355,SIN,3316,MEL,3339,Y,0,744
-#     TOM,5013,ACE,1055,BFS,465,,0,320
-#
-class RawOpenflightRoute
-  include Gorillib::Model
-
-  field :iataicao,              String,   doc: "2-letter (IATA) or 3-letter (ICAO) code of the airline."
-  field :airline_ofid,          Integer,  doc: "Unique OpenFlights identifier for airline (see Airline)."
-  field :from_airport_iataicao, String,   doc: "3-letter (IATA) or 4-letter (ICAO) code of the source airport."
-  field :from_airport_ofid,     Integer,  doc: "Unique OpenFlights identifier for source airport (see Airport)"
-  field :into_airport_iataicao, String,   doc: "3-letter (IATA) or 4-letter (ICAO) code of the destination airport."
-  field :into_airport_ofid,     Integer,  doc: "Unique OpenFlights identifier for destination airport (see Airport)"
-  field :codeshare,             :boolean, doc: "true if this flight is a codeshare (that is, not operated by Airline, but another carrier); empty otherwise."
-  field :stops,                 Integer,  doc: "Number of stops on this flight, or '0' for direct"
-  field :equipment_list,        String,   doc: "3-letter codes for plane type(s) generally used on this flight, separated by spaces"
-
-  def receive_codeshare(val)
-    super(case val when "Y" then true when "N" then false else val ; end)
-  end
-end

data/examples/munging/airline_flights/timezone_fixup.rb

@@ -1,62 +0,0 @@
-require 'date'
-require 'gorillib/hash/zip'
-
-class Airport
-
-  class TimezoneFixup
-
-    YEARS = (2010 .. 2012).to_a
-
-    DST_RULES = {
-      'E' => { name: 'European',       beg_doy: 'last Sunday in March',       end_doy: 'last Sunday in October',                  beg_dates: {}, end_dates: {}, used_in: 'all European countries (except Iceland), as well as Greenland, Lebanon, Russia and Tunisia. Jordan and Syria are almost the same, starting and ending on Friday instead of Sunday. European DST is also used to (crudely) approximate Iranian DST, although they actually use an entirely different calendar.', },
-      'A' => { name: 'US/Canada',      beg_doy: '2nd  Sunday in March',       end_doy: '1st  Sunday in November',                 beg_dates: {}, end_dates: {}, used_in: 'the United States (except Arizona, Hawaii and island territories) and Canada (with convoluted exceptions).', },
-      'S' => { name: 'South American', beg_doy: '3rd  Sunday in March',       end_doy: '3rd  Sunday in October',     southern: true, beg_dates: {}, end_dates: {}, used_in: 'With some variance in the exact dates, in Argentina, Chile, Mexico, Paraguay, Uruguay as well as the African states of Namibia and Mauritius.', },
-      'O' => { name: 'Australia',      beg_doy: '1st  Sunday in April',       end_doy: '1st  Sunday in October',     southern: true, beg_dates: {}, end_dates: {}, used_in: 'Australia, except for Queensland and the Northern Territory.' },
-      'Z' => { name: 'New Zealand',    beg_doy: '1st  Sunday in April',       end_doy: 'last Sunday in September',   southern: true, beg_dates: {}, end_dates: {}, used_in: 'New Zealand', },
-      'N' => { name: 'None',           beg_doy: nil,                          end_doy: nil,                                       beg_dates: {}, end_dates: {}, used_in: 'DST not observed.', },
-      'U' => { name: 'Unknown',        beg_doy: nil,                          end_doy: nil,                                       beg_dates: {}, end_dates: {}, used_in: 'DST status not known. The same as "None".', },
-    }
-
-    DST_RULES['E'][:beg_dates] = { 1987 => "1987-03-29", 1988 => "1988-03-27", 1989 => "1989-03-26", 1990 => "1990-03-25", 1991 => "1991-03-31", 1992 => "1992-03-29", 1993 => "1993-03-28", 1994 => "1994-03-27", 1995 => "1995-03-26", 1996 => "1996-03-31", 1997 => "1997-03-30", 1998 => "1998-03-29", 1999 => "1999-03-28", 2000 => "2000-03-26", 2001 => "2001-03-25", 2002 => "2002-03-31", 2003 => "2003-03-30", 2004 => "2004-03-28", 2005 => "2005-03-27", 2006 => "2006-03-26", 2007 => "2007-03-25", 2008 => "2008-03-30", 2009 => "2009-03-29", 2010 => "2010-03-28", 2011 => "2011-03-27", 2012 => "2012-03-25", 2013 => "2013-03-31", 2014 => "2014-03-30", 2015 => "2015-03-29", 2016 => "2016-03-27", 2017 => "2017-03-26", 2018 => "2018-03-25", 2019 => "2019-03-31", 2020 => "2020-03-29", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['E'][:end_dates] = { 1987 => "1987-10-25", 1988 => "1988-10-30", 1989 => "1989-10-29", 1990 => "1990-10-28", 1991 => "1991-10-27", 1992 => "1992-10-25", 1993 => "1993-10-31", 1994 => "1994-10-30", 1995 => "1995-10-29", 1996 => "1996-10-27", 1997 => "1997-10-26", 1998 => "1998-10-25", 1999 => "1999-10-31", 2000 => "2000-10-29", 2001 => "2001-10-28", 2002 => "2002-10-27", 2003 => "2003-10-26", 2004 => "2004-10-31", 2005 => "2005-10-30", 2006 => "2006-10-29", 2007 => "2007-10-28", 2008 => "2008-10-26", 2009 => "2009-10-25", 2010 => "2010-10-31", 2011 => "2011-10-30", 2012 => "2012-10-28", 2013 => "2013-10-27", 2014 => "2014-10-26", 2015 => "2015-10-25", 2016 => "2016-10-30", 2017 => "2017-10-29", 2018 => "2018-10-28", 2019 => "2019-10-27", 2020 => "2020-10-25", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['A'][:beg_dates] = { 1987 => "1987-03-08", 1988 => "1988-03-13", 1989 => "1989-03-12", 1990 => "1990-03-11", 1991 => "1991-03-10", 1992 => "1992-03-08", 1993 => "1993-03-14", 1994 => "1994-03-13", 1995 => "1995-03-12", 1996 => "1996-03-10", 1997 => "1997-03-09", 1998 => "1998-03-08", 1999 => "1999-03-14", 2000 => "2000-03-12", 2001 => "2001-03-11", 2002 => "2002-03-10", 2003 => "2003-03-09", 2004 => "2004-03-14", 2005 => "2005-03-13", 2006 => "2006-03-12", 2007 => "2007-03-11", 2008 => "2008-03-09", 2009 => "2009-03-08", 2010 => "2010-03-14", 2011 => "2011-03-13", 2012 => "2012-03-11", 2013 => "2013-03-10", 2014 => "2014-03-09", 2015 => "2015-03-08", 2016 => "2016-03-13", 2017 => "2017-03-12", 2018 => "2018-03-11", 2019 => "2019-03-10", 2020 => "2020-03-08", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['A'][:end_dates] = { 1987 => "1987-11-01", 1988 => "1988-11-06", 1989 => "1989-11-05", 1990 => "1990-11-04", 1991 => "1991-11-03", 1992 => "1992-11-01", 1993 => "1993-11-07", 1994 => "1994-11-06", 1995 => "1995-11-05", 1996 => "1996-11-03", 1997 => "1997-11-02", 1998 => "1998-11-01", 1999 => "1999-11-07", 2000 => "2000-11-05", 2001 => "2001-11-04", 2002 => "2002-11-03", 2003 => "2003-11-02", 2004 => "2004-11-07", 2005 => "2005-11-06", 2006 => "2006-11-05", 2007 => "2007-11-04", 2008 => "2008-11-02", 2009 => "2009-11-01", 2010 => "2010-11-07", 2011 => "2011-11-06", 2012 => "2012-11-04", 2013 => "2013-11-03", 2014 => "2014-11-02", 2015 => "2015-11-01", 2016 => "2016-11-06", 2017 => "2017-11-05", 2018 => "2018-11-04", 2019 => "2019-11-03", 2020 => "2020-11-01", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['S'][:beg_dates] = { 1987 => "1987-10-18", 1988 => "1988-10-16", 1989 => "1989-10-15", 1990 => "1990-10-21", 1991 => "1991-10-20", 1992 => "1992-10-18", 1993 => "1993-10-17", 1994 => "1994-10-16", 1995 => "1995-10-15", 1996 => "1996-10-20", 1997 => "1997-10-19", 1998 => "1998-10-18", 1999 => "1999-10-17", 2000 => "2000-10-15", 2001 => "2001-10-21", 2002 => "2002-10-20", 2003 => "2003-10-19", 2004 => "2004-10-17", 2005 => "2005-10-16", 2006 => "2006-10-15", 2007 => "2007-10-21", 2008 => "2008-10-19", 2009 => "2009-10-18", 2010 => "2010-10-17", 2011 => "2011-10-16", 2012 => "2012-10-21", 2013 => "2013-10-20", 2014 => "2014-10-19", 2015 => "2015-10-18", 2016 => "2016-10-16", 2017 => "2017-10-15", 2018 => "2018-10-21", 2019 => "2019-10-20", 2020 => "2020-10-18", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['S'][:end_dates] = { 1987 => "1987-03-15", 1988 => "1988-03-20", 1989 => "1989-03-19", 1990 => "1990-03-18", 1991 => "1991-03-17", 1992 => "1992-03-15", 1993 => "1993-03-21", 1994 => "1994-03-20", 1995 => "1995-03-19", 1996 => "1996-03-17", 1997 => "1997-03-16", 1998 => "1998-03-15", 1999 => "1999-03-21", 2000 => "2000-03-19", 2001 => "2001-03-18", 2002 => "2002-03-17", 2003 => "2003-03-16", 2004 => "2004-03-21", 2005 => "2005-03-20", 2006 => "2006-03-19", 2007 => "2007-03-18", 2008 => "2008-03-16", 2009 => "2009-03-15", 2010 => "2010-03-21", 2011 => "2011-03-20", 2012 => "2012-03-18", 2013 => "2013-03-17", 2014 => "2014-03-16", 2015 => "2015-03-15", 2016 => "2016-03-20", 2017 => "2017-03-19", 2018 => "2018-03-18", 2019 => "2019-03-17", 2020 => "2020-03-15", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['O'][:beg_dates] = { 1987 => "1987-10-04", 1988 => "1988-10-02", 1989 => "1989-10-01", 1990 => "1990-10-07", 1991 => "1991-10-06", 1992 => "1992-10-04", 1993 => "1993-10-03", 1994 => "1994-10-02", 1995 => "1995-10-01", 1996 => "1996-10-06", 1997 => "1997-10-05", 1998 => "1998-10-04", 1999 => "1999-10-03", 2000 => "2000-10-01", 2001 => "2001-10-07", 2002 => "2002-10-06", 2003 => "2003-10-05", 2004 => "2004-10-03", 2005 => "2005-10-02", 2006 => "2006-10-01", 2007 => "2007-10-07", 2008 => "2008-10-05", 2009 => "2009-10-04", 2010 => "2010-10-03", 2011 => "2011-10-02", 2012 => "2012-10-07", 2013 => "2013-10-06", 2014 => "2014-10-05", 2015 => "2015-10-04", 2016 => "2016-10-02", 2017 => "2017-10-01", 2018 => "2018-10-07", 2019 => "2019-10-06", 2020 => "2020-10-04", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['O'][:end_dates] = { 1987 => "1987-04-05", 1988 => "1988-04-03", 1989 => "1989-04-02", 1990 => "1990-04-01", 1991 => "1991-04-07", 1992 => "1992-04-05", 1993 => "1993-04-04", 1994 => "1994-04-03", 1995 => "1995-04-02", 1996 => "1996-04-07", 1997 => "1997-04-06", 1998 => "1998-04-05", 1999 => "1999-04-04", 2000 => "2000-04-02", 2001 => "2001-04-01", 2002 => "2002-04-07", 2003 => "2003-04-06", 2004 => "2004-04-04", 2005 => "2005-04-03", 2006 => "2006-04-02", 2007 => "2007-04-01", 2008 => "2008-04-06", 2009 => "2009-04-05", 2010 => "2010-04-04", 2011 => "2011-04-03", 2012 => "2012-04-01", 2013 => "2013-04-07", 2014 => "2014-04-06", 2015 => "2015-04-05", 2016 => "2016-04-03", 2017 => "2017-04-02", 2018 => "2018-04-01", 2019 => "2019-04-07", 2020 => "2020-04-05", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['Z'][:beg_dates] = { 1987 => "1987-09-27", 1988 => "1988-09-25", 1989 => "1989-09-24", 1990 => "1990-09-30", 1991 => "1991-09-29", 1992 => "1992-09-27", 1993 => "1993-09-26", 1994 => "1994-09-25", 1995 => "1995-09-24", 1996 => "1996-09-29", 1997 => "1997-09-28", 1998 => "1998-09-27", 1999 => "1999-09-26", 2000 => "2000-09-24", 2001 => "2001-09-30", 2002 => "2002-09-29", 2003 => "2003-09-28", 2004 => "2004-09-26", 2005 => "2005-09-25", 2006 => "2006-09-24", 2007 => "2007-09-30", 2008 => "2008-09-28", 2009 => "2009-09-27", 2010 => "2010-09-26", 2011 => "2011-09-25", 2012 => "2012-09-30", 2013 => "2013-09-29", 2014 => "2014-09-28", 2015 => "2015-09-27", 2016 => "2016-09-25", 2017 => "2017-09-24", 2018 => "2018-09-30", 2019 => "2019-09-29", 2020 => "2020-09-27", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-    DST_RULES['Z'][:end_dates] = { 1987 => "1987-04-05", 1988 => "1988-04-03", 1989 => "1989-04-02", 1990 => "1990-04-01", 1991 => "1991-04-07", 1992 => "1992-04-05", 1993 => "1993-04-04", 1994 => "1994-04-03", 1995 => "1995-04-02", 1996 => "1996-04-07", 1997 => "1997-04-06", 1998 => "1998-04-05", 1999 => "1999-04-04", 2000 => "2000-04-02", 2001 => "2001-04-01", 2002 => "2002-04-07", 2003 => "2003-04-06", 2004 => "2004-04-04", 2005 => "2005-04-03", 2006 => "2006-04-02", 2007 => "2007-04-01", 2008 => "2008-04-06", 2009 => "2009-04-05", 2010 => "2010-04-04", 2011 => "2011-04-03", 2012 => "2012-04-01", 2013 => "2013-04-07", 2014 => "2014-04-06", 2015 => "2015-04-05", 2016 => "2016-04-03", 2017 => "2017-04-02", 2018 => "2018-04-01", 2019 => "2019-04-07", 2020 => "2020-04-05", }.tap{|hsh| hsh.each{|year,date_str| hsh[year] = Date.parse(date_str) } }
-
-    def self.parse_boundary(str, *args)
-      require 'chronic'
-      rank, weekday, art, month = str.split(/\s+/)
-      if rank == 'last'
-        val = ['5th', '4th'].map{|wk| Chronic.parse([wk, weekday, art, month].join(' '), *args) }.compact.first
-      else
-        val = Chronic.parse(str, *args)
-      end
-      Date.new(val.year, val.month, val.day)
-    end
-
-    def self.beg_date(rule, year)
-      DST_RULES[rule][:beg_dates][year] ||= parse_boundary(DST_RULES[rule][:beg_doy], now: Time.utc(year, 1, 1))
-    end
-    def self.end_date(rule, year)
-      DST_RULES[rule][:end_dates][year] ||= parse_boundary(DST_RULES[rule][:end_doy], now: Time.utc(year, 1, 1))
-    end
-
-    def self.table
-      %w[E A S O Z].each{|rule| YEARS.each{|year| beg_date(rule, year) ; end_date(rule, year) } }
-      DST_RULES
-    end
-
-    def self.dst?(rule, val)
-      early = beg_date(rule, val.year)
-      late  = end_date(rule, val.year)
-      in_range = (val >= early) && (val < late)
-      DST_RULES[rule][:southern] ? (not in_range) : in_range
-    end
-
-  end
-end

data/examples/munging/airports/40_wbans.txt

@@ -1,40 +0,0 @@
-13874
-13874
-14739
-13881
-13881
-03017
-03017
-03927
-03927
-94847
-94847
-14734
-14734
-53127
-99999
-12960
-12960
-94789
-94789
-23169
-23169
-23174
-23174
-12815
-12815
-12839
-12839
-14922
-14922
-94846
-94846
-13739
-13739
-23183
-23183
-99999
-24233
-24233
-23234
-23234

data/examples/munging/airports/filter_weather_reports.rb

@@ -1,37 +0,0 @@
-#!/usr/bin/env ruby
-# encoding:UTF-8
-
-require 'wukong'
-require 'pathname'
-load '/home/dlaw/dev/wukong/examples/wikipedia/munging_utils.rb'
-
-module WeatherFilter
-  class Mapper < Wukong::Streamer::LineStreamer
-
-    WBAN_FILENAME = '/home/dlaw/dev/wukong/examples/airports/wbans.txt'
-    USA_WBAN_FILENAME = '/home/dlaw/dev/wukong/examples/airports/usa_wbans.txt'
-    FORTY_WBANS_FILENAME = '/home/dlaw/dev/wukong/examples/airports/40_wbans.txt'
-
-    def initialize
-      @wbans = []
-      wban_file = File.open(FORTY_WBANS_FILENAME)
-      wban_file.each_line do |line|
-        @wbans << line[0..-2]
-      end
-    end
-
-    def process line
-      MungingUtils.guard_encoding(line) do |clean_line|
-        wban = Pathname(ENV['map_input_file']).basename.to_s.split('-')[1]
-        if @wbans.include? wban
-          yield line
-        end
-      end
-    end
-  end
-end
-
-Wukong::Script.new(
-  WeatherFilter::Mapper,
-  nil
-).run

data/examples/munging/airports/join.pig

@@ -1,31 +0,0 @@
-/* This was a misguided attempt at generating a list of WBAN IDs assigned to airports by filtering the mshr_enhanced
- * and joining it with isd_stations. This is misguided because mshr_enhanced contains much more data than isd_stations,
- * and also contains multiple entries for each weather station, making it non-obvious how best to join the data.
- * A simpler and better approach, taken in usa_wbans.pig and wbans.pig, is to filter and unique mshr_enhanced.
- */
-
-mshr = LOAD '/Users/dlaw/Desktop/stations/mshr_enhanced.tsv' AS 
- (source_id:chararray, source:chararray, begin_date:chararray, end_date:chararray, station_status:chararray, 
-  ncdcstn_id:chararray, icao_id:chararray, wban_id:chararray, faa_id:chararray, nwsli_id:chararray, wmo_id:chararray, 
-  coop_id:chararray, transmittal_id:chararray, ghcnd_id:chararray, name_principal:chararray, name_principal_short:chararray, 
-  name_coop:chararray, name_coop_short:chararray, name_publication:chararray, name_alias:chararray, nws_clim_div:chararray, 
-  nws_clim_div_name:chararray, state_prov:chararray, county:chararray, nws_st_code:chararray, fips_country_code:chararray, 
-  fips_country_name:chararray, nws_region:chararray, nws_wfo:chararray, elev_ground:chararray, elev_ground_unit:chararray, 
-  elev_barom:chararray, elev_barom_unit:chararray, elev_air:chararray, elev_air_unit:chararray, elev_zerodat:chararray, 
-  elev_zerodat_unit:chararray, elev_unk:chararray, elev_unk_unit:chararray, lat_dec:chararray, lon_dec:chararray, 
-  lat_lon_precision:chararray, relocation:chararray, utc_offset:chararray, obs_env:chararray, platform:chararray);
-
-mshr_grouped = GROUP mshr BY (icao_id, wban_id, faa_id, begin_date, end_date);
-mshr_final = FOREACH mshr_grouped GENERATE FLATTEN(group) AS (wban_id, icao_id, faa_id, begin_date, end_date);
-
-stations = LOAD '/Users/dlaw/Desktop/stations/stations.tsv' AS
- (usaf_id:chararray, wban_id:chararray, station_name:chararray, wmo_country_id:chararray, fips_country_id:chararray, 
- state:chararray, icao_call_sign:chararray, latitude:chararray, longitude:chararray, elevation:chararray, begin:chararray, end:chararray);
-
-first_pass_j = JOIN mshr_final BY (wban_id) RIGHT OUTER, stations BY (wban_id);
-first_pass_f = FILTER first_pass_j BY (mshr_final::icao_id is not null);
-first_pass = FOREACH first_pass_f GENERATE
-  stations::wban_id, mshr_final::icao_id, stations::icao_call_sign, stations::usaf_id,  mshr_final::faa_id, 
-  stations::station_name, stations::wmo_country_id, stations::fips_country_id, stations::state, stations::latitude, stations::longitude, stations::elevation, stations::begin, stations::end;
-
-STORE first_pass INTO '/Users/dlaw/Desktop/stations/airport_stations';

data/examples/munging/airports/to_tsv.rb

@@ -1,33 +0,0 @@
-#!/usr/bin/env ruby
-load 'flat/lib/flat.rb'
-
-# This is a script that uses the flat file parser
-# to transform the mshr enhanced data file and the 
-# ISD stations list from fixed-width to .tsv.
-# The script must be in the same directory with 
-# mshr_enhanced.txt, isd_stations.txt, and the
-# flat file parsing library to work.
-
-# mshr-enhanced format description can be found at
-# ftp://ftp.ncdc.noaa.gov/pub/data/homr/docs/MSHR_Enhanced_Table.txt
-
-# The actual mshr-enhanced table can be found at 
-# http://www.ncdc.noaa.gov/homr/file/mshr_enhanced.txt.zip
-
-# isd_stations can be found at
-# http://www1.ncdc.noaa.gov/pub/data/noaa/ish-history.txt
-
-# Format strings
-MSHR_FORMAT_STRING = %{s20 s10 s8 s8 s20 s20 s20 s20 s20 s20 s20 s20 s20 s20
-                       s100 s30 s100 s30 s100 s100 s10 s40 s10 s50 s2 s2 s100 
-                       s30 s10 s40 s20 s40 s20 s40 s20 s40 s20 s40 s20 s20 s20 
-                       s10 s62 s16 s40 s100}
-ISD_FORMAT_STRING = %{s6 s5 s29 s2 s2 s2 s5 D6e3 D7e3 D6e1 _2 s8 s8}
-
-# Parse mshr_enhanced
-mshr_parser = Flat.create_parser(MSHR_FORMAT_STRING,1)
-mshr_parser.file_to_tsv('mshr_enhanced.txt','mshr_enhanced.tsv')
-
-# Parse isd_stations
-isd_parser = Flat.create_parser(ISD_FORMAT_STRING,1,false)
-isd_parser.file_to_tsv('isd_stations.txt','isd_stations.tsv')

data/examples/munging/airports/usa_wbans.pig

@@ -1,19 +0,0 @@
-// Outputs a list of WBAN ids that are assigned to airports in the USA
-
-mshr = LOAD '/Users/dlaw/Desktop/stations/mshr_enhanced.tsv' AS 
- (source_id:chararray, source:chararray, begin_date:chararray, end_date:chararray, station_status:chararray, 
-  ncdcstn_id:chararray, icao_id:chararray, wban_id:chararray, faa_id:chararray, nwsli_id:chararray, wmo_id:chararray, 
-  coop_id:chararray, transmittal_id:chararray, ghcnd_id:chararray, name_principal:chararray, name_principal_short:chararray, 
-  name_coop:chararray, name_coop_short:chararray, name_publication:chararray, name_alias:chararray, nws_clim_div:chararray, 
-  nws_clim_div_name:chararray, state_prov:chararray, county:chararray, nws_st_code:chararray, fips_country_code:chararray, 
-  fips_country_name:chararray, nws_region:chararray, nws_wfo:chararray, elev_ground:chararray, elev_ground_unit:chararray, 
-  elev_barom:chararray, elev_barom_unit:chararray, elev_air:chararray, elev_air_unit:chararray, elev_zerodat:chararray, 
-  elev_zerodat_unit:chararray, elev_unk:chararray, elev_unk_unit:chararray, lat_dec:chararray, lon_dec:chararray, 
-  lat_lon_precision:chararray, relocation:chararray, utc_offset:chararray, obs_env:chararray, platform:chararray);
-
-mshr_grouped = GROUP mshr BY (icao_id, wban_id, faa_id, fips_country_code);
-mshr_flattened = FOREACH mshr_grouped GENERATE FLATTEN(group) AS (wban_id, icao_id, faa_id, fips_country_code);
-mshr_filtered = FILTER mshr_flattened BY (icao_id is not null and wban_id is not null and fips_country_code == 'US');
-
-mshr_final = FOREACH mshr_filtered GENERATE wban_id;
-STORE mshr_final INTO '/Users/dlaw/Desktop/stations/usa_wbans';

data/examples/munging/airports/usa_wbans.txt

@@ -1,2157 +0,0 @@
-03069
-94119
-04204
-94991
-53928
-54779
-03196
-63844
-63875
-53929
-64773
-04201
-94076
-04849
-63834
-94037
-63839
-63876
-63877
-03051
-63878
-12981
-12971
-53866
-53904
-12978
-53862
-94061
-63848
-63879
-63847
-53990
-53966
-53972
-03053
-94298
-03032
-63827
-92807
-92807
-03064
-93226
-63880
-94993
-53964
-94623
-94070
-94943
-04862
-12832
-03974
-94299
-14737
-13962
-23050
-14929
-14929
-03019
-13869
-04863
-14756
-53991
-54916
-13959
-24283
-93730
-94997
-04847
-53930
-93940
-03970
-03043
-94998
-13705
-03034
-03064
-94968
-93915
-93065
-04827
-04993
-54770
-53909
-14762
-12804
-03820
-13873
-24044
-04828
-04864
-14930
-94999
-53931
-63853
-53175
-53870
-24015
-93773
-93773
-14813
-14735
-12932
-53146
-93097
-03958
-94910
-23061
-24160
-63833
-23047
-23047
-13870
-53989
-54816
-94989
-12899
-13871
-93846
-94974
-14847
-94975
-04850
-14736
-93067
-93227
-12897
-13701
-94849
-04837
-04837
-53983
-54917
-53932
-03949
-54768
-53915
-94889
-93991
-12953
-04808
-94790
-04865
-94987
-53865
-93073
-03973
-54754
-93796
-94224
-54809
-94929
-13874
-03035
-13958
-04825
-94287
-14946
-14605
-04901
-04902
-23224
-03892
-13904
-13958
-14897
-93797
-53933
-03812
-14777
-23191
-04903
-94190
-53959
-04205
-04904
-14910
-63901
-04779
-13861
-53129
-94815
-54772
-93216
-13944
-14775
-13803
-24119
-24119
-54817
-12971
-54904
-93942
-94946
-93240
-53881
-23159
-12803
-94961
-14816
-03036
-14740
-94702
-14702
-94871
-04751
-24028
-24234
-23155
-13838
-04905
-03024
-53882
-04725
-04725
-14606
-23005
-14616
-94055
-13876
-94289
-94793
-94947
-23157
-63835
-24033
-24011
-24011
-04839
-13820
-03065
-93068
-14958
-04842
-53145
-53988
-24130
-23036
-04853
-12982
-13726
-12818
-03872
-94902
-03859
-23158
-24217
-54760
-23225
-13802
-94046
-24180
-03893
-54831
-94700
-03999
-53823
-13897
-25312
-94185
-04906
-24131
-24267
-14739
-12809
-54765
-03044
-24164
-53918
-12917
-53883
-53992
-94938
-14931
-12919
-12919
-13904
-14815
-24135
-64705
-13970
-14742
-14964
-14733
-14733
-23156
-23152
-04866
-93783
-04813
-54921
-03959
-94282
-53934
-54733
-93943
-93808
-93721
-54922
-94054
-13814
-24133
-03182
-92804
-12981
-24132
-04867
-04839
-04868
-14817
-13883
-13883
-24046
-14895
-23051
-14607
-04907
-03732
-04908
-04909
-03037
-13825
-94625
-04101
-53884
-23254
-03038
-54764
-14966
-93129
-94977
-53916
-24017
-23007
-54743
-24286
-14703
-53850
-13884
-93069
-54901
-54828
-93967
-04805
-03935
-94866
-03914
-13882
-53128
-53935
-93736
-13880
-93203
-14990
-04910
-93809
-04869
-93798
-03802
-04911
-03881
-04912
-54920
-03894
-14820
-14820
-54832
-13999
-03904
-94266
-13881
-53845
-23136
-14821
-94870
-14858
-94940
-54923
-04913
-13984
-23086
-93033
-03179
-13981
-93075
-24045
-24136
-12867
-14745
-04914
-93037
-12947
-03945
-13983
-93799
-03701
-24089
-03960
-53981
-03039
-53936
-03027
-54905
-93134
-94199
-94624
-93718
-53860
-12924
-12924
-03177
-53912
-13866
-12879
-93842
-03932
-04915
-03847
-24137
-12833
-03727
-53867
-53993
-63840
-94032
-93814
-23077
-24202
-23008
-04870
-94890
-13941
-94979
-53902
-54774
-14751
-24018
-24018
-94204
-03164
-94056
-93728
-12834
-23161
-13960
-13728
-54791
-93235
-93815
-63881
-94908
-13743
-53852
-13985
-13985
-54781
-03887
-04916
-03017
-23062
-53925
-14822
-94119
-25515
-04851
-03927
-22015
-94057
-13839
-93843
-93042
-04135
-24012
-93771
-04871
-14747
-03073
-22001
-14913
-14913
-54833
-24138
-24219
-23109
-93784
-23078
-03994
-03724
-94984
-63903
-13837
-53885
-04917
-94891
-13707
-93770
-94878
-94892
-24103
-03702
-63842
-03160
-03976
-54844
-93005
-22010
-14933
-94704
-23162
-94962
-53905
-03991
-53853
-94847
-04830
-53937
-53938
-93026
-04872
-04787
-03070
-94928
-94982
-94982
-54924
-03184
-53910
-54734
-04918
-54786
-03809
-13910
-04919
-03987
-53965
-03049
-14905
-94239
-14991
-03983
-03983
-12983
-04920
-13786
-13787
-73805
-23098
-03703
-03197
-23114
-23179
-94721
-94050
-53864
-12906
-54758
-04873
-23063
-03844
-54838
-93076
-63843
-53886
-24213
-04806
-13729
-24121
-63882
-93992
-03977
-14748
-24220
-94964
-23044
-23154
-54757
-24165
-13989
-03165
-03704
-53112
-13909
-53887
-24193
-04845
-54766
-53876
-04921
-53851
-24141
-14608
-03020
-53872
-14860
-25318
-12961
-04874
-14824
-94853
-13935
-03756
-94971
-53110
-04875
-04922
-63872
-24221
-94195
-92808
-04923
-93817
-04111
-53825
-94726
-53939
-93719
-14734
-12971
-53888
-53842
-03705
-12836
-53114
-03706
-53969
-53907
-93735
-93996
-14914
-93193
-93740
-93737
-94969
-24146
-23167
-24047
-94963
-94015
-53831
-13730
-03981
-03981
-03022
-14825
-04876
-04924
-94006
-03736
-53819
-53819
-04925
-94966
-13840
-53841
-03185
-04926
-94276
-03124
-54792
-04780
-04927
-94868
-13763
-54818
-04840
-03103
-12849
-13744
-03918
-13921
-93733
-14704
-23090
-12835
-94957
-94035
-94062
-14826
-53889
-23256
-94933
-13920
-14719
-04928
-04877
-12895
-53890
-54787
-13947
-94948
-14944
-14944
-04929
-13945
-13964
-23091
-04930
-53113
-03018
-13807
-13961
-03888
-03166
-04836
-03707
-14827
-53891
-54793
-03985
-03985
-12885
-03734
-93993
-54773
-03896
-13975
-93764
-12993
-13940
-94959
-03148
-23168
-94023
-23064
-53940
-03195
-04999
-53977
-23055
-24087
-13764
-24157
-53126
-53892
-24048
-53866
-24112
-14916
-14750
-03901
-14976
-94008
-94008
-64774
-04931
-12876
-23066
-03025
-04843
-53907
-23065
-23065
-93929
-13939
-04854
-12923
-53941
-13886
-94992
-03056
-24142
-12990
-94626
-93057
-12816
-53913
-14707
-04878
-24146
-53984
-93874
-94919
-14898
-14898
-53874
-24201
-14935
-03902
-03992
-94860
-94860
-13713
-14829
-13723
-03870
-03870
-14715
-24143
-53893
-53942
-93007
-23081
-94833
-03030
-53838
-13926
-03929
-13978
-03708
-53837
-24051
-53979
-63889
-53967
-04932
-03186
-54762
-04807
-53820
-54850
-53855
-13833
-03709
-93986
-12994
-04933
-04934
-03908
-04935
-94025
-12962
-03023
-03710
-53894
-94038
-63873
-03961
-14752
-93747
-93706
-93218
-03980
-93046
-03167
-94931
-54728
-24101
-94261
-04998
-53119
-53869
-13927
-03810
-93990
-14894
-24144
-03933
-53895
-23002
-94187
-53896
-53127
-04936
-53111
-03711
-93034
-14936
-13806
-03962
-12918
-94745
-94745
-53839
-94225
-63836
-53970
-04113
-03712
-12904
-13971
-03852
-63852
-53897
-93729
-94949
-93757
-12826
-03856
-94814
-64761
-03860
-53857
-63883
-03868
-93823
-14609
-12927
-26522
-13986
-23170
-14758
-94012
-93228
-92809
-54790
-63837
-94720
-12979
-94973
-03968
-04829
-04997
-04857
-53842
-63884
-03923
-93738
-04724
-12960
-94073
-04937
-94990
-03928
-03928
-53118
-24145
-64706
-94039
-53943
-53135
-93167
-04879
-93785
-04989
-53898
-54767
-53944
-04880
-25626
-04938
-03972
-13781
-13748
-13841
-13841
-04833
-04833
-24091
-94893
-93819
-93819
-53972
-23040
-14918
-23141
-93807
-23194
-03984
-14937
-53899
-03144
-14778
-14938
-54827
-92813
-94014
-93726
-04781
-04881
-04826
-94761
-03026
-23104
-23104
-94926
-94623
-12969
-04844
-93909
-93194
-93244
-54772
-24166
-03940
-53945
-13889
-13889
-03953
-13973
-53978
-94051
-03963
-04110
-94789
-03730
-53824
-04720
-63885
-04939
-03889
-03889
-13987
-04940
-14919
-03713
-14834
-63801
-53971
-04726
-53946
-94854
-53947
-53966
-53982
-14833
-54906
-63846
-04882
-03714
-14989
-25314
-24223
-53995
-03013
-14835
-12883
-93091
-14836
-24022
-23169
-03950
-23174
-23042
-23042
-54735
-24023
-23020
-53963
-13776
-12976
-54925
-03937
-03937
-54736
-13812
-04883
-94765
-12819
-94709
-93820
-13702
-93987
-13976
-14732
-23129
-03821
-24148
-94128
-53973
-53844
-03875
-23067
-03731
-93010
-13963
-53813
-53813
-14623
-04941
-03715
-94285
-04114
-53919
-94236
-53975
-24021
-14939
-14886
-04809
-63802
-14921
-54737
-94049
-24172
-64752
-53861
-04831
-04831
-13810
-03849
-23243
-04849
-12907
-03930
-04942
-93041
-04108
-14920
-13829
-23112
-03982
-03982
-13902
-93774
-53964
-23111
-93812
-04995
-12975
-23285
-24150
-04943
-23054
-53827
-54826
-53801
-03997
-94991
-94723
-24149
-24036
-13809
-93767
-04944
-14994
-53879
-93009
-13733
-54926
-03952
-03952
-63803
-63851
-93242
-23023
-23023
-03818
-03817
-94296
-94052
-94894
-14845
-94048
-93919
-23208
-54820
-23257
-12810
-03947
-94040
-03813
-12815
-14940
-03071
-93810
-63804
-14711
-14819
-54911
-93782
-24152
-13865
-13893
-23203
-53996
-14891
-12959
-94985
-24225
-03716
-13864
-04945
-04789
-13895
-04884
-53817
-13736
-53859
-94950
-03936
-23206
-24215
-14710
-03183
-93768
-12839
-94011
-94960
-94895
-13735
-94988
-04946
-13988
-14839
-14840
-63805
-03811
-53997
-93953
-14954
-04834
-12838
-12838
-93950
-24151
-04992
-23176
-14923
-63886
-24154
-24037
-14610
-13942
-03181
-54788
-94976
-94887
-03858
-54738
-94273
-03040
-94896
-04855
-13894
-13894
-23258
-04885
-24013
-26621
-04948
-54789
-94705
-94983
-04947
-04949
-14753
-13766
-94850
-94850
-13827
-13734
-93035
-93765
-54907
-63859
-53832
-23259
-13896
-14837
-24153
-14922
-94725
-54746
-04103
-12916
-14804
-12896
-93013
-93744
-53802
-54780
-63806
-94897
-04104
-54723
-24106
-04950
-93236
-04951
-54771
-93894
-94986
-94724
-03865
-94869
-24110
-63807
-93985
-04952
-14755
-63887
-93136
-53921
-13821
-04953
-53826
-03131
-94182
-03041
-13717
-93205
-04954
-94852
-93831
-93901
-12958
-14855
-93727
-53847
-14780
-63890
-53917
-13762
-03154
-53848
-93102
-13911
-12926
-13750
-12946
-23239
-13721
-14611
-93104
-93837
-12925
-93776
-23199
-93743
-53849
-13754
-03190
-53115
-93107
-23110
-03866
-12974
-24228
-03855
-93839
-93839
-12928
-12850
-94297
-03853
-23240
-93115
-93841
-93116
-93111
-93114
-13769
-93117
-03757
-23244
-24255
-12973
-93121
-14793
-94728
-13773
-03145
-93832
-93101
-14790
-93112
-94299
-93241
-93753
-23230
-93217
-94980
-12861
-53948
-03741
-03031
-94958
-04886
-54912
-14949
-14941
-93775
-94063
-53854
-24126
-26632
-53833
-03967
-53121
-13967
-54807
-53949
-03717
-94703
-14622
-94017
-24227
-03196
-94951
-63808
-53822
-04955
-14942
-03718
-94197
-92822
-04956
-04957
-93040
-24162
-24285
-03102
-03737
-54819
-54919
-12882
-63809
-53868
-54752
-04958
-04959
-94846
-54756
-13737
-53998
-94746
-12841
-04224
-53962
-14808
-94855
-04804
-94927
-24284
-14950
-04106
-03948
-93210
-04960
-94994
-04961
-53803
-54704
-93786
-64707
-93110
-04962
-03850
-04887
-03957
-03163
-54813
-94898
-94899
-94173
-03170
-04109
-04856
-12968
-24222
-03816
-03816
-13846
-23289
-27506
-93988
-04742
-64776
-54913
-12844
-04832
-92824
-54852
-04963
-53863
-24155
-24155
-24229
-04964
-54778
-03058
-54927
-03882
-03162
-12812
-13783
-04852
-93741
-13739
-04888
-24024
-23183
-14842
-53808
-12873
-24156
-12957
-24025
-94823
-03804
-94967
-14962
-12986
-94733
-14841
-23182
-25625
-92805
-94086
-13969
-94732
-54914
-13899
-04889
-13923
-13714
-03180
-03931
-03993
-03975
-14757
-03052
-03998
-53950
-14604
-53858
-04965
-93209
-23184
-63810
-63874
-25330
-93955
-14761
-24163
-14763
-13868
-04743
-93983
-93138
-12935
-93714
-26526
-94817
-12936
-93978
-23149
-54782
-93058
-93141
-94129
-03021
-64708
-14765
-53951
-24174
-23097
-03954
-54915
-53952
-04838
-14764
-94263
-54769
-03045
-54825
-04134
-94818
-03171
-14806
-24090
-12995
-93772
-03971
-24231
-24216
-12984
-24006
-54928
-03738
-24257
-14712
-14767
-04966
-24230
-94925
-13722
-23021
-94022
-94107
-94822
-04967
-53104
-04803
-63811
-93232
-13740
-03846
-03016
-23069
-23119
-24061
-24061
-94601
-12972
-53953
-24027
-53834
-14717
-64775
-93801
-03735
-54821
-12911
-04968
-23043
-53120
-23185
-54824
-94248
-13741
-14768
-53954
-04969
-23009
-04970
-54909
-53965
-04890
-14864
-03029
-53999
-92802
-04891
-04971
-93997
-53958
-14925
-63841
-12894
-23052
-94288
-53920
-63812
-94737
-53908
-14992
-93759
-24057
-23901
-94194
-54834
-63813
-04972
-93781
-04113
-23232
-93084
-23049
-23188
-63814
-12921
-03822
-94836
-54910
-23190
-23122
-04841
-14848
-93206
-94033
-93720
-04741
-23237
-04973
-23187
-93821
-03192
-03178
-94028
-24233
-03172
-53143
-14770
-03969
-53904
-94290
-12854
-94990
-94176
-64709
-23234
-03719
-04107
-04996
-64710
-13995
-13995
-63888
-92814
-12977
-04974
-13925
-23186
-93760
-04975
-94227
-24029
-13957
-13957
-03763
-24259
-23293
-23293
-93027
-23034
-23034
-04892
-24114
-12909
-03012
-14846
-04976
-24127
-24232
-53955
-04893
-53141
-94740
-03919
-93998
-03879
-93937
-63815
-93225
-24196
-93197
-24237
-54785
-23273
-93184
-03050
-93911
-23233
-04112
-24030
-03059
-03720
-03101
-53117
-93804
-04201
-03028
-92806
-93822
-13966
-14972
-04894
-93231
-53985
-23902
-12871
-93083
-53906
-13849
-12970
-13878
-14847
-14926
-04895
-13993
-13994
-14927
-23213
-23274
-92815
-04824
-94161
-03966
-03721
-23202
-54908
-14943
-93063
-24258
-63816
-13824
-14714
-03965
-23033
-24235
-54930
-14771
-13930
-04129
-12975
-12957
-03044
-12984
-12980
-23070
-54777
-63817
-03938
-63818
-92801
-23048
-93806
-24207
-93045
-03722
-24241
-04848
-94741
-54822
-03011
-04102
-93778
-94087
-13919
-04896
-94274
-12898
-54931
-53914
-54953
-93805
-93805
-03979
-12888
-63819
-04977
-03191
-03122
-04978
-03878
-94830
-13996
-94053
-12842
-92825
-23153
-93984
-94978
-53956
-13877
-93201
-53911
-03104
-03723
-24242
-54823
-14792
-12886
-13968
-73804
-93862
-23160
-14850
-94956
-93230
-03996
-94178
-04979
-03986
-13977
-13972
-13891
-54776
-63825
-94194
-04114
-04116
-04110
-94184
-94281
-94794
-63820
-03742
-94047
-04897
-94981
-14880
-94240
-93989
-63821
-23275
-64753
-04980
-03946
-53901
-04898
-54739
-53806
-23903
-53903
-14787
-12985
-53871
-13857
-53856
-93780
-93214
-03978
-93234
-93241
-12912
-23131
-92816
-63822
-94030
-53123
-13997
-93144
-53818
-93999
-03838
-93845
-54829
-03995
-23130
-94930
-53873
-13858
-04846
-63823
-12843
-54740
-04858
-04981
-24032
-03042
-53101
-63824
-94298
-92817
-93795
-04982
-04983
-63845
-04899
-63832
-04221
-93775
-03725
-26618
-93739
-53986
-03832
-24198
-53130
-03159
-13932
-24128
-13860
-14706
-24062
-14794
-23277
-14615
-03726
-53957
-94163
-92802
-53976
-12868
-53922
-04984
-14853
-24243
-94911
-14852
-23195
-53116
-93044
-93824
-25708
-25715
-25348
-25375
-25331
-27401
-26615
-26415
-26645
-46402
-27502
-26533
-26497
-25316
-25624
-26418
-25361
-26410
-26445
-26633
-26646
-26643
-26649
-25704
-25513
-25501
-26486
-25616
-26641
-96401
-26401
-26422
-25623
-26407
-25357
-26650
-26650
-26523
-26554
-26411
-26498
-25369
-26555
-26551
-26519
-26501
-26501
-26509
-26465
-26513
-26425
-26628
-26703
-25310
-25322
-25335
-26521
-26548
-26548
-26552
-25323
-25507
-26651
-26522
-26499
-25376
-25521
-25404
-25519
-25506
-26535
-46403
-25518
-25309
-96402
-25503
-26602
-25503
-26542
-25325
-27408
-26502
-25367
-26652
-26491
-26546
-27517
-26480
-26631
-26557
-26510
-25402
-25403
-26512
-26524
-25377
-26621
-26635
-26409
-46404
-26622
-26451
-26516
-26435
-25308
-26644
-25378
-26617
-26653
-26412
-26616
-25628
-26639
-24690
-25329
-25508
-25604
-25613
-26623
-27515
-46405
-26654
-26561
-26704
-27406
-25617
-26625
-25333
-26558
-26553
-26647
-25713
-25516
-26439
-25341
-26414
-26534
-26514
-26559
-45715
-26529
-26634
-25629
-26528
-26536
-26492
-26492
-27518
-46406
-26508
-26627
-26560
-26442
-26479
-26642
-26442
-26438
-25338
-27503
-27503
-26630
-26648
-26444
-26443
-26445
-26484
-25339
-46407
-26413
-22501
-22508
-22552
-22551
-22548
-22550
-21510
-22536
-22534
-22547
-22539
-22514
-22519
-22521
-22517
-22524
-22516
-22526
-21508
-21504
-22513
-27403
-26624
-26638
-11640
-11624
-11603
-11653
-11630
-11630
-11641
-11655
-11649