wukong 3.0.0.pre → 3.0.0.pre2

data/examples/munging/geo/tasks.rake

@@ -0,0 +1,71 @@
+require_relative('../../rake_helper')
+
+Pathname.register_paths(
+  geo_data:                  [:data, 'geo'],
+  geo_work:                  [:work, 'geo'],
+  geo_code:                  File.dirname(__FILE__),
+  #
+  iso_3166:                  [:geo_data, 'iso_codes', "iso_3166.tsv"   ],
+  geonames_countries:        [:geo_data, 'geonames',  "geonames_countries.json"   ],
+  #
+  countries_json:            [:geo_work, "countries.json"   ],
+  country_name_lookup:       [:geo_work, "country_name_lookup.tsv"   ],
+  )
+
+chain :geo do
+  code_files = FileList[Pathname.of(:geo_code, '*.rb').to_s]
+  chain(:countries) do
+
+    task(:load) do
+      require_relative('./geo_models')
+      require_relative('./geo_json')
+      require_relative('./geonames_models')
+      require_relative('./iso_codes')
+      require_relative('./reconcile_countries')
+      CountryReconciler.load_reconciled_countries
+    end
+
+    # desc 'load the ISO 3166 countries'
+    # task(:countries_iso_3166, after: [code_files, :force]) do |dest|
+    #   require_relative('./iso_codes')
+    #   p Wukong::Data::CountryCode.for_any_name('Bolivia')
+    # end
+
+    # step(:geonames_countries, doc: 'load the Geonames countries',
+    #   invoke: 'geo:countries:load',
+    #   # , after: [code_files, :force]
+    #   ) do |dest|
+    #   Wukong::Data::GeonamesGeoJson.load(:geonames_countries)
+    # end
+
+    desc 'Add the iso_codes data to the geonames countries'
+    create_file(:countries_json, invoke: 'geo:countries:load', after: [code_files, :force]) do |dest|
+      Geo::Country.values.each do |country|
+        dest << country.to_json << "\n"
+      end
+    end
+
+    desc 'Add the iso_codes data to the geonames countries'
+    create_file(:country_name_lookup, invoke: 'geo:countries:load', after: [code_files, :force]) do |dest|
+      Geo::Country.values.each do |ct|
+        ct.names.each do |alt_name| 
+          dest << [ct.country_id, ct.country_al3id, ct.country_numid,
+            ct.tld_id, ct.geonames_id,
+            ct.name,
+            Geo::Place.slugify_name(alt_name), alt_name
+          ].join("\t") << "\n"
+        end
+      end
+    end
+
+    # task(:country_name_lookup => :load) do
+    #   Geo::CountryNameLookup.load
+    # end
+
+  end
+end
+
+task :default => [
+  # 'geo:countries',
+  'geo:countries:country_name_lookup'
+]

data/examples/munging/rake_helper.rb

@@ -0,0 +1,62 @@
+require 'gorillib'
+require 'gorillib/data_munging'
+require 'configliere'
+
+S3_BUCKET = 'bigdata.chimpy.us'
+S3_DATA_ROOT = "s3n://#{S3_BUCKET}/data"
+HDFS_DATA_ROOT = '/data'
+
+Settings.define :orig_data_root, default: HDFS_DATA_ROOT, description: "directory root for input data"
+Settings.define :scratch_data_root, default: HDFS_DATA_ROOT, description: "directory root for scratch data"
+Settings.define :results_data_root, default: HDFS_DATA_ROOT, description: "directory root for results data"
+Settings.define :mini, description: 'Run in mini mode - operate inside the mini version of the specified universe',type: :boolean, default: false
+Settings.define :universe, description: 'Universe to draw data from', finally: ->(c){ c.universe ||= (c.mini? ? "mini" : "full") }
+Settings.define :pig_path, default: '/usr/local/bin/pig'
+Settings.define :local, type: :boolean, default: false
+
+def Settings.mini?; !! Settings.mini ; end # BANG BANG BANG
+def Settings.wu_run_cmd; (local ? '--run=local' : '--run') ; end;
+
+def dir_exists? (dir)
+  if Settings.local
+    return File.exists? dir
+  else
+    `hadoop fs -test -e #{dir}`
+    return $?.exitstatus == 0
+  end
+end
+
+def wukong(script, input, output, options={})
+  input = Pathname.of(input)
+  output = Pathname.of(output)
+  if dir_exists? output
+    puts "#{output} exists. Assuming that this job has already run..."
+    return
+  end
+  opts = ['--rm']
+  options.each_pair do |k,v|
+    opts << "--#{k}=#{v}"
+  end
+  opts << input
+  opts << output
+  ruby(script, Settings.wu_run_cmd,*opts)
+end
+
+def wukong_xml(script, input, output, split_tag)
+  wukong(script,input,output,{split_on_xml_tag: split_tag})
+end
+
+def pig(script_name, options={})
+  cmd = Settings.pig_path
+  options.each_pair do |k,v|
+    v = Pathname.of(v) if v.is_a? Symbol
+    if k.to_s.include? '_out' and dir_exists? v
+      puts "#{v} already exists. Assuming that this job has already run..."
+      return
+    else
+      cmd += " -param #{k.upcase}=#{v}"
+    end
+  end
+  cmd += " #{script_name}"
+  sh cmd
+end

data/examples/munging/weather/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/examples/munging/weather/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+  
+gem 'gorillib',    :path => '/Users/dlaw/dev/gorillib'
+gem 'wukong', :path =>'/Users/dlaw/dev/wukong_og'

data/examples/munging/weather/Rakefile

@@ -0,0 +1,28 @@
+require 'configliere'
+Settings.use :commandline
+
+require_relative '../rake_helper'
+
+Settings.resolve!
+
+Pathname.register_paths(
+  project: 'noaa_ish', 
+  universe: 'full',
+
+  orig: [Settings.orig_data_root,'orig'],
+  scratch: [Settings.scratch_data_root, 'scratch'],
+  results: [Settings.results_data_root, 'results'],
+  
+  #Origin
+  noaa_ish_orig: [:orig, 'www1.ncdc.noaa.gov','pub','data','noaa'],
+  noaa_ish_test: [:noaa_ish_orig, '010010-99999-2012'],
+  #Results
+  noaa_ish_results: [:results, :project, :universe],
+)
+
+namespace :extract do
+  desc 'Extract the NOAA ISH weather data from flat files'
+  task :ish do 
+    wukong('extract_ish.rb', :noaa_ish_test, :noaa_ish_results)
+  end
+end

data/examples/munging/weather/extract_ish.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'wukong'
+require 'wukong/streamer/flatpack_streamer'
+
+module Weather
+  class Mapper < Wukong::Streamer::FlatPackStreamer
+    format "_4  i6    i5   s8      s4  sD6e3  D7e3   s5   i5   s5   s4  i3 ssD4e1ii5   ssbi6    sssD5e1 sD5e1 sD5e1 ss*"
+  end
+end
+
+Wukong::Script.new(Weather::Mapper, nil).run

data/examples/munging/weather/models/weather.rb

@@ -0,0 +1,119 @@
+require 'gorillib'
+require 'gorillib/model'
+require 'gorillib/model/serialization'
+require 'gorillib/model/positional_fields'
+
+class RawWeatherReport
+  include Gorillib::Model
+  include Gorillib::Model::PositionalFields
+
+  field :usaf_station_id, Integer
+
+  # wban id appears to have 99999 as a blank value even though 
+  # it is not specified as such in the docs
+  field :wban_station_id, Integer 
+  
+  field :obs_date, String
+  field :obs_time, String
+
+  field :obs_data_source, String, blankish: ["9", '', nil]
+
+  field :wstn_latitude, Float, blankish: [99.999, '', nil]
+  field :wstn_longitude, Float, blankish: [999.999, '' , nil]
+
+  field :report_type_code, String, blankish: ["99999", '', nil]
+    
+  field :wstn_elevation, Integer, blankish: [9999, '', nil]
+  
+  field :wstn_call_letters, String, blankish: ["99999", '', nil]
+  
+  field :quality_control_process_name, String
+ 
+  field :wind_direction, Integer, blankish: [999, '', nil]
+  field :wind_direction_qual, String
+  field :wind_observation_type, String, blankish: ["9", '', nil]
+  field :wind_speed, Float, blankish: [999.9, '', nil]
+  field :wind_speed_qual, String
+  
+  field :ceiling_height, Integer, blankish: [99999, '', nil]
+  field :ceiling_qual, String
+  field :ceiling_determination, String, blankish:['9', '', nil]
+  field :cavok, :boolean
+
+  field :visibility, Integer, blankish: [999999, '', nil]
+  field :visibility_qual, String
+  field :visibility_variability_code, String, blankish: ['9', '', nil]
+  field :visibility_variability_code_qual, String
+
+  field :air_temp, Float, blankish: [999.9, '', nil]
+  field :air_temp_qual, String
+
+  field :dew_point, Float, blankish: [999.9, '', nil]
+  field :dew_point_qual, String
+
+  field :sea_level_pressure, Float, blankish: [9999.9, '' , nil]
+  field :sea_level_pressure_qual, String
+
+  field :raw_extended_observations, String
+end
+
+class ReportMetadata
+  include Gorillib::Model
+  field :wind_direction_qual, String
+  field :wind_speed_qual, String
+  field :ceiling_qual, String
+  field :visibility_qual, String
+  field :visibility_variability_code_qual, String
+  field :air_temp_qual, String
+  field :dew_point_qual, String
+  field :sea_level_pressure_qual, String
+ 
+end
+  
+class WeatherReport
+  include Gorillib::Model
+
+  field :wstn_id, String #wban-usad
+
+  field :wstn_latitude, Float
+  field :wstn_longitude, Float
+  field :wstn_elevation, Float
+
+  field :obs_date, String
+  field :obs_time, String
+ 
+  field :wind_direction, Integer
+  field :wind_observation_type, String
+  field :wind_speed, Float
+  
+  field :ceiling_height, Integer
+  field :ceiling_determination, String
+  field :cavok, :boolean
+
+  field :visibility, Integer
+  field :visibility_variability_code, :boolean
+
+  field :air_temp, Float
+
+  field :dew_point, Float
+
+  field :sea_level_pressure, Float
+
+  field :metadata, ReportMetadata, default: ReportMetadata.new
+
+  def receive!(hsh={})
+    # prune the quality fields
+    hsh.keys.each do |key|
+      next if (key.to_s =~ /[^_]*_qual/).nil?
+      val = hsh.delete(key)
+      metadata.send("receive_#{key.to_s}", val)
+    end
+    # transform the ids
+    if hsh.keys.include? :usaf_station_id and hsh.keys.include? :wban_station_id
+      id = hsh.delete(:usaf_station_id).to_s
+      id += "-#{hsh.delete :wban_station_id}"
+      hsh[:wstn_id] = id
+    end
+    super(hsh)
+  end
+end

data/examples/munging/weather/utils/noaa_downloader.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+# encoding:UTF-8
+
+require 'open-uri'
+require 'configliere'
+
+NOAA_URL = 'http://www1.ncdc.noaa.gov/pub/data/noaa/'
+Settings.use :commandline
+
+Settings({
+  years: [1901],
+  verbose: false,
+  out_dir: /data/rawd/noaa/isd/,
+  un_gzip: false,
+})
+
+Settings.define :years, flag 'y', description: "Years to download"
+Settings.define :verbose, flag 'v', description: "Get chatty", type: :boolean
+Settings.define :un_gzip, flag 'g', description: "Unzip the files as they are uploaded", type: :boolean
+Settings.define :out_dir, flag 'o', description: "The directory in the hdfs to put the files"
+
+Settings.resolve!
+
+def get_files_for_year(year)
+  year_page = open("#{NOAA_URL}/#{year}")
+  years = []
+  year_page.each_line do |line|
+    next unless line =~ /<a href="[^.]*\.gz">/
+    match = /<a href="([^.]*\.gz)">/.match(line)
+    years << match[1] if not match.nil?
+  end
+  return years
+end
+
+years.each do |year|
+  puts "Uploading files for year #{year}..." if Settings[:verbose]
+  get_files_for_year(year).each do |file|
+    puts "  Uploading #{file}..." if Settings[:verbose]
+    path = "#{NOAA_URL}/#{year}/#{file}"
+    if Settings[:un_gzip]
+      `curl '#{path}' | zcat | hdp-put #{Settings[:out_dir]}/#{year}/#{file}`
+    else
+      `curl #{file} | hdp-put #{Settings[:out_dir]}/#{year}/#{file}`
+    end
+  end
+end

data/examples/munging/wikipedia/README.md

@@ -0,0 +1,34 @@
+## Encodings
+All SQL dumps are theoretically encoded in UTF-8, but the Wikipedia dumps contain malformed characters. You might see a 'Invalid UTF-8 byte sequence' error when running a Wukong because of this.
+
+To fix this, use `guard_encoding` in `MungingUtils` to filter out malformed characters before attempting to process them. `guard_encoding` replaces all invalid characters with '�'.
+
+If you need to ensure that all characters are valid UTF-8 when piping things around on the command line, then pipe your stream through `char_filter.rb`.
+
+If you need an invalid UTF-8 character, pretty much any single-byte character above \x79 will do. e.g:
+
+    > char = "\x80"
+    => "\x80"
+    > char.encoding.name
+    => "UTF-8"
+    > char.valid_encoding?
+    => false
+
+[James Gray's blog](http://blog.grayproductions.net/articles/understanding_m17n) is really valuable for further reading on this.
+
+## Dates
+Date information should be formatted as follows:
+
+    +----------+--------+--------------------------+-------------+
+    | int      | int    | long or float            | int         |
+    +----------+--------+--------------------------+-------------+
+    | YYYYMMDD | HHMMSS | Seconds since Unix epoch | Day of week |
+    +----------+--------+--------------------------+-------------+
+
+Should always be in the UTC time zone.
+
+Hours go from 0 to 23
+
+Months go from 01 to 12
+
+Day of week goes from 0 to 6 (Sunday to Saturday)

data/examples/munging/wikipedia/Rakefile

@@ -0,0 +1,193 @@
+require 'configliere'
+Settings.use :commandline
+
+require_relative '../rake_helper'
+
+DUMPS = ['20110722','20110803','20110901','20111007','20111115',
+         '20111201','20120104','20120211','20120307','20120403',
+         '20120502','20120601','20120702','20120802']
+
+Settings.define :pageviews_date_range_slug_in, description: 'The pageviews date range', default: '2012/2012-08'
+Settings.define :pageviews_date_range_slug_out, description: 'The pageviews date range', default: '2012/2012-08'
+Settings.define :dump, description: 'The wikipedia dump to use', default: DUMPS[-1]
+Settings.define :n1_node_id, description: 'Node to construct the N1 subuniverse around', default: '13692155'
+Settings.define :n1_subuniverse, description: 'The output universe for N1 subuniverse generation', finally: ->(c) {c.n1_subuniverse ||= "n1_#{c.n1_node_id}"}
+Settings.define :num_reducers, type: Integer, default: nil
+Settings.resolve!
+
+if (not DUMPS.include? Settings.dump)
+  puts "Invalid dump specified. Must be one of [#{DUMPS.join(', ')}].\nExiting..."
+  exit
+end
+
+=begin
+  Universe is the universe that data is drawn from.
+  It is also the default universe the data is written into.
+  There are tasks (namely subuniverse generation) that do not write out
+  into the supplied universe. Be careful
+=end
+
+Pathname.register_paths(
+  project: 'wikipedia', 
+  universe: [Settings.universe],
+
+  orig: [Settings.orig_data_root,'ripd'],
+  scratch: [Settings.scratch_data_root, 'scratch'],
+  results: [Settings.results_data_root, 'results'],
+  
+  #Origin
+  wiki_dumps: [:orig,'dumps.wikimedia.org'],
+  orig_enwiki: [:wiki_dumps, 'enwiki'],
+  orig_pageviews: [:wiki_dumps, 'other', 'pagecounts-raw', Settings.pageviews_date_range_slug_in],
+  orig_articles: [:orig_enwiki, Settings.dump, "enwiki-#{Settings.dump}-pages-articles.xml.gz"],
+  orig_pages: [:orig_enwiki, Settings.dump, "enwiki-#{Settings.dump}-page.sql.gz"],
+  orig_pagelinks: [:orig_enwiki, Settings.dump,"enwiki-#{Settings.dump}-pagelinks.sql.gz"],
+
+  # Scratch
+  wiki_scratch: [:scratch, :project, :universe],
+  page_metadata_scratch: [:wiki_scratch,'page_metadata'],
+  articles_scratch: [:wiki_scratch, 'articles'],
+  pageviews_scratch: [:wiki_scratch, 'pageviews',Settings.pageviews_date_range_slug_out],
+  pagelinks_scratch: [:wiki_scratch, 'pagelinks'],
+
+  # Results 
+  wiki_results: [:results, :project, :universe],
+  page_metadata_results: [:wiki_results, 'page_metadata'],
+  pageviews_results: [:wiki_results, 'pageviews'],
+  articles_results: [:wiki_results, 'articles'],
+  pagelinks_results: [:wiki_results, 'pagelinks'],
+  undirected_pagelinks_results: [:wiki_results, 'undirected_pagelinks'],
+  redirects_pagelinks_results: [:wiki_results, 'redirects_pagelinks'],
+  redirects_page_metadata_results: [:wiki_results, 'redirects_page_metadata'],
+
+  # N1 Subuniverse
+  n1_results: [:results,'wikipedia', Settings.n1_subuniverse],
+  n1_nodes_results: [:n1_results, 'nodes'],
+  n1_edges_results: [:n1_results, 'edges'],
+  n1_page_metadata_results: [:n1_results, 'page_metadata'],
+  n1_articles_results: [:n1_results, 'articles'],
+  n1_pageviews_results: [:n1_results, 'pageviews'],
+
+)
+
+namespace :utils do
+  desc 'Fetch a list of all Wikipedia namespaces and their IDs'
+  task :get_namespaces do
+    if File.exists 'utils/namespaces.json'
+      puts 'utils/namespaces.json exists... Assuming that namespaces have already been downloaded'
+      return
+    end
+    ruby('utils/get_namespaces.rb')
+  end
+end
+namespace :extract do
+  desc 'Extract the Wikipedia article corpus from bzipped XML files'
+  task :articles do 
+    wukong_xml('articles/extract_articles.rb', :orig_articles, :articles_results)
+  end
+
+  desc 'Extract the Wikipedia pages table from gzipped SQL dumps'
+  task :page_metadata do 
+    wukong('page_metadata/extract_page_metadata.rb', :orig_pages, :page_metadata_results)
+  end
+
+  desc 'Extract Wikipedia pageview data from gzipped server logs'
+  task :pageviews do 
+    if Settings.num_reducers.nil?
+      wukong('pageviews/extract_pageviews.rb', :orig_pageviews, :pageviews_scratch)
+    else
+      wukong('pageviews/extract_pageviews.rb', :orig_pageviews, :pageviews_scratch,{reduce_tasks: Settings.num_reducers})
+    end
+  end
+
+  desc 'Extract Wikipedia pagelinks data from gzipped SQL dumps'
+  task :pagelinks do 
+    wukong('pagelinks/extract_pagelinks.rb', :orig_pagelinks, :pagelinks_scratch)
+  end
+end
+namespace :augment do
+  desc 'Augment extracted Wikipedia pageview data with page ID and other metadata'
+  task :pageviews => ["extract:pageviews", "extract:page_metadata"] do 
+    pig('pageviews/augment_pageviews.pig',{
+      page_metadata: :page_metadata_results, 
+      extracted_pageviews: :pageviews_scratch, 
+      augmented_pageviews_out: :pageviews_results,
+    })
+  end
+
+  desc 'Augment Wikipedia pagelinks data with page metadata'
+  task :pagelinks => ["extract:pagelinks","extract:page_metadata"] do 
+    pig('pagelinks/augment_pagelinks.pig',{
+      page_metadata: :page_metadata_results,
+      extracted_pagelinks: :pagelinks_scratch,
+      augmented_pagelinks_out: :pagelinks_results,
+    })
+  end
+
+  desc 'Undirect the Wikipedia pagelinks graph'
+  task :pagelinks_undirect => "augment:pagelinks" do 
+    pig('pagelinks/undirect_pagelinks.pig',{
+      augmented_pagelinks: :pagelinks_results,
+      undirected_pagelinks_out: :pagelinks_undirected_results,
+    })
+  end
+end
+namespace :n1 do
+  desc 'Generate a list of node ids for the N1 neighborhood of the specified node'
+  task :nodes  => 'augment:pagelinks_undirect' do 
+    pig('n1_subuniverse/n1_nodes.pig',{
+      undirected_pagelinks: :undirected_pagelinks_results,
+      hub: Settings.n1_node_id,
+      n1_nodes_out: :n1_nodes_results,
+    })
+  end
+  desc 'Extract pagelinks for the N1 neighborhood of the specified node'
+  task :undirected_pagelinks => ['augment:pagelinks_undirect', :nodes] do
+    pig('subuniverse/sub_undirected_pagelinks_within.pig',{
+      undirected_pagelinks: :undirected_pagelinks_results,
+      sub_nodes: :n1_nodes_results,
+      sub_pagelinks_out: :n1_edges_results,
+    })
+  end
+  desc 'Extract page metadata for the N1 neighborhood of the specified node'
+  task :page_metadata => ['augment:page_metadata', :nodes] do 
+    pig('subuniverse/sub_page_metadata.pig',{
+      page_metadata: :page_metadata_results,
+      sub_nodes: :n1_nodes_results,
+      sub_page_metadata_out: :n1_page_metadata_results,
+    })
+  end
+  desc 'Extract articles for the N1 neighborhood of the specified node'
+  task :articles => ['extract:articles', :nodes] do 
+    pig('subuniverse/sub_articles.pig',{
+      articles: :articles_results,
+      sub_nodes: :n1_nodes_results,
+      sub_articles_out: :n1_articles_results,
+    })
+  end
+  desc 'Extract pageview data for the N1 neighborhood of the specified node'
+  task :pageviews => ['augment:pageviews', :nodes] do 
+    pig('subuniverse/sub_pageviews.pig',{
+      pageviews: :pageviews_results,
+      sub_nodes: :n1_nodes_results,
+      sub_pageviews_out: :n1_pageviews_results,
+    })
+  end
+  end
+namespace :redirects do
+  desc 'Extract redirects from pagemetadata table'
+  task :redirects_page_metadata => 'extract:page_metadata' do 
+    pig('redirects/redirects_page_metadata.pig',{
+      page_metadata: :page_metadata_results,
+      redirects_out: :redirects_page_metadata_results,
+    })
+  end
+  desc 'Extract redirect links from pagelinks table'
+  task :redirect_pagelinks => ['redirects_page_metadata','augment:pagelinks'] do 
+    pig('subuniverse/sub_pagelinks_from.pig',{
+      pagelinks: :pagelinks_results,
+      sub_nodes: :redirects_page_metadata_results,
+      sub_pagelinks_out: :redirects_pagelinks_results,
+    })
+  end
+end