wukong 3.0.0.pre2 → 3.0.0.pre3

data/examples/munging/rake_helper.rb

@@ -1,62 +0,0 @@
-require 'gorillib'
-require 'gorillib/data_munging'
-require 'configliere'
-
-S3_BUCKET = 'bigdata.chimpy.us'
-S3_DATA_ROOT = "s3n://#{S3_BUCKET}/data"
-HDFS_DATA_ROOT = '/data'
-
-Settings.define :orig_data_root, default: HDFS_DATA_ROOT, description: "directory root for input data"
-Settings.define :scratch_data_root, default: HDFS_DATA_ROOT, description: "directory root for scratch data"
-Settings.define :results_data_root, default: HDFS_DATA_ROOT, description: "directory root for results data"
-Settings.define :mini, description: 'Run in mini mode - operate inside the mini version of the specified universe',type: :boolean, default: false
-Settings.define :universe, description: 'Universe to draw data from', finally: ->(c){ c.universe ||= (c.mini? ? "mini" : "full") }
-Settings.define :pig_path, default: '/usr/local/bin/pig'
-Settings.define :local, type: :boolean, default: false
-
-def Settings.mini?; !! Settings.mini ; end # BANG BANG BANG
-def Settings.wu_run_cmd; (local ? '--run=local' : '--run') ; end;
-
-def dir_exists? (dir)
-  if Settings.local
-    return File.exists? dir
-  else
-    `hadoop fs -test -e #{dir}`
-    return $?.exitstatus == 0
-  end
-end
-
-def wukong(script, input, output, options={})
-  input = Pathname.of(input)
-  output = Pathname.of(output)
-  if dir_exists? output
-    puts "#{output} exists. Assuming that this job has already run..."
-    return
-  end
-  opts = ['--rm']
-  options.each_pair do |k,v|
-    opts << "--#{k}=#{v}"
-  end
-  opts << input
-  opts << output
-  ruby(script, Settings.wu_run_cmd,*opts)
-end
-
-def wukong_xml(script, input, output, split_tag)
-  wukong(script,input,output,{split_on_xml_tag: split_tag})
-end
-
-def pig(script_name, options={})
-  cmd = Settings.pig_path
-  options.each_pair do |k,v|
-    v = Pathname.of(v) if v.is_a? Symbol
-    if k.to_s.include? '_out' and dir_exists? v
-      puts "#{v} already exists. Assuming that this job has already run..."
-      return
-    else
-      cmd += " -param #{k.upcase}=#{v}"
-    end
-  end
-  cmd += " #{script_name}"
-  sh cmd
-end

data/examples/munging/weather/.gitignore

@@ -1 +0,0 @@
-Gemfile.lock

data/examples/munging/weather/Gemfile

@@ -1,4 +0,0 @@
-source 'http://rubygems.org'
-  
-gem 'gorillib',    :path => '/Users/dlaw/dev/gorillib'
-gem 'wukong', :path =>'/Users/dlaw/dev/wukong_og'

data/examples/munging/weather/Rakefile

@@ -1,28 +0,0 @@
-require 'configliere'
-Settings.use :commandline
-
-require_relative '../rake_helper'
-
-Settings.resolve!
-
-Pathname.register_paths(
-  project: 'noaa_ish', 
-  universe: 'full',
-
-  orig: [Settings.orig_data_root,'orig'],
-  scratch: [Settings.scratch_data_root, 'scratch'],
-  results: [Settings.results_data_root, 'results'],
-  
-  #Origin
-  noaa_ish_orig: [:orig, 'www1.ncdc.noaa.gov','pub','data','noaa'],
-  noaa_ish_test: [:noaa_ish_orig, '010010-99999-2012'],
-  #Results
-  noaa_ish_results: [:results, :project, :universe],
-)
-
-namespace :extract do
-  desc 'Extract the NOAA ISH weather data from flat files'
-  task :ish do 
-    wukong('extract_ish.rb', :noaa_ish_test, :noaa_ish_results)
-  end
-end

data/examples/munging/weather/extract_ish.rb

@@ -1,13 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: UTF-8
-
-require 'wukong'
-require 'wukong/streamer/flatpack_streamer'
-
-module Weather
-  class Mapper < Wukong::Streamer::FlatPackStreamer
-    format "_4  i6    i5   s8      s4  sD6e3  D7e3   s5   i5   s5   s4  i3 ssD4e1ii5   ssbi6    sssD5e1 sD5e1 sD5e1 ss*"
-  end
-end
-
-Wukong::Script.new(Weather::Mapper, nil).run

data/examples/munging/weather/models/weather.rb

@@ -1,119 +0,0 @@
-require 'gorillib'
-require 'gorillib/model'
-require 'gorillib/model/serialization'
-require 'gorillib/model/positional_fields'
-
-class RawWeatherReport
-  include Gorillib::Model
-  include Gorillib::Model::PositionalFields
-
-  field :usaf_station_id, Integer
-
-  # wban id appears to have 99999 as a blank value even though 
-  # it is not specified as such in the docs
-  field :wban_station_id, Integer 
-  
-  field :obs_date, String
-  field :obs_time, String
-
-  field :obs_data_source, String, blankish: ["9", '', nil]
-
-  field :wstn_latitude, Float, blankish: [99.999, '', nil]
-  field :wstn_longitude, Float, blankish: [999.999, '' , nil]
-
-  field :report_type_code, String, blankish: ["99999", '', nil]
-    
-  field :wstn_elevation, Integer, blankish: [9999, '', nil]
-  
-  field :wstn_call_letters, String, blankish: ["99999", '', nil]
-  
-  field :quality_control_process_name, String
- 
-  field :wind_direction, Integer, blankish: [999, '', nil]
-  field :wind_direction_qual, String
-  field :wind_observation_type, String, blankish: ["9", '', nil]
-  field :wind_speed, Float, blankish: [999.9, '', nil]
-  field :wind_speed_qual, String
-  
-  field :ceiling_height, Integer, blankish: [99999, '', nil]
-  field :ceiling_qual, String
-  field :ceiling_determination, String, blankish:['9', '', nil]
-  field :cavok, :boolean
-
-  field :visibility, Integer, blankish: [999999, '', nil]
-  field :visibility_qual, String
-  field :visibility_variability_code, String, blankish: ['9', '', nil]
-  field :visibility_variability_code_qual, String
-
-  field :air_temp, Float, blankish: [999.9, '', nil]
-  field :air_temp_qual, String
-
-  field :dew_point, Float, blankish: [999.9, '', nil]
-  field :dew_point_qual, String
-
-  field :sea_level_pressure, Float, blankish: [9999.9, '' , nil]
-  field :sea_level_pressure_qual, String
-
-  field :raw_extended_observations, String
-end
-
-class ReportMetadata
-  include Gorillib::Model
-  field :wind_direction_qual, String
-  field :wind_speed_qual, String
-  field :ceiling_qual, String
-  field :visibility_qual, String
-  field :visibility_variability_code_qual, String
-  field :air_temp_qual, String
-  field :dew_point_qual, String
-  field :sea_level_pressure_qual, String
- 
-end
-  
-class WeatherReport
-  include Gorillib::Model
-
-  field :wstn_id, String #wban-usad
-
-  field :wstn_latitude, Float
-  field :wstn_longitude, Float
-  field :wstn_elevation, Float
-
-  field :obs_date, String
-  field :obs_time, String
- 
-  field :wind_direction, Integer
-  field :wind_observation_type, String
-  field :wind_speed, Float
-  
-  field :ceiling_height, Integer
-  field :ceiling_determination, String
-  field :cavok, :boolean
-
-  field :visibility, Integer
-  field :visibility_variability_code, :boolean
-
-  field :air_temp, Float
-
-  field :dew_point, Float
-
-  field :sea_level_pressure, Float
-
-  field :metadata, ReportMetadata, default: ReportMetadata.new
-
-  def receive!(hsh={})
-    # prune the quality fields
-    hsh.keys.each do |key|
-      next if (key.to_s =~ /[^_]*_qual/).nil?
-      val = hsh.delete(key)
-      metadata.send("receive_#{key.to_s}", val)
-    end
-    # transform the ids
-    if hsh.keys.include? :usaf_station_id and hsh.keys.include? :wban_station_id
-      id = hsh.delete(:usaf_station_id).to_s
-      id += "-#{hsh.delete :wban_station_id}"
-      hsh[:wstn_id] = id
-    end
-    super(hsh)
-  end
-end

data/examples/munging/weather/utils/noaa_downloader.rb

@@ -1,46 +0,0 @@
-#!/usr/bin/env ruby
-# encoding:UTF-8
-
-require 'open-uri'
-require 'configliere'
-
-NOAA_URL = 'http://www1.ncdc.noaa.gov/pub/data/noaa/'
-Settings.use :commandline
-
-Settings({
-  years: [1901],
-  verbose: false,
-  out_dir: /data/rawd/noaa/isd/,
-  un_gzip: false,
-})
-
-Settings.define :years, flag 'y', description: "Years to download"
-Settings.define :verbose, flag 'v', description: "Get chatty", type: :boolean
-Settings.define :un_gzip, flag 'g', description: "Unzip the files as they are uploaded", type: :boolean
-Settings.define :out_dir, flag 'o', description: "The directory in the hdfs to put the files"
-
-Settings.resolve!
-
-def get_files_for_year(year)
-  year_page = open("#{NOAA_URL}/#{year}")
-  years = []
-  year_page.each_line do |line|
-    next unless line =~ /<a href="[^.]*\.gz">/
-    match = /<a href="([^.]*\.gz)">/.match(line)
-    years << match[1] if not match.nil?
-  end
-  return years
-end
-
-years.each do |year|
-  puts "Uploading files for year #{year}..." if Settings[:verbose]
-  get_files_for_year(year).each do |file|
-    puts "  Uploading #{file}..." if Settings[:verbose]
-    path = "#{NOAA_URL}/#{year}/#{file}"
-    if Settings[:un_gzip]
-      `curl '#{path}' | zcat | hdp-put #{Settings[:out_dir]}/#{year}/#{file}`
-    else
-      `curl #{file} | hdp-put #{Settings[:out_dir]}/#{year}/#{file}`
-    end
-  end
-end

data/examples/munging/wikipedia/README.md

@@ -1,34 +0,0 @@
-## Encodings
-All SQL dumps are theoretically encoded in UTF-8, but the Wikipedia dumps contain malformed characters. You might see a 'Invalid UTF-8 byte sequence' error when running a Wukong because of this.
-
-To fix this, use `guard_encoding` in `MungingUtils` to filter out malformed characters before attempting to process them. `guard_encoding` replaces all invalid characters with '�'.
-
-If you need to ensure that all characters are valid UTF-8 when piping things around on the command line, then pipe your stream through `char_filter.rb`.
-
-If you need an invalid UTF-8 character, pretty much any single-byte character above \x79 will do. e.g:
-
-    > char = "\x80"
-    => "\x80"
-    > char.encoding.name
-    => "UTF-8"
-    > char.valid_encoding?
-    => false
-
-[James Gray's blog](http://blog.grayproductions.net/articles/understanding_m17n) is really valuable for further reading on this.
-
-## Dates
-Date information should be formatted as follows:
-
-    +----------+--------+--------------------------+-------------+
-    | int      | int    | long or float            | int         |
-    +----------+--------+--------------------------+-------------+
-    | YYYYMMDD | HHMMSS | Seconds since Unix epoch | Day of week |
-    +----------+--------+--------------------------+-------------+
-
-Should always be in the UTC time zone.
-
-Hours go from 0 to 23
-
-Months go from 01 to 12
-
-Day of week goes from 0 to 6 (Sunday to Saturday)

data/examples/munging/wikipedia/Rakefile

@@ -1,193 +0,0 @@
-require 'configliere'
-Settings.use :commandline
-
-require_relative '../rake_helper'
-
-DUMPS = ['20110722','20110803','20110901','20111007','20111115',
-         '20111201','20120104','20120211','20120307','20120403',
-         '20120502','20120601','20120702','20120802']
-
-Settings.define :pageviews_date_range_slug_in, description: 'The pageviews date range', default: '2012/2012-08'
-Settings.define :pageviews_date_range_slug_out, description: 'The pageviews date range', default: '2012/2012-08'
-Settings.define :dump, description: 'The wikipedia dump to use', default: DUMPS[-1]
-Settings.define :n1_node_id, description: 'Node to construct the N1 subuniverse around', default: '13692155'
-Settings.define :n1_subuniverse, description: 'The output universe for N1 subuniverse generation', finally: ->(c) {c.n1_subuniverse ||= "n1_#{c.n1_node_id}"}
-Settings.define :num_reducers, type: Integer, default: nil
-Settings.resolve!
-
-if (not DUMPS.include? Settings.dump)
-  puts "Invalid dump specified. Must be one of [#{DUMPS.join(', ')}].\nExiting..."
-  exit
-end
-
-=begin
-  Universe is the universe that data is drawn from.
-  It is also the default universe the data is written into.
-  There are tasks (namely subuniverse generation) that do not write out
-  into the supplied universe. Be careful
-=end
-
-Pathname.register_paths(
-  project: 'wikipedia', 
-  universe: [Settings.universe],
-
-  orig: [Settings.orig_data_root,'ripd'],
-  scratch: [Settings.scratch_data_root, 'scratch'],
-  results: [Settings.results_data_root, 'results'],
-  
-  #Origin
-  wiki_dumps: [:orig,'dumps.wikimedia.org'],
-  orig_enwiki: [:wiki_dumps, 'enwiki'],
-  orig_pageviews: [:wiki_dumps, 'other', 'pagecounts-raw', Settings.pageviews_date_range_slug_in],
-  orig_articles: [:orig_enwiki, Settings.dump, "enwiki-#{Settings.dump}-pages-articles.xml.gz"],
-  orig_pages: [:orig_enwiki, Settings.dump, "enwiki-#{Settings.dump}-page.sql.gz"],
-  orig_pagelinks: [:orig_enwiki, Settings.dump,"enwiki-#{Settings.dump}-pagelinks.sql.gz"],
-
-  # Scratch
-  wiki_scratch: [:scratch, :project, :universe],
-  page_metadata_scratch: [:wiki_scratch,'page_metadata'],
-  articles_scratch: [:wiki_scratch, 'articles'],
-  pageviews_scratch: [:wiki_scratch, 'pageviews',Settings.pageviews_date_range_slug_out],
-  pagelinks_scratch: [:wiki_scratch, 'pagelinks'],
-
-  # Results 
-  wiki_results: [:results, :project, :universe],
-  page_metadata_results: [:wiki_results, 'page_metadata'],
-  pageviews_results: [:wiki_results, 'pageviews'],
-  articles_results: [:wiki_results, 'articles'],
-  pagelinks_results: [:wiki_results, 'pagelinks'],
-  undirected_pagelinks_results: [:wiki_results, 'undirected_pagelinks'],
-  redirects_pagelinks_results: [:wiki_results, 'redirects_pagelinks'],
-  redirects_page_metadata_results: [:wiki_results, 'redirects_page_metadata'],
-
-  # N1 Subuniverse
-  n1_results: [:results,'wikipedia', Settings.n1_subuniverse],
-  n1_nodes_results: [:n1_results, 'nodes'],
-  n1_edges_results: [:n1_results, 'edges'],
-  n1_page_metadata_results: [:n1_results, 'page_metadata'],
-  n1_articles_results: [:n1_results, 'articles'],
-  n1_pageviews_results: [:n1_results, 'pageviews'],
-
-)
-
-namespace :utils do
-  desc 'Fetch a list of all Wikipedia namespaces and their IDs'
-  task :get_namespaces do
-    if File.exists 'utils/namespaces.json'
-      puts 'utils/namespaces.json exists... Assuming that namespaces have already been downloaded'
-      return
-    end
-    ruby('utils/get_namespaces.rb')
-  end
-end
-namespace :extract do
-  desc 'Extract the Wikipedia article corpus from bzipped XML files'
-  task :articles do 
-    wukong_xml('articles/extract_articles.rb', :orig_articles, :articles_results)
-  end
-
-  desc 'Extract the Wikipedia pages table from gzipped SQL dumps'
-  task :page_metadata do 
-    wukong('page_metadata/extract_page_metadata.rb', :orig_pages, :page_metadata_results)
-  end
-
-  desc 'Extract Wikipedia pageview data from gzipped server logs'
-  task :pageviews do 
-    if Settings.num_reducers.nil?
-      wukong('pageviews/extract_pageviews.rb', :orig_pageviews, :pageviews_scratch)
-    else
-      wukong('pageviews/extract_pageviews.rb', :orig_pageviews, :pageviews_scratch,{reduce_tasks: Settings.num_reducers})
-    end
-  end
-
-  desc 'Extract Wikipedia pagelinks data from gzipped SQL dumps'
-  task :pagelinks do 
-    wukong('pagelinks/extract_pagelinks.rb', :orig_pagelinks, :pagelinks_scratch)
-  end
-end
-namespace :augment do
-  desc 'Augment extracted Wikipedia pageview data with page ID and other metadata'
-  task :pageviews => ["extract:pageviews", "extract:page_metadata"] do 
-    pig('pageviews/augment_pageviews.pig',{
-      page_metadata: :page_metadata_results, 
-      extracted_pageviews: :pageviews_scratch, 
-      augmented_pageviews_out: :pageviews_results,
-    })
-  end
-
-  desc 'Augment Wikipedia pagelinks data with page metadata'
-  task :pagelinks => ["extract:pagelinks","extract:page_metadata"] do 
-    pig('pagelinks/augment_pagelinks.pig',{
-      page_metadata: :page_metadata_results,
-      extracted_pagelinks: :pagelinks_scratch,
-      augmented_pagelinks_out: :pagelinks_results,
-    })
-  end
-
-  desc 'Undirect the Wikipedia pagelinks graph'
-  task :pagelinks_undirect => "augment:pagelinks" do 
-    pig('pagelinks/undirect_pagelinks.pig',{
-      augmented_pagelinks: :pagelinks_results,
-      undirected_pagelinks_out: :pagelinks_undirected_results,
-    })
-  end
-end
-namespace :n1 do
-  desc 'Generate a list of node ids for the N1 neighborhood of the specified node'
-  task :nodes  => 'augment:pagelinks_undirect' do 
-    pig('n1_subuniverse/n1_nodes.pig',{
-      undirected_pagelinks: :undirected_pagelinks_results,
-      hub: Settings.n1_node_id,
-      n1_nodes_out: :n1_nodes_results,
-    })
-  end
-  desc 'Extract pagelinks for the N1 neighborhood of the specified node'
-  task :undirected_pagelinks => ['augment:pagelinks_undirect', :nodes] do
-    pig('subuniverse/sub_undirected_pagelinks_within.pig',{
-      undirected_pagelinks: :undirected_pagelinks_results,
-      sub_nodes: :n1_nodes_results,
-      sub_pagelinks_out: :n1_edges_results,
-    })
-  end
-  desc 'Extract page metadata for the N1 neighborhood of the specified node'
-  task :page_metadata => ['augment:page_metadata', :nodes] do 
-    pig('subuniverse/sub_page_metadata.pig',{
-      page_metadata: :page_metadata_results,
-      sub_nodes: :n1_nodes_results,
-      sub_page_metadata_out: :n1_page_metadata_results,
-    })
-  end
-  desc 'Extract articles for the N1 neighborhood of the specified node'
-  task :articles => ['extract:articles', :nodes] do 
-    pig('subuniverse/sub_articles.pig',{
-      articles: :articles_results,
-      sub_nodes: :n1_nodes_results,
-      sub_articles_out: :n1_articles_results,
-    })
-  end
-  desc 'Extract pageview data for the N1 neighborhood of the specified node'
-  task :pageviews => ['augment:pageviews', :nodes] do 
-    pig('subuniverse/sub_pageviews.pig',{
-      pageviews: :pageviews_results,
-      sub_nodes: :n1_nodes_results,
-      sub_pageviews_out: :n1_pageviews_results,
-    })
-  end
-  end
-namespace :redirects do
-  desc 'Extract redirects from pagemetadata table'
-  task :redirects_page_metadata => 'extract:page_metadata' do 
-    pig('redirects/redirects_page_metadata.pig',{
-      page_metadata: :page_metadata_results,
-      redirects_out: :redirects_page_metadata_results,
-    })
-  end
-  desc 'Extract redirect links from pagelinks table'
-  task :redirect_pagelinks => ['redirects_page_metadata','augment:pagelinks'] do 
-    pig('subuniverse/sub_pagelinks_from.pig',{
-      pagelinks: :pagelinks_results,
-      sub_nodes: :redirects_page_metadata_results,
-      sub_pagelinks_out: :redirects_pagelinks_results,
-    })
-  end
-end