wukong 1.4.7 → 1.4.9

data/examples/foo.rb

@@ -1,9 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.expand_path("~/ics/backend/configliere/lib")
-
-require "wukong"
-
-p Wukong::Script.new(nil,nil).options
-p Wukong::Script.new(nil,nil).non_wukong_params
-
-Wukong::Script.new(nil,nil).run

data/examples/package-local.rb

@@ -1,100 +0,0 @@
-#!/usr/bin/env ruby
-$: << File.dirname(__FILE__)+'/../lib'
-
-require 'wukong'
-
-#
-# This is so very very kludgey
-#
-# Input is an 'ls' file, listing files to .bz2 package.
-#
-# Mapper takes each in turn and creates, within a parallel directory tree under
-# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
-#
-# So, the file
-#   /user/me/fixd/all-20090103
-# is packaged onto the DFS as
-#   /user/me/pkgd/user/me/fixd/all-20090103
-#
-#   listing=tmp/fixd-all-package-listing
-#   hdp-rm $listing
-#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
-#
-#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
-#
-module ExportPackager
-  PKGD_DIR = '/workspace/flip/pkgd'
-
-  #
-  #
-  class Reducer < Wukong::Streamer::Base
-    def announce *args
-      $stdout.puts *args
-      $stderr.puts *args
-    end
-
-    def handle_existing_target output_filename
-      return true unless File.exist?(output_filename)
-      #   announce "Exists! #{output_filename}"
-      #   return false
-      announce "Removing target file #{output_filename}"
-      begin announce `rm #{output_filename}`
-      rescue Exception => e ; announce e ; end
-      true
-    end
-
-    def mkdir_target_safely output_filename
-      output_dir = File.dirname(output_filename)
-      announce "Ensuring directory #{output_dir} exists"
-      begin announce `mkdir -p #{output_dir}`
-      rescue Exception => e ; announce e ; end
-    end
-
-    def bzip_into_pkgd_file input_filename, output_filename
-      announce "bzip'ing into #{output_filename}"
-      announce `( hadoop dfs -cat #{input_filename}/[^_]\** ) | bzip2 -c > #{output_filename}`
-    end
-
-    def gen_output_filename input_filename
-      input_filename += '.tsv' unless input_filename =~ /.*\.\w{2,}/
-      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(/^\//, '')]
-    end
-
-    def rsync host, local_path, remote_path=nil
-      remote_path ||= local_path
-      announce `/usr/bin/rsync -Cuvrtlp #{local_path} #{host}:#{remote_path}`
-      sleep 5
-    end
-
-    def process input_filename
-      output_filename = gen_output_filename(input_filename)
-      handle_existing_target(output_filename) or return
-      mkdir_target_safely    output_filename
-      bzip_into_pkgd_file    input_filename, output_filename
-      rsync :lab3, output_filename
-      #
-    end
-
-    def recordize line
-      # handle ls or straight file list, either
-      line.split(/\s/).last
-    end
-
-    def stream
-      super
-      rsync :lab3, PKGD_DIR+'/'
-    end
-  end
-
-  class Script < Wukong::Script
-    def default_options
-      super.merge :map_tasks => 1,
-        :max_node_reduce_tasks => 1, # only one reducer per local filesystem
-        :timeout => 40 * 60 * 1000   # timeout in ms
-    end
-  end
-  # Execute the script
-  Script.new(nil, Reducer).run
-end
-
-

data/examples/package.rb

@@ -1,96 +0,0 @@
-#!/usr/bin/env ruby
-$: << ENV['WUKONG_PATH'] if ENV['WUKONG_PATH']
-require 'wukong'
-
-#
-# This is so very very kludgey
-#
-# Input is an 'ls' file, listing files to .bz2 package.
-#
-# Reducer takes each in turn and creates, within a parallel directory tree under
-# ~/pkgd on the HDFS, a .bz2 compressed version of the file.
-#
-# So, the file
-#   /user/me/fixd/all-20090103
-# is packaged onto the DFS as
-#   /user/me/pkgd/user/me/fixd/all-20090103
-#
-#   listing=tmp/fixd-all-package-listing
-#   hdp-rm $listing
-#   hadoop dfs -lsr fixd | egrep '(part-|\.tsv$)' | hdp-put - $listing ;
-#
-#   ./package.rb --run --rm --map_tasks=1 $listing $pkgd_log
-#
-module ExportPackager
-  PKGD_DIR = 'pkgd'
-
-  #
-  #
-  class Reducer < Wukong::Streamer::Base
-    def announce str
-      return if str.blank?
-      $stderr.puts str
-      $stdout.puts str
-    end
-
-    def remove_target_filename output_filename
-      begin announce "rm\t#{"%-70s"%output_filename}\t" +
-          `( hadoop dfs -rmr #{output_filename} ) 2>&1`
-      rescue ; nil ; end
-    end
-
-    def mkdir_target_safely output_filename
-      output_dir = File.dirname(output_filename)
-      begin announce "mkdir\t#{"%-70s"%output_dir}\t" +
-          `( hadoop dfs -mkdir #{output_dir} ) 2>&1`
-      rescue ; nil ; end
-    end
-
-    def bzip_into_pkgd_file input_filename, output_filename
-      announce "cat|bz\t#{"%-70s"%input_filename}\t" +
-        `( hadoop dfs -cat #{input_filename}/[^_]\\* | bzip2 -c | hadoop dfs -put - #{output_filename} ) 2>&1`
-    end
-
-    def verify input_filename, output_filename
-      announce "sha1sum\t#{"%-70s"%output_filename}\t" +
-        `( hadoop dfs -cat #{output_filename}        | bzcat - | sha1sum ) 2>&1`
-      announce "sha1sum\t#{"%-70s"%input_filename}\t" +
-        `( hadoop dfs -cat #{input_filename}/[^_]\\*           | sha1sum ) 2>&1`
-    end
-
-    def gen_output_filename input_filename
-      "%s/%s.bz2" % [PKGD_DIR, input_filename.gsub(%r{^/},"")]
-    end
-
-    def process input_filename, output_filename
-      # remove_target_filename output_filename
-      # mkdir_target_safely    output_filename
-      bzip_into_pkgd_file    input_filename, output_filename
-      verify                 input_filename, output_filename
-    end
-
-    def stream
-      announce `hostname`
-      $stdin.each do |input_filename|
-        # handle ls or straight file list, either
-        input_filename = input_filename.chomp.strip.split(/\s/).last
-        output_filename = gen_output_filename input_filename
-        announce "********************************************************"
-        announce "Packing\t#{"%-70s"%input_filename}\t#{output_filename}"
-        process input_filename, output_filename
-        announce "Done\t#{"%-70s"%input_filename}\t#{output_filename}\n\n"
-      end
-    end
-  end
-
-  class Script < Wukong::Script
-    def default_options
-      super.merge :timeout => (24 * 60 * 60 * 1000)  # milliseconds in one day
-    end
-  end
-end
-
-#
-# Execute the script
-#
-ExportPackager::Script.new(nil, ExportPackager::Reducer, :reduce_tasks => 1000).run

data/examples/run_all.sh

@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-src_path="tmp/README.textile"
-out_root="tmp/test"
-hdp_opts="--map_tasks=1 --reduce_tasks=1"
-
-# ---------------------------------------------------------------------------
-#
-# Set up directories and copy over sample input
-#
-
-# hdp-rm ${src_path}
-# hdp-put `dirname $0`/../README.textile tmp/
-# hdp-mkdir $out_root 
-
-# ---------------------------------------------------------------------------
-#
-# Run scripts
-#
-
-cmd="word_count"
-    # hdp-rm -r ${out_root}/${cmd}
-    # ./examples/${cmd}.rb  	--run $hdp_opts $src_path ${out_root}/${cmd}
-    # hdp-catd ${out_root}/${cmd} | head -n 20
-    word_count=${out_root}/${cmd}
-
-cmd="sample_records"
-    # hdp-rm -r ${out_root}/${cmd}
-    # ./examples/${cmd}.rb  	--sampling_fraction=0.8 \
-    #     			--run $hdp_opts $src_path ${out_root}/${cmd}
-    # hdp-catd ${out_root}/${cmd} | head -n 200 | tail -n 20
-    sample_records=${out_root}/${cmd}
-
-
-# cmd="size"
-#     hdp-rm -r ${out_root}/${cmd}
-#     ./examples/${cmd}.rb  	--run $hdp_opts $src_path ${out_root}/${cmd}
-#     hdp-catd ${out_root}/${cmd} 
-#     size=${out_root}/${cmd}
-
-
-cmd="count_keys"
-    hdp-rm -r ${out_root}/${cmd}
-    ./examples/${cmd}.rb  	--run $hdp_opts $word_count ${out_root}/${cmd}
-    hdp-catd ${out_root}/${cmd}  | head -n 200 | tail -n 20
-    count_keys=${out_root}/${cmd}
-