mrflip-wukong 0.1.0

data/bin/cutc

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+#
+# cut 1
+#
+# Example:
+# 
+# A quickie histogram of timestamps; say that for the object in the foo/bar
+# directory, field 3 holds a flat timestamp (YYYYmmddHHMMSS) and you want a
+# histogram by hour (and that foo/bar is small enough to be worth sucking
+# through a single machine):
+#
+#   hdp-catd foo/bar | cuttab 3 | cutc 12 | sort | uniq -c
+#
+# If foo/bar is already sorted leave out the call to sort.
+# 
+
+
+#
+# Set it to cut up to $1 (if defined), or if not, up to $CUTC_MAX (if defined), or 200 chars as a fallback.
+#
+CUTC_MAX=${CUTC_MAX-200}
+CUTC_MAX=${1-$CUTC_MAX}
+cutchars="1-${CUTC_MAX}"
+shift
+
+#
+# Do the cuttin'
+#
+cut -c"${cutchars}" "$@"

data/bin/cuttab

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+fields=${1-"1-"}
+shift
+cut  -d'	' -f"$fields" "$@"

data/bin/greptrue

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# runs grep but always returns a true exit status. (Otherwise hadoop vomits)
+grep "$@"
+true
+# runs grep but always returns a true exit status. (Otherwise hadoop vomits)
+egrep "$@"
+true

data/bin/hdp-cat

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop dfs -cat "$@"

data/bin/hdp-catd

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+args=`echo "$@" | ruby -ne 'a = $_.split(/\s+/); puts a.map{|arg| arg+"/[^_]*" }.join(" ")'`
+hadoop dfs -cat $args

data/bin/hdp-du

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+OPTIONS={}
+
+#
+# grok options
+#
+if ARGV[0] =~ /-[a-z]+/
+  flags = ARGV.shift
+  OPTIONS[:summary]  = flags.include?('s')
+  OPTIONS[:humanize] = flags.include?('h')
+end
+
+#
+# Prepare command
+#
+def prepare_command
+  dfs_cmd  = OPTIONS[:summary] ? 'dus' : 'du'
+  dfs_args = "'" + ARGV.join("' '") + "'"
+  %Q{ hadoop dfs -#{dfs_cmd} #{dfs_args} }
+end
+
+Numeric.class_eval do
+  def bytes()       self                    ; end
+  alias :byte :bytes
+  def kilobytes()   self * 1024             ; end
+  alias :kilobyte :kilobytes
+  def megabytes()   self * 1024.kilobytes   ; end
+  alias :megabyte :megabytes
+  def gigabytes()   self * 1024.megabytes   ; end
+  alias :gigabyte :gigabytes
+  def terabytes()   self * 1024.gigabytes   ; end
+  alias :terabyte :terabytes
+  def petabytes()   self * 1024.terabytes   ; end
+  alias :petabyte :petabytes
+  def exabytes()    self * 1024.petabytes   ; end
+  alias :exabyte :exabytes
+end
+
+# Formats the bytes in +size+ into a more understandable representation
+# (e.g., giving it 1500 yields 1.5 KB). This method is useful for
+# reporting file sizes to users. This method returns nil if
+# +size+ cannot be converted into a number. You can change the default
+# precision of 1 using the precision parameter +precision+.
+#
+# ==== Examples
+#  number_to_human_size(123)           # => 123 Bytes
+#  number_to_human_size(1234)          # => 1.2 KB
+#  number_to_human_size(12345)         # => 12.1 KB
+#  number_to_human_size(1234567)       # => 1.2 MB
+#  number_to_human_size(1234567890)    # => 1.1 GB
+#  number_to_human_size(1234567890123) # => 1.1 TB
+#  number_to_human_size(1234567, 2)    # => 1.18 MB
+#  number_to_human_size(483989, 0)     # => 4 MB
+def number_to_human_size(size, precision=1)
+  size = Kernel.Float(size)
+  case
+  when size.to_i == 1;    "1 Byte"
+  when size < 1.kilobyte; "%d Bytes" % size
+  when size < 1.megabyte; "%.#{precision}f KB"  % (size / 1.0.kilobyte)
+  when size < 1.gigabyte; "%.#{precision}f MB"  % (size / 1.0.megabyte)
+  when size < 1.terabyte; "%.#{precision}f GB"  % (size / 1.0.gigabyte)
+  else                    "%.#{precision}f TB"  % (size / 1.0.terabyte)
+  end.sub(/([0-9]\.\d*?)0+ /, '\1 ' ).sub(/\. /,' ')
+rescue
+  nil
+end
+
+def format_output file, size
+  human_size = number_to_human_size(size) || 3
+  file = file.gsub(%r{hdfs://[^/]+/}, '/') # kill off hdfs paths, otherwise leave it alone
+  "%-71s\t%15d\t%15s" % [file, size.to_i, human_size]
+end
+
+
+%x{ #{prepare_command} }.split("\n").each do |line|
+  if line =~ /^Found \d+ items$/ then puts line ; next end
+  info = line.split(/\s+/)
+  if OPTIONS[:summary] then file, size = info else size, file = info end
+  puts format_output(file, size)
+end

data/bin/hdp-get

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop dfs -copyToLocal "$1" "$2"

data/bin/hdp-kill

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop job -kill "$@"

data/bin/hdp-ls

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [ "$1" == "-r" ] || [ "$1" == "-R" ] ; then
+    shift
+    action=lsr
+else
+    action=ls
+fi
+
+hadoop dfs -$action "$@"

data/bin/hdp-mkdir

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop dfs -mkdir "$@"

data/bin/hdp-mv

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop dfs -mv "$@"

data/bin/hdp-parts_to_keys.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+dir_to_rename = ARGV[0]
+dest_ext = '.tsv'
+
+unless dir_to_rename && (! dir_to_rename.empty?)
+  warn "Need a directory or file spec to rename."
+  exit
+end
+
+#
+# Setup
+#
+warn "\nPlease IGNORE the 'cat: Unable to write to output stream.' errors\n"
+
+#
+# Examine the files
+#
+file_listings = `hdp-ls #{dir_to_rename}`.split("\n")
+command_lists = { }
+file_listings[1..-1].each do |file_listing|
+  m = %r{[-drwx]+\s+[\-\d]+\s+\w+\s+\w+\s+(\d+)\s+[\d\-]+\s+[\d\:]+\s+(.+)$}.match(file_listing)
+  if !m then warn "Couldn't grok #{file_listing}" ; next ; end
+  size, filename = m.captures
+  case
+  when size.to_i == 0 then (command_lists[:deletes]||=[]) << filename
+  else
+    firstline = `hdp-cat #{filename} | head -qn1 `
+    file_key, _ = firstline.split("\t", 2)
+    unless file_key && (file_key =~ /\A[\w\-\.]+\z/)
+      warn "Don't want to rename to '#{file_key}'... skipping"
+      next
+    end
+    dirname = File.dirname(filename)
+    destfile = File.join(dirname, file_key)+dest_ext
+    (command_lists[:moves]||=[]) << "hdp-mv #{filename} #{destfile}"
+  end
+end
+
+#
+# Execute the command_lists
+#
+command_lists.each do |type, command_list|
+  case type
+  when :deletes
+    command = "hdp-rm #{command_list.join(" ")}"
+    puts command
+    `#{command}`
+  when :moves
+    command_list.each do |command|
+      puts command
+      `#{command}`
+    end
+  end
+end
+
+
+# -rw-r--r--   3 flip supergroup          0 2008-12-20 05:51 /user/flip/out/sorted-tweets-20081220/part-00010
+
+# # Killing empty files
+# find . -size 0 -print -exec rm {} \;
+#
+# for foo in part-0* ; do
+#   newname=`
+#     head -n1 $foo |
+#     cut -d'   ' -f1 |
+#     ruby -ne 'puts $_.chomp.gsub(/[^\-\w]/){|s| s.bytes.map{|c| "%%%02X" % c }}'
+#     `.tsv ;
+#   echo "moving $foo to $newname"
+#   mv "$foo" "$newname"
+# done
+#
+# # dir=`basename $PWD`
+# # for foo in *.tsv ; do
+# #   echo "Compressing $dir"
+# #   bzip2 -c $foo > ../$dir-bz2/$foo.bz2
+# # done

data/bin/hdp-ps

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop job -list all

data/bin/hdp-put

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+hadoop dfs -put "$1" "$2"

data/bin/hdp-rm

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [ "$1" == "-r" ] ; then
+    shift
+    action=rmr
+else
+    action=rm
+fi
+echo hadoop dfs -$action "$@"
+# read -p "Hit ctrl-C to abort or enter to do this...."
+hadoop dfs -$action "$@"

data/bin/hdp-sort

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# hadoop dfs -rmr out/parsed-followers
+
+input_file=${1} 		; shift
+output_file=${1} 		; shift
+map_script=${1-/bin/cat}	; shift
+reduce_script=${1-/usr/bin/uniq}	; shift
+fields=${1-2} 			; shift
+
+if [ "$reduce_script" == "" ] ; then echo "$0 input_file output_file [sort_fields] [mapper] [reducer] [args]" ; exit ; fi
+
+HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
+
+${HADOOP_HOME}/bin/hadoop \
+     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar		\
+    -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner 			\
+    -jobconf     map.output.key.field.separator='\t'					\
+    -jobconf     num.key.fields.for.partition=1 					\
+    -jobconf 	 stream.map.output.field.separator='\t'					\
+    -jobconf 	 stream.num.map.output.key.fields="$fields"				\
+    -mapper  	 "$map_script"  							\
+    -reducer	 "$reduce_script"							\
+    -input       "$input_file"								\
+    -output  	 "$output_file"								\
+    "$@"
+
+
+# -jobconf mapred.map.tasks=3                                                       \
+# -jobconf mapred.reduce.tasks=3                                                    \

data/bin/hdp-stream

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# hadoop dfs -rmr out/parsed-followers
+
+input_file=${1} 		; shift
+output_file=${1} 		; shift
+map_script=${1-/bin/cat}	; shift
+reduce_script=${1-/usr/bin/uniq}	; shift
+fields=${1-2} 			; shift
+
+if [ "$reduce_script" == "" ] ; then echo "$0 input_file output_file [sort_fields] [mapper] [reducer] [args]" ; exit ; fi
+
+HADOOP_HOME=${HADOOP_HOME-/usr/lib/hadoop}
+
+${HADOOP_HOME}/bin/hadoop \
+     jar         ${HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar		\
+    -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner 			\
+    -jobconf     map.output.key.field.separator='\t'					\
+    -jobconf     num.key.fields.for.partition=1 					\
+    -jobconf 	 stream.map.output.field.separator='\t'					\
+    -jobconf 	 stream.num.map.output.key.fields="$fields"				\
+    -mapper  	 "$map_script"  							\
+    -reducer	 "$reduce_script"							\
+    -input       "$input_file"								\
+    -output  	 "$output_file"								\
+    "$@"
+
+
+# -jobconf mapred.map.tasks=3                                                       \
+# -jobconf mapred.reduce.tasks=3                                                    \

data/bin/hdp-stream-flat

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+input_file=${1} 			; shift
+output_file=${1} 			; shift
+map_script=${1-/bin/cat}		; shift
+reduce_script=${1-/usr/bin/uniq}	; shift
+
+if [ "$reduce_script" == "" ] ; then echo "$0 input_file output_file [sort_fields] [mapper] [reducer] [args]" ; exit ; fi
+
+hadoop jar /home/flip/hadoop/h/contrib/streaming/hadoop-*-streaming.jar			\
+    -mapper  	"$map_script"  								\
+    -reducer	"$reduce_script"							\
+    -input      "$input_file"								\
+    -output  	"$output_file"								\
+    "$@"
+
+# -jobconf mapred.map.tasks=3                                                       \
+# -jobconf mapred.reduce.tasks=3                                                    \

data/bin/hdp-sync

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'wukong'
+
+src_dir, dest_dir = ARGV[0..1]
+src_files   = Dir[src_dir + '/*']
+dest_files  = Wukong::Dfs.list_files dest_dir
+Wukong::Dfs.compare_listings(src_files, dest_files) do |comparison, src_file, dest_file|
+  case comparison
+  when :missing
+    dest_filename = "%s/%s" % [dest_dir, dest_file]
+    puts "Copying #{src_file} #{dest_filename}"
+    puts `hadoop dfs -put #{src_file} #{dest_filename}`
+  when :differ
+    src_ls = `ls -l #{src_file}`.split(/\s+/).join("\t")
+    puts "Differ: #{src_ls} \n#{dest_file}"
+  end
+end

data/bin/hdp-wc

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+require 'wukong'
+NEWLINE_LENGTH = $/.length # KLUDGE
+
+#
+#
+#
+# !! The +words+ count comes out higher than that of +wc+ -- don't know
+# why. (It's close: a 10GB, 1M line dataset it showed 367833839 vs. 367713271)
+#
+class WcMapper < Wukong::Streamer::LineStreamer
+  attr_accessor :lines, :fields, :words, :chars, :bytes
+
+  def before_stream
+    self.lines, self.fields, self.words, self.chars, self.bytes = [0,0,0,0,0]
+  end
+
+  def process line
+    return unless line
+    self.lines  += 1
+    self.fields += 1 + line.count("\t")
+    self.words  += 1 + line.strip.scan(/\s+/).length unless line.blank?
+    self.chars  += line.chars.to_a.length + NEWLINE_LENGTH
+    self.bytes  += line.bytesize          + NEWLINE_LENGTH
+    $stderr.puts line if (line.chars.to_a.length != line.bytesize)
+  end
+
+  def after_stream
+    emit [lines, fields, words, chars,  bytes]
+  end
+end
+
+#
+#
+class WcReducer < Wukong::Streamer::Base
+  attr_accessor :lines, :fields, :words, :chars, :bytes
+
+  def before_stream
+    self.lines, self.fields, self.words, self.chars, self.bytes = [0,0,0,0,0]
+  end
+
+  def process m_lines, m_fields, m_words, m_chars, m_bytes
+    self.lines  += m_lines.to_i
+    self.fields += m_fields.to_i
+    self.words  += m_words.to_i
+    self.chars  += m_chars.to_i
+    self.bytes  += m_bytes.to_i
+  end
+
+  def after_stream
+    emit [lines, fields, words, chars,  bytes]
+  end
+end
+
+Wukong::Script.new(WcMapper, WcReducer, :reduce_tasks => 1).run
+
+# class FooScript < Wukong::Script
+#   def map_command
+#     '/usr/bin/wc'
+#   end
+#   def reduce_command
+#     '/bin/cat'
+#   end
+# end
+# FooScript.new(nil, nil, :reduce_tasks => 1).run
+#
+#  ruby -ne 'wc_v = `echo "#{$_.chomp}" | wc`; gr_v=($_.strip.empty? ? 0 : $_.strip.scan(/\s+/).length + 1 ) ; puts [wc_v.chomp, " ", gr_v, $_.chomp].join("\t")'

data/bin/md5sort

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+""" sorts lines (or tab-sep records) by md5.  (e.g. for train/test splits).
+optionally prepends with the md5 id too. 
+brendan o'connor - anyall.org - gist.github.com/brendano """
+
+import hashlib,sys,optparse
+p = optparse.OptionParser()
+p.add_option('-k',  type='int', default=False)
+p.add_option('-p', action='store_true')
+opts,args=p.parse_args()
+
+lines = sys.stdin.readlines()
+getter=lambda s: hashlib.md5(s[:-1]).hexdigest()
+if opts.k:
+  getter=lambda s: hashlib.md5(s[:-1].split("\t")[opts.k-1]).hexdigest()
+
+lines.sort(key=lambda s: getter(s))
+for line in lines:
+  if opts.p:  line = getter(line) + "\t" + line
+  print line,