swineherd 0.0.1

data/examples/pagerank/pagerank.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH << '../../lib'
+require 'swineherd'        ; include Swineherd
+require 'swineherd/script' ; include Swineherd::Script
+require 'swineherd/filesystem'
+
+Settings.define :flow_id,     :required => true,                     :description => "Flow id required to make run of workflow unique"
+Settings.define :iterations,  :type => Integer,  :default => 10,     :description => "Number of pagerank iterations to run"
+Settings.define :hadoop_home, :default => '/usr/local/share/hadoop', :description => "Path to hadoop config"
+Settings.resolve!
+
+flow = Workflow.new(Settings.flow_id) do
+
+  # The filesystems we're going to be working with
+  hdfs    = Swineherd::FileSystem.get(:hdfs)
+  localfs = Swineherd::FileSystem.get(:file)
+
+  # The scripts we're going to use
+  initializer = PigScript.new('scripts/pagerank_initialize.pig')
+  iterator    = PigScript.new('scripts/pagerank.pig')
+  finisher    = WukongScript.new('scripts/cut_off_list.rb')
+  plotter     = RScript.new('scripts/histogram.R')
+
+  #
+  # Runs simple pig script to initialize pagerank. We must specify the input
+  # here as this is the first step in the workflow. The output attribute is to
+  # ensure idempotency and the options attribute is the hash that will be
+  # converted into command-line args for the pig interpreter.
+  #
+  task :pagerank_initialize do
+    initializer.options = {:adjlist => "/tmp/pagerank_example/seinfeld_network.tsv", :initgrph => next_output(:pagerank_initialize)}
+    initializer.run(:hadoop) unless hdfs.exists? latest_output(:pagerank_initialize)
+  end
+
+  #
+  # Runs multiple iterations of pagerank with another pig script and manages all
+  # the intermediate outputs.
+  #
+  task :pagerank_iterate => [:pagerank_initialize] do
+    iterator.options[:damp]           = '0.85f'
+    iterator.options[:curr_iter_file] = latest_output(:pagerank_initialize)
+    Settings.iterations.times do
+      iterator.options[:next_iter_file] = next_output(:pagerank_iterate)
+      iterator.run(:hadoop) unless hdfs.exists? latest_output(:pagerank_iterate)
+      iterator.refresh!
+      iterator.options[:curr_iter_file] = latest_output(:pagerank_iterate)
+    end
+  end
+
+  #
+  # Here we use a wukong script to cut off the last field (a big pig bag of
+  # links). Notice how every wukong script MUST have an input but pig scripts do
+  # not.
+  #
+  task :cut_off_adjacency_list => [:pagerank_iterate] do
+    finisher.input  << latest_output(:pagerank_iterate)
+    finisher.output << next_output(:cut_off_adjacency_list)
+    finisher.run :hadoop unless hdfs.exists? latest_output(:cut_off_adjacency_list)
+  end
+
+  #
+  # We want to pull down one result file, merge the part-000.. files into one file
+  #
+  task :merge_results => [:cut_off_adjacency_list] do
+    merged_results = next_output(:merge_results)
+    hdfs.merge(latest_output(:cut_off_adjacency_list), merged_results) unless hdfs.exists? merged_results
+  end
+
+  #
+  # Cat results into a local directory with the same structure
+  # eg. #{work_dir}/#{flow_id}/pull_down_results-0.
+  #
+  # FIXME: Bridging filesystems is cludgey.
+  #
+  task :pull_down_results => [:merge_results] do
+    local_results = next_output(:pull_down_results)
+    hdfs.copy_to_local(latest_output(:merge_results), local_results) unless localfs.exists? local_results
+  end
+
+  #
+  # Plot 2nd column of the result as a histogram (requires R and
+  # ggplot2). Note that the output here is a png file but doesn't have that
+  # extension. Ensmarten me as to the right way to handle that?
+  #
+  task :plot_results =>  [:pull_down_results] do
+    plotter.attributes = {
+      :pagerank_data => latest_output(:pull_down_results),
+      :plot_file     => next_output(:plot_results), # <-- this will be a png...
+      :raw_rank      => "aes(x=d$V2)"
+    }
+    plotter.run(:hadoop) unless localfs.exists? latest_output(:plot_results)
+  end
+
+end
+
+flow.workdir = "/tmp/pagerank_example"
+flow.describe
+flow.run(:plot_results)

data/examples/pagerank/scripts/cut_off_list.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'wukong'
+
+#
+# Does the very simple job of cutting of the giant adjacency list
+#
+class CutMapper < Wukong::Streamer::RecordStreamer
+  def process *args
+    node_a, node_b, list = args
+    yield [node_a, node_b]
+  end
+end
+
+Wukong::Script.new(CutMapper, nil).run

data/examples/pagerank/scripts/histogram.R

@@ -0,0 +1,5 @@
+library(ggplot2);
+png('<%= plot_file %>', width=900, res=132);
+d <- read.table('<%= pagerank_data %>', header=FALSE, sep='\t');
+p <- ggplot(d, <%= raw_rank %>) + geom_histogram() + xlab("") + ylab("");
+p;

data/examples/pagerank/scripts/pagerank.pig

@@ -0,0 +1,20 @@
+--
+-- Runs exactly one pagerank iteration
+--
+network      = LOAD '$CURR_ITER_FILE' AS (node_a:chararray, rank:float, out_links:bag { link:tuple (node_b:chararray) });
+sent_shares  = FOREACH network GENERATE FLATTEN(out_links) AS node_b, (float)(rank / (float)SIZE(out_links)) AS share:float;
+sent_links   = FOREACH network GENERATE node_a, out_links;
+rcvd_shares  = COGROUP sent_links BY node_a INNER, sent_shares BY node_b;
+next_iter    = FOREACH rcvd_shares
+               {
+                   raw_rank    = (float)SUM(sent_shares.share);
+                   -- treat the case that a node has no in links                   
+                   damped_rank = ((raw_rank IS NOT NULL AND raw_rank > 1.0e-12f) ? raw_rank*$DAMP + 1.0f - $DAMP : 0.0f);
+                   GENERATE
+                       group         AS node_a,
+                       damped_rank   AS rank,
+                       FLATTEN(sent_links.out_links) -- hack, should only be one bag, unbag it
+                   ;
+               };
+
+STORE next_iter INTO '$NEXT_ITER_FILE';

data/examples/pagerank/scripts/pagerank_initialize.pig

@@ -0,0 +1,24 @@
+--
+-- Create initial graph on which to iterate the pagerank algorithm.
+--
+
+--
+-- Generate a unique list of nodes with in links to cogroup on. This allows
+-- us to treat the case where nodes have in links but no out links.
+--
+network     = LOAD '$ADJLIST' AS (node_a:chararray, node_b:chararray);
+cut_rhs     = FOREACH network GENERATE node_b;
+uniq_rhs    = DISTINCT cut_rhs;
+list_links  = COGROUP network BY node_a, uniq_rhs BY node_b;
+count_links = FOREACH list_links
+              {
+                  -- if network.node_b is empty there are no out links, set to dummy value
+                  out_links = (IsEmpty(network.node_b) ? {('dummy')} : network.node_b);
+                  GENERATE
+                      group     AS node_a,
+                      1.0f      AS rank,
+                      out_links AS out_links
+                  ;
+              };
+
+STORE count_links INTO '$INITGRPH';

data/lib/swineherd.rb

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'configliere' ; Configliere.use(:commandline, :env_var, :define)
+require 'rake'
+require 'gorillib/logger/log'
+
+module Swineherd
+  autoload :Template,     'swineherd/template'
+  autoload :FileSystem,   'swineherd/filesystem'
+  autoload :Script,       'swineherd/script'
+  autoload :Workflow,     'swineherd/workflow'
+end

data/lib/swineherd/filesystem.rb

@@ -0,0 +1,26 @@
+module Swineherd
+  autoload :BaseFileSystem,   'swineherd/filesystem/basefilesystem'
+  autoload :LocalFileSystem,  'swineherd/filesystem/localfilesystem'
+  autoload :HadoopFileSystem, 'swineherd/filesystem/hadoopfilesystem'
+  autoload :S3FileSystem,     'swineherd/filesystem/s3filesystem'
+
+  class FileSystem
+
+    FILESYSTEMS = {
+      'file' => Swineherd::LocalFileSystem,
+      'hdfs' => Swineherd::HadoopFileSystem,
+      's3'   => Swineherd::S3FileSystem
+    }
+
+    # A factory function that returns an instance of the requested class
+    def self.get scheme, *args
+      begin
+        FILESYSTEMS[scheme.to_s].new *args
+      rescue NoMethodError => e
+        raise "Filesystem with scheme #{scheme} does not exist.\n #{e.message}"
+      end
+    end
+
+  end
+
+end

data/lib/swineherd/filesystem/README_filesystem.textile

@@ -0,0 +1,47 @@
+h1.  File System Abstraction
+
+Hackboxen need to access files and directories in order to do their
+stuff.  We currently expect them to use at least the following types
+of filesystems:
+
+* Local File System
+* Ephemeral Hadoop cluster HDFS
+* s3/HDFS
+
+Each of these filesystem types has different methods to accomplish the same operations. In order to make this diversity more easily used by hackboxen, an abstraction layer has been created.  
+
+h2. Interface
+
+A new @FileSystem@ class has a single class method @get@ taking two argugments:
+
+* @scheme@: A token which specifies the filesystem scheme. Currently, only @:file@ is supported.
+* @*args@: Optional arguments (e.g. credentitals)
+
+The returned (abstracted) filesystem instnace has the following methods:
+
+* @open(path,mode,blk)@: Return a @File@ like file handle object.  @mode@ and @blk@ arguments are optional and work like the standard ruby @File.open@ arguments.
+* @rm(path)@: Works like UNIX @rm -r@.
+* @exists?(path)@: Returns @true@ if the file/directory exists
+* @mv(srcpath,dstpath)@: Renames/moves the file/directory.
+* @cp(srcpath,dstpath)@: Works like UNIX @cp -r@.
+* @mkpath(dirpath)@: Creates a directory and all required parent directories.
+* @type(path)@: Returns one of "dir", "file", or "symlink".
+* @entries(dirpath)@: Returns the the files/subdirectories in this directory
+
+The @File@ object returned by the @open@ methods has the following methods:
+
+* @read@: Return the contents of the entire file as a string.
+* @readline@: Return the next line in the file, or nil if there no more lines.
+* @write(string)@: Write @string@ to the file.
+* @close@:  Close the file
+
+h2. Creating an abstraction
+
+Each abstraction is not expected to catch and rethrow exceptions of the abstracted subsystems. Rather, exceptions should pass through.  However, each method should try to be built to behave similarly to the corresponding native ruby @File@ and @FileUtils@ methods.
+
+h2. Current State
+
+The only currently implemented filesystem abstraction is @:file@ (local file system).
+
+
+

data/lib/swineherd/filesystem/basefilesystem.rb

@@ -0,0 +1,125 @@
+module Swineherd
+
+  #
+  # All methods a filesystem should have
+  #
+  module BaseFileSystem
+
+    #
+    # Return a new instance of 'this' filesystem. Classes that include this
+    # module are expected to know how to pull their particular set of arguments
+    # from *args and initialize themselves by opening any required connections, &c.
+    #
+    def initialize *args
+    end
+
+    #
+    # Open a file in this filesystem. Should return a usable file handle for in
+    # the mode (read 'r' or 'w') given. File classes should, at minimum, have
+    # the methods defined in BaseFile
+    #
+    def open path, mode="r", &blk
+    end
+
+    #
+    # Recursively delete the path and all paths below it.
+    #
+    def rm path
+    end
+
+    #
+    # Returns true if the file or path exists and false otherwise.
+    #
+    def exists? path
+    end
+
+    #
+    # Moves the source path to the destination path
+    #
+    def mv srcpath, dstpath
+    end
+
+    #
+    # Recursively copies all files and directories under srcpath to dstpath
+    #
+    def cp srcpath, dstpath
+    end
+
+    #
+    # Make directory path if it does not (partly) exist
+    #
+    def mkpath path
+    end
+
+    #
+    # Return file type ("directory" or "file" or "symlink")
+    #
+    def type path
+    end
+
+    #
+    # Give contained files/dirs
+    #
+    def entries dirpath
+    end
+
+    #
+    # For running tasks idempotently. Returns true if no paths exist, false if all paths exist,
+    # and raises an error otherwise.
+    #
+    def check_paths paths
+      exist_count = paths.inject(0){|cnt, path| cnt += 1 if exists?(path); cnt}
+      raise "Indeterminate output state" if (exist_count > 0) && (exist_count < paths.size)
+      return true if exist_count == 0
+      false
+    end
+
+    #
+    # Needs to close the filesystem by cleaning up any open connections, &c.
+    #
+    def close *args
+    end
+
+    class BaseFile
+      attr_accessor :path, :scheme, :mode
+
+
+      def initialize *args, &blk
+      end
+
+      #
+      # A new file in the filesystem needs to be instantiated with a
+      # path, a mode (read 'r' or write 'w').
+      #
+      def open path, mode="r", &blk
+      end
+
+      #
+      # Return whole file and as a string
+      #
+      def read
+      end
+
+      #
+      # Return a line from stream
+      #
+      def readline
+      end
+
+      #
+      # Writes a string to the file
+      #
+      def write string
+      end
+
+      #
+      # Close the file
+      #
+      def close *args
+      end
+
+    end
+
+  end
+
+end

data/lib/swineherd/filesystem/filesystems.rb

@@ -0,0 +1,103 @@
+require 'fileutils'
+
+class FileSystem
+
+  # A factory function that returns an instance of the requested class
+  def self.get(scheme, *args)
+    if scheme == :file
+      LocalFileSystem.new()
+    else
+      nil
+    end
+  end
+   
+  class LocalFileSystem
+    
+    # Open a file in this filesystem
+    def open(path,mode="r",&blk)
+      return LocalFile.new(path,mode,&blk)
+    end
+
+    # Works like rm -r
+    def rm(path)
+      FileUtils.rm_r(path)
+    end
+    
+    # Does this exist?
+    def exists?(path)
+      File.exists?(path)
+    end
+    
+    # Works like UNIX mv
+    def mv(srcpath,dstpath)
+      FileUtils.mv(srcpath,dstpath)
+    end
+    
+    # Works like UNIX cp -r
+    def cp(srcpath,dstpath)
+      FileUtils.cp_r(srcpath,dstpath)
+    end
+    
+    # Make directory path if it does not (partly) exist
+    def mkpath(path)
+      FileUtils.mkpath
+    end
+    
+    # Return file type ("dir" or "file" or "symlink")
+    def type(path)
+      if File.symlink?(path)
+        return "symlink"
+      end
+      if File.directory?(path)
+        return "directory"
+      end
+      if File.file?(path)
+        return "file"
+      end
+      "unknown"
+    end
+    
+    # Give contained files/dirs
+    def entries(dirpath)
+      if type(dirpath) != "directory"
+        return nil
+      end
+      Dir.entries(dirpath)
+    end
+    
+    class LocalFile
+      attr_accessor :path, :scheme, :mode
+      
+      def initialize(path,mode="r",&blk)
+        @path=path
+        @mode=mode
+        @handle=File.open(path,mode,&blk)
+      end
+      
+      def open(path,mode="r")
+        # Only "r" and "w" modes are supported.
+        initialize(path,mode)
+      end
+      
+      # Return whole file and as a string
+      def read
+        @handle.read
+      end
+      
+      # Return a line from stream
+      def readline
+        @handle.gets
+      end
+      
+      # Writes to the file
+      def write(string)
+        @handle.write(string)
+      end
+      
+      # Close file
+      def close
+        @handle.close
+      end
+    end
+  end
+end