wukong 3.0.0.pre → 3.0.0.pre2

data/lib/away/escapement.rb

@@ -1,129 +0,0 @@
-require 'htmlentities'
-require 'addressable/uri'
-
-# Fix a bug (?) in the HTMLEntities encoder class with $KCODE='NONE'
-HTMLEntities::Encoder.class_eval do
-private
-  def extended_entity_regexp
-    @extended_entity_regexp ||= (
-      if encoding_aware?
-        regexp = '[^\u{20}-\u{7E}]'
-      else
-        # regexp = '[^\x20-\x7E]'
-        regexp = '[\x00-\x1f]|[\xc0-\xfd][\x80-\xbf]+'
-      end
-      regexp += "|'" if @flavor == 'html4'
-      Regexp.new(regexp)
-      )
-  end
-end
-
-module Wukong
-  #
-  # By default (or explicitly with the :xml strategy), convert string to
-  # * XML-encoded ASCII,
-  #
-  # * with a guarantee that the characters " quote, ' apos \\ backslash,
-  #   carriage-return \r newline \n and tab \t (as well as all other control
-  #   characters) are encoded.
-  #
-  # * Any XML-encoding in the original text is encoded with no introspection:
-  #     encode_str("<a href=\"foo\">")
-  #     # => "<a href="foo">"
-  #
-  # * Useful: http://rishida.net/scripts/uniview/conversion.php
-  #
-  # With the :url strategy,
-  # * URL-encode the string
-  # * This is as strict as possible: encodes all but alphanumeric and _ underscore.
-  #   The resulting string is thus XML- and URL-safe.
-  #   http://addressable.rubyforge.org/api/classes/Addressable/URI.html#M000010
-  #
-  # Wukong.decode_str(Wukong.encode_str(str)) returns the original str
-  #
-  # If you're seeing bad_encoding errors, try
-  #   $KCODE='u' unless "1.9".respond_to?(:encoding)
-  # at the start of your script.
-  #
-  def self.encode_str str, strategy=:xml
-    begin
-      case strategy
-      when :xml        then self.html_encoder.encode(str, :basic, :named, :decimal).gsub(/\\/, '\')
-      when :url        then Addressable::URI.encode_component(str, /[^\w]/)
-      else raise "Don't know how to encode with strategy #{strategy}"
-      end
-    rescue ArgumentError => e
-      '!bad_encoding!! ' + str.gsub(/[^\w\s\.\-@#%]+/, '')
-    end
-  end
-  # HTMLEntities encoder instance
-  def self.html_encoder
-    @html_encoder ||= HTMLEntities.new
-  end
-
-  #
-  # Decode string from its encode_str representation.  This can include
-  # dangerous things such as tabs, newlines, backslashes and cryptofascist
-  # propaganda.
-  #
-  def self.decode_str str, strategy=:xml
-    case strategy
-    when :xml        then self.html_encoder.decode(str)
-    when :url        then Addressable::URI.unencode_component(str)
-    else raise "Don't know how to decode with strategy #{strategy}"
-    end
-  end
-
-  #
-  # Replace each given field in the hash with its
-  # encoded value
-  #
-  def self.encode_components hsh, *fields
-    fields.each do |field|
-      hsh[field] = hsh[field].to_s.wukong_encode if hsh[field]
-    end
-  end
-end
-
-String.class_eval do
-
-  #
-  # Strip control characters that might harsh our buzz, TSV-wise
-  # See Wukong.encode_str
-  #
-  def wukong_encode! *args
-    replace self.wukong_encode(*args)
-  end
-
-  def wukong_encode(*args)
-    Wukong.encode_str(self, *args)
-  end
-
-  #
-  # Decode string into original (and possibly unsafe) form
-  # See Wukong.encode_str and Wukong.decode_str
-  #
-  def wukong_decode!(*args)
-    replace self.wukong_decode(*args)
-  end
-
-  def wukong_decode(*args)
-    Wukong.decode_str(self, *args)
-  end
-
-  #
-  # Takes an XML-encoded or plaintext string and forces it into canonical encoding
-  #
-  def wukong_recode!(*args)
-    replace self.wukong_decode(*args).wukong_encode(*args)
-  end
-  def wukong_recode
-    Wukong.encode_str(Wukong.decode_str(self, *args), *args)
-  end
-end
-
-Struct.class_eval do
-  def recode!(*args)
-    each_pair{|k,v| v.wukong_recode!(*args) if (v && v.respond_to?(:wukong_recode!)) }
-  end
-end

data/lib/away/exe.rb

@@ -1,11 +0,0 @@
-#
-# This file must be explicitly required
-#
-
-require 'wukong'
-
-#
-
-Kernel.at_exit do
-  puts "Not implemented yet"
-end

data/lib/away/experimental.rb

@@ -1,5 +0,0 @@
-def Wu(&block)
-  Wukong.class_eval{ def self.load_examples_helper() require File.expand_path("../../examples/examples_helper", File.dirname(__FILE__)) ; end }
-  Wukong.instance_eval(&block)
-  Wukong.run
-end

data/lib/away/from_file.rb

@@ -1,52 +0,0 @@
-#
-# Borrowed from Opscode Chef -- thanks guys
-#
-# Author:: Adam Jacob (<adam@opscode.com>)
-# Author:: Christopher Walters (<cw@opscode.com>)
-# Copyright:: Copyright (c) 2008 Opscode, Inc.
-# License:: Apache License, Version 2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-module Wukong
-  module Mixin
-    module FromFile
-
-      # Loads a given ruby file, and runs instance_eval against it in the context of the current
-      # object.
-      #
-      # Raises an IOError if the file cannot be found, or is not readable.
-      def from_file(filename)
-        if File.exists?(filename) && File.readable?(filename)
-          self.instance_eval(IO.read(filename), filename, 1)
-        else
-          raise IOError, "Cannot open or read #{filename}!"
-        end
-      end
-
-      # Loads a given ruby file, and runs class_eval against it in the context of the current
-      # object.
-      #
-      # Raises an IOError if the file cannot be found, or is not readable.
-      def class_from_file(filename)
-        if File.exists?(filename) && File.readable?(filename)
-          self.class_eval(IO.read(filename), filename, 1)
-        else
-          raise IOError, "Cannot open or read #{filename}!"
-        end
-      end
-
-    end
-  end
-end

data/lib/away/job.rb

@@ -1,56 +0,0 @@
-module Wukong
-  #
-  #
-  #
-  #
-  class Job < Wukong::Graph
-    # invokable resources
-    attr_reader :resources
-
-    def to_s
-      ['<job', handle,
-        "resources={#{resources.join(' | ')}}",
-        "chain={#{chain.join(' | ')}}",
-      ].join(' ')+'>'
-    end
-
-
-    def add_resource(type, handle=nil, *args, &block)
-      rsrc = Wukong.create(type, handle, *args, &block)
-      rsrc.graph = self
-      @resources << rsrc
-      rsrc
-    end
-
-  end
-
-  module Task
-    extend Gorillib::Concern
-    include Wukong::Stage
-
-    module ClassMethods
-      def define_action(name, options={}, &block)
-        self.actions = self.actions.merge(name => options.merge(:block => block))
-      end
-
-      def class_defaults
-        super
-        # field :actions,     Array, :of => Symbol, :description => 'list of actions this stage responds to'
-        class_attribute :actions
-        self.actions ||= Hash.new
-        class_attribute :default_action
-
-        define_action :nothing, :description => 'ze goggles, zey do nussing'
-      end
-
-    end
-    included do
-      self.class_defaults
-    end
-  end
-
-  def self.job(handle, *args, &block)
-    @jobs ||= Hash.new
-    @jobs[handle] ||= Job.new(handle, *args, &block)
-  end
-end

data/lib/away/job/rake_compat.rb

@@ -1,17 +0,0 @@
-module Wukong
-
-  #
-  # Let `Wukong::Job`s invoke and depend on `Rake::Task`s.
-  #
-  # for example, Rails defines the `:environment` task:
-  #
-  #     task :email_expiring, :depends => :environment do
-  #       desc "Email expiring accounts to let them know"
-  #       date = ENV['from'] ? Date.parse(ENV['from']) : Date.today
-  #       Account.notify_expiring(date)
-  #     end
-  #
-  #
-  module RakeCompat
-  end
-end

data/lib/away/registry.rb

@@ -1,79 +0,0 @@
-module Wukong
-  @@registries ||= Hash.new
-
-  def self.registry(type, options={})
-    type = type.to_sym
-    plural = options[:plural] || "#{type}s"
-    return if @@registries[type]
-    @@registries[type] = Registry.new(type)
-    self.singleton_class.class_eval do
-      # def self.sources() @sources ; end
-      define_method(plural){ @@registries[type] }
-      # def self.source(handle) sources.find(handle) ; end
-      define_method("#{type}_klass"){|handle| @@registries[type].find(handle) }
-      # def self.register_source(klass) sources.register(klass) ; end
-      define_method("register_#{type}"){   |klass|  @@registries[type].register(klass) }
-      # def self.unregister_source(handle) sources.register(klass) ; end
-      define_method("unregister_#{type}"){ |handle| @@registries[type].unregister(handle) }
-      # def self.source_exists?(handle) sources.exists?(handle) ; end
-      define_method("#{type}_exists?"){       |handle| @@registries[type].exists?(handle) }
-    end
-  end
-
-  def self.create(type, *args, &block)
-    @@registries[type].create(*args, &block)
-  end
-
-  class Registry < Mash
-    attr_reader :type
-
-    def initialize(type)
-      @type = type
-    end
-
-    def all
-      self.dup.freeze
-    end
-
-    # given example of registry's class, return it;
-    # otherwise, look up the handle and return that.
-    def find(handle)
-      return handle if handle.is_a?(Class)
-      self[handle]
-    end
-
-    def find!(handle, *args)
-      find(handle, *args) or raise ArgumentError, "cannot find #{type} named '#{handle}'"
-    end
-
-    def exists?(handle)
-      self.has_key?(handle)
-    end
-
-    def create(handle, *args, &block)
-      find!(handle).new(*args, &block)
-    end
-
-    # add given class to registry
-    def register(klass)
-      self[klass.handle] = klass
-    end
-
-    def unregister(klass)
-      self.delete(klass.handle)
-    end
-
-    def convert_key(key)
-      key.is_a?(Class) ? key.handle : super(key)
-    end
-
-    # A valid identifier starts with a letter and has only letters, numbers and underscores
-    VALID_IDENTIFIER_RE = /\A[a-z]\w+\z/i
-
-    def self.valid_handle?(handle)
-      handle.to_s =~ VALID_IDENTIFIER_RE
-    end
-
-  end
-
-end

data/lib/away/runner.rb

@@ -1,276 +0,0 @@
-module Wukong
-  class RunnerResult
-    field :runner,      Runner, :doc => 'Runner object that created this job'
-    field :command,     Array, :of => String, :doc => 'launch command'
-    field :beg_time,    Time
-    field :end_time,    Time
-    field :raw_out,     String
-    field :raw_err,     String
-  end
-
-  #
-  # A uniform interface for launching processes.
-  #
-  # * accepts humanized and standardized args
-  # * synthesize args into a command
-  # * launch the process
-  # * parse its output
-  #
-  class Runner
-    class_attribute :result_parser ; self.result_parser = RunnerResult
-
-    field :name, Symbol, :required => true
-    field :executor_path, Pathname, :required => true
-
-    def to_long_params(arg_hsh, dash='-')
-      arg_hsh.inject([]) do |acc, (param, val)|
-        param = param.to_s.gsub(/[\-_\W]+/, dash)
-        acc << "--#{param.to_s}" << val.to_s
-      end
-    end
-
-    def native_args(arg_hsh)
-      to_dashed_params(arg_hsh)
-    end
-
-    def command(arg_hsh)
-      [executor_path, *native_args(arg_hsh)]
-    end
-
-    def run(input, arg_hsh)
-      cmd        = command(input, arg_hsh)
-      beg_time   = Time.now
-      out, err   = launch( *cmd )
-      end_time   = Time.now
-
-      result_parser.new({
-          :runner     => self,
-          :command    => cmd,
-          :beg_time   => beg_time,
-          :end_time   => end_time,
-          :input      => input,
-          :arg_hsh    => arg_hsh,
-          :raw_out    => out,
-          :raw_err    => err,
-        })
-    end
-
-    class << self
-      def executor(*args)
-        ArgumentError.check_arity!(args, 1)
-        @executor = args.first if args.present?
-        @executor
-      end
-
-      def launch(*cmd)
-        out = `#{cmd.join(' ')}`
-      end
-
-      def which(basename)
-        raise ArgumentError, "which wants a basename, not a path (#{basename})" if basename =~ %r{\/}
-        out, err = launch('which', basename)
-        out.chomp
-      end
-
-    end
-  end
-
-  module RunnerWithInputOutput
-    extend Gorillib::Concern
-    include Hanuman::IsOwnInputSlot
-    include Hanuman::IsOwnOutputSlot
-
-    # sugar for a command that takes input to produce output.
-    #
-    # @param [Array<String>, String] inputs -- added as the `:inputs` arg (converting to an array if necessary)
-    # @param [String] output -- added as the `:output` arg
-    #
-    def run(inputs, output, args={})
-      inputs = Array.wrap(inputs)
-      super args.merge(:inputs => inputs, :output => output)
-    end
-  end
-
-  #
-  # Wukong::Runner interface for the `cp` command
-  #
-  # @example
-  #   runner = Wukong::CpRunner.new
-  #   runner.run('my_src.jpg', 'my_dest.jpg')
-  #
-  class CpRunner
-    include  RunnerWithInputOutput
-    executor which('cp')
-
-    argument :verbose,      Boolean, :native => '-v', :solo => true, :doc => 'show files as they are copied'
-    argument :duplicate,    Boolean, :native => '-a', :solo => true, :doc => 'Preserves structure and attributes of files'
-  end
-
-  class ScpRunner
-    include  RunnerWithInputOutput
-    executor which('scp')
-
-    argument :verbose,      Boolean,  :native => '-v', :solo => true, :doc => 'show files as they are copied'
-    argument :duplicate,    Boolean,  :native => '-p', :solo => true, :doc => 'Preserves structure and attributes of files'
-    #
-    argument :ssh_user,     String
-    argument :dest_host,    String
-    argument :ssh_key_file, Pathname, :native => '-i'
-    argument :dest_port,    Integer,  :native => '-P'
-
-    argument :compression,  Boolean,  :native => '-C'
-    argument :recursive,    Boolean,  :native => '-r'
-
-    self.success_exit_status = 0
-  end
-
-  module RunnerForJava
-
-    argument :java_home, :env_var => 'JAVA_HOME', :doc => 'path to the java environment; $JAVA_HOME/bin usually holds your java runner'
-
-    argument :java_prog, :finally => ->(){ path_to(arg_val(:java_home), 'bin', 'java') }
-
-    argument :jar
-
-    argument :classpath
-
-    def java_conf
-    end
-
-  end
-
-  class HadoopRunner
-    include  RunnerWithInputOutput
-    executor which('hadoop')
-
-    argument :verbose,      Boolean,  :native => '-v', :solo => true, :doc => 'show files as they are copied'
-
-    argument :hadoop_home, :default => '/usr/lib/hadoop', :doc => "Path to hadoop installation; ENV['HADOOP_HOME'] by default. HADOOP_HOME/bin/hadoop is used to run hadoop.", :env_var => 'HADOOP_HOME'
-    argument :hadoop_runner,                            :doc => "Path to hadoop script. Usually set --hadoop_home instead of this."
-
-    #
-    # Translate simplified args to their hairy hadoop equivalents
-    #
-    argument :job_name,               :jobconf => 'mapred.job.name'
-    #
-    argument :io_sort_mb,             :jobconf => 'io.sort.mb'
-    argument :io_sort_record_percent, :jobconf => 'io.sort.record.percent'
-    argument :key_field_separator,    :jobconf => 'map.output.key.field.separator'
-    argument :map_speculative,        :jobconf => 'mapred.map.tasks.speculative.execution'
-    argument :map_tasks,              :jobconf => 'mapred.map.tasks'
-    argument :max_maps_per_cluster,   :jobconf => 'mapred.max.maps.per.cluster'
-    argument :max_maps_per_node,      :jobconf => 'mapred.max.maps.per.node'
-    argument :max_node_map_tasks,     :jobconf => 'mapred.tasktracker.map.tasks.maximum'
-    argument :max_node_reduce_tasks,  :jobconf => 'mapred.tasktracker.reduce.tasks.maximum'
-    argument :max_record_length,      :jobconf => 'mapred.linerecordreader.maxlength', :doc => "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
-    argument :max_reduces_per_cluster,:jobconf => 'mapred.max.reduces.per.cluster'
-    argument :max_reduces_per_node,   :jobconf => 'mapred.max.reduces.per.node'
-    argument :max_tracker_failures,   :jobconf => 'mapred.max.tracker.failures'
-    argument :max_map_attempts,       :jobconf => 'mapred.map.max.attempts'
-    argument :max_reduce_attempts,    :jobconf => 'mapred.reduce.max.attempts'
-    argument :min_split_size,         :jobconf => 'mapred.min.split.size'
-    argument :output_field_separator, :jobconf => 'stream.map.output.field.separator'
-    argument :partition_fields,       :jobconf => 'num.key.fields.for.partition'
-    argument :reduce_tasks,           :jobconf => 'mapred.reduce.tasks'
-    argument :respect_exit_status,    :jobconf => 'stream.non.zero.exit.is.failure'
-    argument :reuse_jvms,             :jobconf => 'mapred.job.reuse.jvm.num.tasks'
-    argument :sort_fields,            :jobconf => 'stream.num.map.output.key.fields'
-    argument :timeout,                :jobconf => 'mapred.task.timeout'
-    argument :noempty,                                  :doc => "don't create zero-byte reduce files (hadoop mode only)"
-    argument :split_on_xml_tag,                         :doc => "Parse XML document by specifying the tag name: 'anything found between <tag> and </tag> will be treated as one record for map tasks'"
-
-
-    argument :mapper_command,  String, :native => '-mapper'
-    argument :reducer_command, String, :native => '-reducer'
-
-    repeated_argument :file, String, :native => '-file'
-
-    # emit a -jobconf hadoop option if the simplified command line arg is present
-    def jobconf option
-      if settings[option]
-        # "-jobconf %s=%s" % [settings.definition_of(option, :description), settings[option]]
-        "-D %s=%s" % [settings.definition_of(option, :description), settings[option]]
-      end
-    end
-
-    def finalize_settings
-      settings[:reuse_jvms] = '-1'             if (settings[:reuse_jvms] == true)
-      settings[:respect_exit_status] = 'false' if (settings[:ignore_exit_status] == true)
-      settings[:reduce_tasks] = 0 if (! settings[:reduce_command])
-    end
-
-    def hadoop_other_args
-      extra_str_args  = [ settings[:extra_args] ]
-      if settings.split_on_xml_tag
-        extra_str_args << %Q{-inputreader 'StreamXmlRecordReader,begin=<#{settings.split_on_xml_tag}>,end=</#{settings.split_on_xml_tag}>'}
-      end
-      extra_str_args   << ' -lazyOutput' if settings[:noempty]  # don't create reduce file if no records
-      extra_str_args   << ' -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' unless settings[:partition_fields].blank?
-      extra_str_args
-    end
-
-    def hadoop_recycle_env
-      %w[RUBYLIB].map do |var|
-        %Q{-cmdenv '#{var}=#{ENV[var]}'} if ENV[var]
-      end.compact
-    end
-
-    # The path to the hadoop runner script
-    def hadoop_runner
-      settings[:hadoop_runner] || (settings[:hadoop_home]+'/bin/hadoop')
-    end
-
-    #
-    # Assemble the hadoop command to execute
-    # and launch the hadoop runner to execute the script across all tasktrackers
-    #
-    # FIXME: Should add some simple logic to ensure that commands are in the
-    # right order or hadoop will complain. ie. -D settings MUST come before
-    # others
-    #
-    def execute_hadoop_workflow
-      # Input paths join by ','
-      input_paths = @input_paths.join(',')
-      #
-      # Use Settings[:hadoop_home] to set the path your config install.
-      hadoop_commandline = [
-        hadoop_runner,
-        "jar #{settings[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
-        hadoop_jobconf_settings,
-        "-D mapred.job.name='#{job_name}'",
-        hadoop_other_args,
-        "-mapper  '#{mapper_commandline}'",
-        "-reducer '#{reducer_commandline}'",
-        "-input   '#{input_paths}'",
-        "-output  '#{output_path}'",
-        "-file    '#{this_script_filename}'",
-        hadoop_recycle_env,
-      ].flatten.compact.join(" \t\\\n  ")
-      Log.info "  Launching hadoop!"
-      execute_command!(hadoop_commandline)
-    end
-
-  end
-
-
-  #
-  # Req
-  #
-  class HadoopJob
-    field :job_id
-    field :k
-
-    def from_jobtracker(jobtracker_host)
-      contents = fetch_jobtracker_raw(jobtracker_host)
-      attrs = parse_jobtracker_raw(contents)
-    end
-
-    def fetch_jobtracker_raw(jobtracker_host)
-    end
-
-    def parse_jobtracker_raw(contents)
-    end
-  end
-
-
-end