wukong 3.0.0.pre3 → 3.0.0

data/lib/wukong/server.rb

@@ -1,119 +0,0 @@
-module Wukong
-  module EventMachineServer
-    include DriverMethods
-
-    def self.included klass
-      klass.class_eval do
-        attr_accessor :dataflow, :settings  
-
-        def self.add_signal_traps
-          Signal.trap('INT')  { log.info 'Received SIGINT. Stopping.'  ; EM.stop }
-          Signal.trap('TERM') { log.info 'Received SIGTERM. Stopping.' ; EM.stop }                  
-        end
-      end
-    end
-
-    def initialize(label, settings)
-      super
-      @settings = settings      
-      @dataflow = construct_dataflow(label, settings)
-    end
-      
-  end
-
-  class StdioServer < EM::P::LineAndTextProtocol
-    include EventMachineServer
-    include Processor::StdoutProcessor
-    include Logging
-    
-    def self.start(label, settings = {})
-      EM.attach($stdin, self, label, settings)
-    end
-
-    def post_init      
-      self.class.add_signal_traps
-      setup_dataflow
-    end
-
-    def receive_line line
-      driver.send_through_dataflow(line)
-    end
-
-    def unbind
-      finalize_and_stop_dataflow
-      EM.stop
-    end
-    
-  end
-
-  class TCPServer < EM::P::LineAndTextProtocol
-    include EventMachineServer
-    include Processor::BufferedProcessor
-    include Logging
-       
-    def self.start(label, settings = {})
-      host = settings[:host] || Socket.gethostname
-      port = settings[:port] || 9000
-      EM.start_server(host, port, self, label, settings)
-      log.info "Server started on #{host} on port #{port}"
-      add_signal_traps
-    end
-
-    def post_init
-      port, ip = Socket.unpack_sockaddr_in(get_peername)
-      log.info "Connected to #{ip} on #{port}"
-      setup_dataflow
-    end
-
-    def receive_line line
-      @buffer = []      
-      operation = proc { driver.send_through_dataflow(line) }
-      callback  = proc { flush_buffer @buffer }
-      EM.defer(operation, callback)
-    end
-
-    def flush_buffer records
-      send_data(records.join("\n") + "\n")
-      records.clear
-    end
-
-    def unbind
-      EM.stop
-    end
-    
-  end
-end
-
-class StupidServer
-
-  attr_accessor :dataflow, :settings
-  
-  def initialize(label, settings)
-    @settings = settings
-    builder   = (Wukong.registry.retrieve(label.to_sym) or raise Wukong::Error.new("No such processor or dataflow: #{label}"))
-    dataflow  = builder.build(settings)
-    @dataflow = dataflow.respond_to?(:stages) ? dataflow.directed_sort.map{ |name| dataflow.stages[name] } : [ dataflow ]
-    @dataflow << self
-  end
-
-  def dumb_driver
-    @dumb_driver ||= Wukong::Driver.new(dataflow)
-  end
-
-  def stop() ; end
-  def setup() ; end
-  def process(record) $stdout.puts record ; end
-
-  def run!
-    dataflow.each(&:setup)
-
-    while line = $stdin.readline.chomp rescue nil do
-      dumb_driver.send_through_dataflow(line)
-    end
-    dataflow.each do |stage|
-      stage.finalize(&dumb_driver.advance(stage)) if stage.respond_to?(:finalize)
-      stage.stop
-    end      
-
-  end  
-end

data/lib/wukong/spec_helpers/integration_driver.rb

@@ -1,157 +0,0 @@
-require 'open3'
-
-module Wukong
-  module SpecHelpers
-
-    # Provides a `command` method for writing integration tests for
-    # commands.
-    module IntegrationRunner
-
-      # Spawn a command and capture its STDOUT, STDERR, and exit code.
-      #
-      # The `args` will be joined together into a command line.
-      #
-      # It is expected that you will use the matchers defined in
-      # IntegrationMatchers in your integration tests:
-      #
-      # @example Check output of 'ls' includes a string 'foo.txt'
-      #   it "lists files" do
-      #     command('ls').should have_output('foo.txt')
-      #   end
-      #
-      # @example More complicated
-      #   context "long format" do
-      #     it "lists files with timestamps" do
-      #       command('ls', '-l').should have_output('foo.txt', /\w+ \d+ \d+:\d+/)
-      #     end
-      #   end
-      #
-      # @param [Array<String>] args
-      #
-      # @overload command(*args, options={})
-      #   If the last element of `args` is a Hash it will be used for
-      #   options.
-      #
-      #   The :env option specifies the command line environment to
-      #   use for the command.  By default this will be the value of
-      #   the Ruby process's own `ENV` variable.  If running in a
-      #   context in which the `integration_env` method is defined,
-      #   its return value will be merged on top of `ENV`.  An
-      #   explicitly provided :env option will again be merged on top.
-      #
-      #   The :cwd option specifies the working directory to start in.
-      #   It defaults to the value of <tt>Dir.pwd</tt>
-      #
-      #   @param [Array<String>] args
-      #   @param [Hash] options
-      #   @option options [Hash] env the shell environment to spawn the command with
-      #   @option options [Hash] cwd the directory to execute the command in
-      def command *args
-        a = args.flatten.compact
-        options = (a.last.is_a?(Hash) ? a.pop : {})
-
-        env = ENV.to_hash.dup
-        env.merge!(integration_env) if respond_to?(:integration_env)
-        env.merge!(options[:env] || {})
-
-        cwd   = options[:cwd]
-        cwd ||= (respond_to?(:integration_cwd) ? integration_cwd : Dir.pwd)
-
-        IntegrationDriver.new(a, cwd: cwd, env: env)
-      end
-    end
-
-    # A driver for running commands in a subprocess.
-    class IntegrationDriver
-
-      # The command to execute
-      attr_accessor :cmd
-      
-      # The directory in which to execute the command.
-      attr_accessor :cwd
-
-      # The ID of the spawned subprocess (while it was running).
-      attr_accessor :pid
-
-      # The STDOUT of the spawned process.
-      attr_accessor :stdout
-
-      # The STDERR of the spawned process.
-      attr_accessor :stderr
-
-      # The exit code of the spawned process.
-      attr_accessor :exit_code
-
-      # Run the command and capture its outputs and exit code.
-      #
-      # @return [true, false]
-      def run!
-        return false if ran?
-        FileUtils.cd(cwd) do
-          Open3.popen3(env, cmd) do |i, o, e, wait_thr|
-            self.pid = wait_thr.pid
-            
-            @inputs.each { |input| i.puts(input) }
-            i.close
-            
-            self.stdout    = o.read
-            self.stderr    = e.read
-            self.exit_code = wait_thr.value.to_i
-          end
-        end
-        @ran = true
-      end
-
-      # Initialize a new IntegrationDriver to run a given command.
-      def initialize args, options
-        @args   = args
-        @env    = options[:env]
-        @cwd    = options[:cwd]
-        @inputs = []
-      end
-
-      def cmd
-        @args.compact.map(&:to_s).join(' ')
-      end
-
-      def on *events
-        @inputs.concat(events)
-        self
-      end
-      alias_method :<, :on
-
-      def in dir
-        @cwd = dir
-        self
-      end
-
-      def using env
-        @env = env
-        self
-      end
-
-      def env
-        ENV.to_hash.merge(@env || {})
-      end
-
-      def ran?
-        @ran
-      end
-      
-      def cmd_summary
-        [
-         cmd,
-         "with env #{env_summary}",
-         "in dir #{cwd}"
-        ].join("\n")
-      end
-
-      def env_summary
-        { "PATH" => env["PATH"], "RUBYLIB" => env["RUBYLIB"] }.inspect
-      end
-      
-    end
-  end
-end
-
-  

data/lib/wukong/spec_helpers/processor_helpers.rb

@@ -1,89 +0,0 @@
-module Wukong
-  module SpecHelpers
-    module ProcessorHelpers
-
-      # Creates a new processor in a variety of convenient ways.
-      #
-      # Most simply, called without args, will return a new instance of
-      # a the klass named in the containing `describe` or `context`:
-      #
-      #   context MyApp::Tokenizer do
-      #     it "uses whitespace as the default separator between tokens" do
-      #       processor.separator.should == /\s+/
-      #     end
-      #   end
-      #
-      # if your processor has been registered (you created it with the
-      # <tt>Wukong.processor</tt> helper method or otherwise
-      # registered it yourself) then you can use its name:
-      #
-      #   context :tokenizer do
-      #     it "uses whitespace as the default separator between tokens" do
-      #       processor.separator.should == /\s+/
-      #     end
-      #   end
-      #
-      # The `processor` method can also be used inside RSpec's
-      # `subject` and `let` methods:
-      #
-      #   context "with no arguments" do
-      #     subject { processor }
-      #       it "uses whitespace as the default separator between tokens" do
-      #         separator.should == /\s+/
-      #       end
-      #     end
-      #   end
-      #
-      # and you can easily pass arguments, just like you would on the
-      # command line or in a dataflow definition:
-      # 
-      #   context "with arguments" do
-      #     subject { processor(separator: ' ') }
-      #       it "uses whitespace as the default separator between tokens" do
-      #         separator.should == ' '
-      #       end
-      #     end
-      #   end
-      #
-      # You can even name the processor directly if you want to:
-      #
-      #   context "tokenizers" do
-      #     let(:default_tokenizer) { processor(:tokenizer)                          }
-      #     let(:complex_tokenizer) { processor(:complex_tokenizer, stemming: true)  }
-      #     let(:french_tokenizer)  { processor(:complex_tokenizer, stemming: true)  }
-      #     ...
-      #   end
-      def processor *args, &block
-        options = args.extract_options!
-        name    = args.first || self.class.description
-        create_processor(name, options, &block)
-      end
-      alias_method :flow, :processor
-
-      # Is the given +klass+ a Wukong::Processor?
-      #
-      # @param [Class] klass
-      # @return [true, false]
-      def processor? klass
-        klass.build.is_a?(Processor)
-      end
-
-      # :nodoc:
-      def create_processor name_or_klass, options={}, &block
-        if name_or_klass.is_a?(Class)
-          klass = name_or_klass
-        else
-          klass = Wukong.registry.retrieve(name_or_klass.to_s.to_sym)
-          raise Error.new("Could not find a Wukong::Processor class named '#{name_or_klass}'") if klass.nil?
-        end
-        raise Error.new("#{klass} is not a subclass of Wukong::Processor") unless processor?(klass)
-        settings = Configliere::Param.new
-        Wukong.boot!(settings)
-        proc = klass.build(settings.merge(options))
-        proc.setup
-        proc.instance_eval(&block) if block_given?
-        proc
-      end
-    end
-  end
-end

data/lib/wukong/spec_helpers/spec_driver.rb

@@ -1,28 +0,0 @@
-module Wukong
-  module SpecHelpers
-    class SpecDriver < Array
-
-      attr_reader :processor
-
-      def initialize processor
-        super()
-        @processor = processor
-      end
-      
-      def run
-        return false unless processor
-        processor.given_records.each do |input|
-          processor.process(input) do |output|
-            self << output
-          end
-        end
-        processor.finalize do |output|
-          self << output
-        end
-        processor.stop
-        self
-      end
-
-    end
-  end
-end

data/spec/wukong/local_runner_spec.rb

@@ -1,31 +0,0 @@
-require 'spec_helper'
-# require 'wukong'
-# require 'wukong/local_runner'
-
-# describe Wukong::LocalRunner, :examples_spec => true, :helpers => true do
-
-#   context 'examples' do
-
-#     subject{
-#       test_sink = test_sink()
-#       Wukong.dataflow(:integers) do
-#         input   :default, Wukong::Source::Integers.new(:size => 100)
-#         output  :default, test_sink
-
-#         input(:default)  >
-#           map(&:to_s)    >
-#           re(/..+/)      >
-#           map(&:reverse) >
-#           limit(20)      >
-#           output(:default)
-#       end
-#       Wukong::LocalRunner.receive(:flow => Wukong.dataflow(:integers))
-#     }
-
-#     it 'runs' do
-#       subject.run(:default)
-#       subject.flow.output(:default).records.should == %w[01 11 21 31 41 51 61 71 81 91 02 12 22 32 42 52 62 72 82 92]
-#     end
-
-#   end
-# end

data/spec/wukong/wu_local_spec.rb

@@ -1,125 +0,0 @@
-require 'spec_helper'
-
-describe 'wu-local' do
-
-  let(:input) { %w[1 2 3] }
-  
-  context "without any arguments" do
-    subject { command('wu-local') }
-    it {should exit_with(:non_zero) }
-    it "displays help on STDERR" do
-      should have_stderr("usage: wu-local")
-    end
-  end
-
-  context "running outside any Ruby project" do
-    subject { command('wu-local count').in(examples_dir('empty')) < input }
-    it { should exit_with(0) }
-    it "runs the processor" do
-      should have_stdout("3")
-    end
-    context "when passed a BUNDLE_GEMFILE" do
-      context "that doesn't belong to a deploy pack" do
-        subject { command('wu-local count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('ruby_project', 'Gemfile').to_s)) < input }
-        it { should exit_with(0) }
-        it "runs the processor" do
-          should have_stdout("3")
-        end
-      end
-      context "that belongs to a deploy pack" do
-        subject { command('wu-local count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < input }
-        it { should exit_with(0) }
-        it "runs the processor" do
-          should have_stdout("3")
-        end
-        context "loading the deploy pack" do
-          subject { command('wu-local string_reverser').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < 'hi' }
-          it { should exit_with(0) }
-          it "runs the processor" do
-            should have_stdout("ih")
-          end
-        end
-      end
-    end
-  end
-
-  context "running within a Ruby project" do
-    context "at its root" do
-      subject { command('wu-local count').in(examples_dir('ruby_project')) < input }
-      it { should exit_with(0) }
-      it "runs the processor" do
-        should have_stdout("3")
-      end
-    end
-    context "deep within it" do
-      subject { command('wu-local count').in(examples_dir('ruby_project')) < input }
-      it { should exit_with(0) }
-      it "runs the processor" do
-        should have_stdout("3")
-      end
-    end
-  end
-
-  context "running within a deploy pack" do
-    context "at its root" do
-      subject { command('wu-local count').in(examples_dir('deploy_pack')) < input }
-      it { should exit_with(0) }
-      it "runs the processor" do
-        should have_stdout("3")
-      end
-      context "loading the deploy pack" do
-        subject { command('wu-local string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
-        it { should exit_with(0) }
-        it "runs the processor" do
-          should have_stdout("ih")
-        end
-      end
-    end
-    context "deep within it" do
-      subject { command('wu-local count').in(examples_dir('deploy_pack')) < input }
-      it { should exit_with(0) }
-      it "runs the processor" do
-        should have_stdout("3")
-      end
-      context "loading the deploy pack" do
-        subject { command('wu-local string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
-        it { should exit_with(0) }
-        it "runs the processor" do
-          should have_stdout("ih")
-        end
-      end
-    end
-  end
-  
-  # context "running within a deploy pack" do
-  #   context "at its root" do
-  #     let(:subject) { command('wu-local', :cwd => examples_dir('deploy_pack')) }
-  #   end
-  #   context "deep within it" do
-  #     let(:subject) { command('wu-local', :cwd => examples_dir('deploy_pack', 'a','b','c')) }
-  #   end
-  # end
-  
-  # context "in local mode" do
-  #   context "on a map-only job" do
-  #     let(:subject) { command('wu-hadoop', example_script('tokenizer.rb'), "--mode=local", "--input=#{example_script('sonnet_18.txt')}") }
-  #     it { should exit_with(0) }
-  #     it { should have_stdout('Shall', 'I', 'compare', 'thee', 'to', 'a', "summer's", 'day') }
-  #   end
-    
-  #   context "on a map-reduce job" do
-  #     let(:subject) { command('wu-hadoop', example_script('word_count.rb'), "--mode=local", "--input=#{example_script('sonnet_18.txt')}") }
-  #     it { should exit_with(0) }
-  #     it { should have_stdout(/complexion\s+1/, /Death\s+1/, /temperate\s+1/) }
-  #   end
-  # end
-
-  # context "in Hadoop mode" do
-  #   context "on a map-only job" do
-  #     let(:subject) { command('wu-hadoop', example_script('tokenizer.rb'), "--mode=hadoop", "--input=/data/in", "--output=/data/out", "--dry_run") }
-  #     it { should exit_with(0) }
-  #     it { should have_stdout(%r{jar.*hadoop.*streaming.*\.jar}, %r{-mapper.+tokenizer\.rb}, %r{-input.*/data/in}, %r{-output.*/data/out}) }
-  #   end
-  # end
-  
-end