wukong 3.0.0.pre3 → 3.0.0

data/spec/wukong/wu-local_spec.rb

@@ -0,0 +1,109 @@
+require 'spec_helper'
+
+describe 'wu-local' do
+
+  let(:input) { %w[1 2 3] }
+  
+  context "without any arguments" do
+    subject { wu_local() }
+    it {should exit_with(:non_zero) }
+    it "displays help on STDERR" do
+      should have_stderr(/provide a processor.*to run/)
+    end
+  end
+
+  context "running outside any Ruby project" do
+    subject { wu_local('count').in(examples_dir('empty')) < input }
+    it { should exit_with(0) }
+    it "runs the processor" do
+      should have_stdout("3")
+    end
+    context "when passed a BUNDLE_GEMFILE" do
+      context "that doesn't belong to a deploy pack" do
+        subject { wu_local('count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('ruby_project', 'Gemfile').to_s)) < input }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("3")
+        end
+      end
+      context "that belongs to a deploy pack" do
+        subject { wu_local('count').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < input }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("3")
+        end
+        context "loading the deploy pack" do
+          subject { wu_local('string_reverser').in(examples_dir('empty')).using(integration_env.merge("BUNDLE_GEMFILE" => examples_dir('deploy_pack', 'Gemfile').to_s)) < 'hi' }
+          it { should exit_with(0) }
+          it "runs the processor" do
+            should have_stdout("ih")
+          end
+        end
+      end
+    end
+  end
+
+  context "running within a Ruby project" do
+    context "at its root" do
+      subject { wu_local('count').in(examples_dir('ruby_project')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+    end
+    context "deep within it" do
+      subject { wu_local('count').in(examples_dir('ruby_project')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+    end
+  end
+
+  context "running within a deploy pack" do
+    context "at its root" do
+      subject { wu_local('count').in(examples_dir('deploy_pack')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+      context "loading the deploy pack" do
+        subject { wu_local('string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("ih")
+        end
+      end
+    end
+    context "deep within it" do
+      subject { wu_local('count').in(examples_dir('deploy_pack')) < input }
+      it { should exit_with(0) }
+      it "runs the processor" do
+        should have_stdout("3")
+      end
+      context "loading the deploy pack" do
+        subject { wu_local('string_reverser').in(examples_dir('deploy_pack')) < 'hi' }
+        it { should exit_with(0) }
+        it "runs the processor" do
+          should have_stdout("ih")
+        end
+      end
+    end
+  end
+
+  context "parsing records with the --from argument" do
+    subject { wu_local('identity --from=tsv') < "1\t2\t3" }
+    it { should exit_with(0) }
+    it "should emit a row for each input column" do
+      should have_stdout("1\n2\n3")
+    end
+  end
+
+  context "formatting records with the --to argument" do
+    subject { wu_local('identity --from=tsv --to=json') < "1\t2\t3" }
+    it { should exit_with(0) }
+    it "should emit a single JSON array for the input record" do
+      should have_stdout('["1","2","3"]')
+    end
+  end
+end

metadata

@@ -1,8 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: wukong
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre3
-  prerelease: 6
+  version: 3.0.0
+  prerelease: 
 platform: ruby
 authors:
 - Infochimps
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-17 00:00:00.000000000 Z
+date: 2013-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: configliere
@@ -181,8 +181,9 @@ files:
 - bin/wu-lign
 - bin/wu-local
 - bin/wu-plus
-- bin/wu-server
 - bin/wu-sum
+- diagrams/wu_local.dot
+- diagrams/wu_local.dot.png
 - examples/Gemfile
 - examples/README.md
 - examples/basic/string_reverser.rb
@@ -214,6 +215,7 @@ files:
 - examples/geo/tile_fetcher.rb
 - examples/graph/implied_geolocation/README.md
 - examples/graph/minimum_spanning_tree/airfares_graphviz.rb
+- examples/loadable.rb
 - examples/munging/airline_flights/airline_flights.rake
 - examples/munging/airline_flights/airplane.rb
 - examples/munging/airline_flights/airport_id_unification.rb
@@ -264,6 +266,7 @@ files:
 - examples/serverlogs/visit_paths/serverlogs-02-histograms-mapper.rb
 - examples/serverlogs/visit_paths/serverlogs-03-breadcrumbs-full.rb
 - examples/serverlogs/visit_paths/serverlogs-04-page_page_edges-full.rb
+- examples/string_reverser.rb
 - examples/text/pig_latin/pig_latinizer.rb
 - examples/text/pig_latin/pig_latinizer_widget.rb
 - examples/text/regional_flavor/README.md
@@ -288,11 +291,17 @@ files:
 - lib/wu/social/models/twitter.rb
 - lib/wu/wikipedia/models.rb
 - lib/wukong.rb
-- lib/wukong/boot.rb
-- lib/wukong/configuration.rb
 - lib/wukong/dataflow.rb
+- lib/wukong/doc_helpers.rb
+- lib/wukong/doc_helpers/dataflow_handler.rb
+- lib/wukong/doc_helpers/field_handler.rb
+- lib/wukong/doc_helpers/processor_handler.rb
 - lib/wukong/driver.rb
-- lib/wukong/emitter.rb
+- lib/wukong/local.rb
+- lib/wukong/local/event_machine_driver.rb
+- lib/wukong/local/runner.rb
+- lib/wukong/local/stdio_driver.rb
+- lib/wukong/local/tcp_driver.rb
 - lib/wukong/logger.rb
 - lib/wukong/model/faker.rb
 - lib/wukong/model/flatpack_parser/flat.rb
@@ -300,17 +309,23 @@ files:
 - lib/wukong/model/flatpack_parser/lang.rb
 - lib/wukong/model/flatpack_parser/parser.rb
 - lib/wukong/model/flatpack_parser/tokens.rb
+- lib/wukong/plugin.rb
 - lib/wukong/processor.rb
+- lib/wukong/rake_helper.rb
 - lib/wukong/runner.rb
-- lib/wukong/server.rb
+- lib/wukong/runner/boot_sequence.rb
+- lib/wukong/runner/code_loader.rb
+- lib/wukong/runner/deploy_pack_loader.rb
+- lib/wukong/runner/help_message.rb
 - lib/wukong/spec_helpers.rb
-- lib/wukong/spec_helpers/integration_driver.rb
-- lib/wukong/spec_helpers/integration_driver_matchers.rb
-- lib/wukong/spec_helpers/processor_helpers.rb
-- lib/wukong/spec_helpers/processor_methods.rb
+- lib/wukong/spec_helpers/integration_tests.rb
+- lib/wukong/spec_helpers/integration_tests/integration_test_matchers.rb
+- lib/wukong/spec_helpers/integration_tests/integration_test_runner.rb
 - lib/wukong/spec_helpers/shared_examples.rb
-- lib/wukong/spec_helpers/spec_driver.rb
-- lib/wukong/spec_helpers/spec_driver_matchers.rb
+- lib/wukong/spec_helpers/unit_tests.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_driver.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_matchers.rb
+- lib/wukong/spec_helpers/unit_tests/unit_test_runner.rb
 - lib/wukong/version.rb
 - lib/wukong/widget/filters.rb
 - lib/wukong/widget/processors.rb
@@ -351,8 +366,12 @@ files:
 - spec/wu/geo/geolocated_spec.rb
 - spec/wu/model/reconcilable_spec.rb
 - spec/wukong/dataflow_spec.rb
-- spec/wukong/local_runner_spec.rb
+- spec/wukong/driver_spec.rb
+- spec/wukong/local/runner_spec.rb
+- spec/wukong/local_spec.rb
+- spec/wukong/logger_spec.rb
 - spec/wukong/model/faker_spec.rb
+- spec/wukong/processor_spec.rb
 - spec/wukong/runner_spec.rb
 - spec/wukong/widget/filters_spec.rb
 - spec/wukong/widget/processors_spec.rb
@@ -364,7 +383,7 @@ files:
 - spec/wukong/widget/serializers_spec.rb
 - spec/wukong/widget/sink_spec.rb
 - spec/wukong/widget/source_spec.rb
-- spec/wukong/wu_local_spec.rb
+- spec/wukong/wu-local_spec.rb
 - spec/wukong/wukong_spec.rb
 - wukong.gemspec
 homepage: https://github.com/infochimps-labs/wukong
@@ -383,9 +402,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -420,8 +439,12 @@ test_files:
 - spec/wu/geo/geolocated_spec.rb
 - spec/wu/model/reconcilable_spec.rb
 - spec/wukong/dataflow_spec.rb
-- spec/wukong/local_runner_spec.rb
+- spec/wukong/driver_spec.rb
+- spec/wukong/local/runner_spec.rb
+- spec/wukong/local_spec.rb
+- spec/wukong/logger_spec.rb
 - spec/wukong/model/faker_spec.rb
+- spec/wukong/processor_spec.rb
 - spec/wukong/runner_spec.rb
 - spec/wukong/widget/filters_spec.rb
 - spec/wukong/widget/processors_spec.rb
@@ -433,6 +456,6 @@ test_files:
 - spec/wukong/widget/serializers_spec.rb
 - spec/wukong/widget/sink_spec.rb
 - spec/wukong/widget/source_spec.rb
-- spec/wukong/wu_local_spec.rb
+- spec/wukong/wu-local_spec.rb
 - spec/wukong/wukong_spec.rb
 has_rdoc: 

data/bin/wu-server

@@ -1,70 +0,0 @@
-#!/usr/bin/env ruby
-require 'configliere'
-require 'wukong'
-require 'eventmachine'
-require 'em-synchrony'
-require 'multi_json'
-
-Settings({
-  host: "localhost",
-  port: 9500
-})
-
-Settings.use :commandline
-Settings.resolve!
-
-# Load the file on the command line
-wu_file = ARGV.shift
-load wu_file
-$processor = File.basename(wu_file, '.rb').to_sym
-
-class Wukong::Server
-  def prepare(options = {})
-    dataflow_class = Wukong.dataflow(:server){ send(options[:processor]) }
-    flow_builder = Wukong.registry.retrieve(:server)
-    flow = flow_builder.build(processor: $processor)
-    
-    @buffer = []
-    @processor = flow.stages.values.first
-    @processor.emitter = ->(value){ @buffer << value }
-  end
-  
-  def process(record)
-    @process.process(record)
-  end
-  
-  def cleanup
-  end
-end
-
-
-# EventMachine server
-
-class WukongMachine < EM::Protocols::LineAndTextProtocol
-  def post_init
-    puts "[server] Client connected"
-    @wukong = WukongInterface.new
-    @wukong.prepare(processor: $processor)
-  end
-
-  def receive_data(data)
-    @buffer = []
-    input = MultiJson.load data
-    
-    op = proc { @wukong.process(input) }
-    callback = proc { send_data MultiJson.dump(@buffer) + "\n" }
-    EM.defer(op, callback)
-      
-  rescue MultiJson::DecodeError => ex
-    STDERR.puts "[server] Dropped: Malformed request"
-  end
-
-  def unbind
-    puts "[server] Client disconnected."
-  end
-end
-
-EM::run {
-  EM::start_server(Settings[:host], Settings[:port], WukongMachine)
-  puts "Listening on #{Settings[:host]}:#{Settings[:port]}"
-}

data/lib/wukong/boot.rb

@@ -1,96 +0,0 @@
-module Wukong
-
-  # Boots Wukong, reading any and all relevant `settings` for your
-  # processors.
-  #
-  # This is most useful for any script which wants to run within a
-  # deploy.  See Wukong::Deploy for more details.
-  #
-  # @param [Configliere::Param] settings the relevant settings object that should be configured to boot within a deploy pack
-  def self.boot! settings
-    # First we load the deploy pack's environment, but only if we seem
-    # to be running within a deploy pack.
-    Boot.load_environment        if in_deploy_pack?
-
-    # Next pass the `settings` to the Deploy pack itself to add
-    # options it needs.
-    Deploy.configure(settings) if loaded_deploy_pack?
-
-    # Resolve the `settings` so we can capture all the options we need
-    # from the command line, the environment, &c.
-    settings.resolve!
-
-    # Now boot the deploy pack itself, passing in its known location
-    # and the `settings`
-    Deploy.boot!(settings, Boot.deploy_pack_dir) if loaded_deploy_pack?
-  end
-
-  # Is execution likely happening within a deploy pack?
-  #
-  # See Wukong::Deploy for more information on deploy packs.
-  #
-  # @return [true, false]
-  def self.in_deploy_pack?
-    return @in_deploy_pack unless @in_deploy_pack.nil?
-    @in_deploy_pack = (Boot.deploy_pack_dir != '/')
-  end
-
-  # Have we already loaded the environment of a deploy pack?
-  #
-  # See Wukong::Deploy for more information on deploy packs.
-  #
-  # @return [true, false]
-  def self.loaded_deploy_pack?
-    in_deploy_pack? && defined?(::Wukong::Deploy)
-  end
-
-  # Lets Wukong bootstrap by requiring an enclosing deploy pack's
-  # environment file if available.
-  #
-  # We use a simple heuristic (presence of 'Gemfile' and
-  # 'config/environment.rb' in a non-root parent directory) to
-  # determine whether or not we are in a deploy pack.
-  module Boot
-
-    # Return the directory of the enclosing deploy pack.  Will return
-    # the root ('/') if no deeper directory is identified as a deploy
-    # pack.
-    #
-    # @return [String]
-    def self.deploy_pack_dir
-      return File.dirname(ENV["BUNDLE_GEMFILE"]) if ENV["BUNDLE_GEMFILE"] && is_deploy_pack_dir?(File.dirname(ENV["BUNDLE_GEMFILE"]))
-      return @deploy_pack_dir if @deploy_pack_dir
-      wd     = Dir.pwd
-      parent = File.dirname(wd)
-      until wd == parent
-        return wd if is_deploy_pack_dir?(wd)
-        wd     = parent
-        parent = File.dirname(wd)
-      end
-      @deploy_pack_dir = wd
-    end
-
-    # The default environment file that will be require'd when
-    # booting.
-    #
-    # @return [String]
-    def self.environment_file
-      File.join(deploy_pack_dir, 'config', 'environment.rb')
-    end
-
-    # Load the actual deploy pack environment.  Will not swallow any
-    # load errors.
-    def self.load_environment
-      require environment_file
-    end
-
-    # Could `dir` be a deploy pack dir?
-    #
-    # @param [String] dir
-    # @return [true, false]
-    def self.is_deploy_pack_dir? dir
-      dir && !dir.empty? && File.directory?(dir) && File.exist?(File.join(dir, 'Gemfile')) && File.exist?(File.join(dir, 'config', 'environment.rb'))
-    end
-  end
-  
-end

data/lib/wukong/configuration.rb

@@ -1,8 +0,0 @@
-module Wukong
-  module Local
-
-    # All local configuration for Wukong lives within this object.
-    Configuration = Configliere::Param.new unless defined? Configuration
-    
-  end
-end

data/lib/wukong/emitter.rb

@@ -1,22 +0,0 @@
-module Wukong
-  class Emitter < Hanuman::DirectedLink
-    
-    def initialize(from, into)
-      @from = from
-      @into = into
-      @from.write_attribute(:emitter, self)
-    end
-    
-    def call(record)
-      into.process(record)
-    end
-
-    def to_s
-      "#<#{self.class}(#{from.label} -> #{into.label})>"
-    end
-
-  end
-  
-  Hanuman::LinkFactory.register(:emitter, ->(from_stage, into_stage){ Wukong::Emitter.new(from_stage, into_stage) })
-  
-end