wukong-load 0.0.2 → 0.1.0

data/spec/wukong-load/models/http_request_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Wukong::HttpRequest do
+  context "reporting the best IP address" do
+    let(:real_ip)      { '10.122.122.122' }
+    let(:forwarded_ip) { '10.123.123.123' }
+    context "without an X-Forwarded-For header" do
+      subject { Wukong::HttpRequest.receive(:ip_address => real_ip) }
+      its(:best_ip_address) { should == real_ip }
+    end
+    context "with an X-Forwarded-For header" do
+      subject do
+        Wukong::HttpRequest.receive({
+          :ip_address => real_ip,
+          :headers    => {'X-Forwarded-For' => [forwarded_ip, '10.124.124.124'].join(', ') }
+        })
+      end
+      its(:best_ip_address) { should == forwarded_ip }
+    end
+  end
+end

data/wukong-load.gemspec

@@ -17,14 +17,15 @@ Gem::Specification.new do |gem|
   * MongoDB
   * HBase
   * MySQL
+  * Kafka
 
 and others.
 EOF
 
   gem.files         = `git ls-files`.split("\n")
-  gem.executables   = ['wu-load']
+  gem.executables   = ['wu-load', 'wu-source']
   gem.test_files    = gem.files.grep(/^spec/)
   gem.require_paths = ['lib']
 
-  gem.add_dependency('wukong',      '3.0.0.pre3')
+  gem.add_dependency('wukong', '3.0.0')
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wukong-load
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-18 00:00:00.000000000 Z
+date: 2013-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wukong
@@ -21,7 +21,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0.pre3
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,29 +29,41 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.0.pre3
+        version: 3.0.0
 description: ! "  Lets you load data from the command-line into data stores like\n\n
-  \ * Elasticsearch\n  * MongoDB\n  * HBase\n  * MySQL\n\nand others.\n"
+  \ * Elasticsearch\n  * MongoDB\n  * HBase\n  * MySQL\n  * Kafka\n\nand others.\n"
 email: coders@infochimps.com
 executables:
 - wu-load
+- wu-source
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .yardopts
 - Gemfile
 - LICENSE.md
 - README.md
 - Rakefile
 - bin/wu-load
+- bin/wu-source
 - lib/wukong-load.rb
-- lib/wukong-load/configuration.rb
-- lib/wukong-load/elasticsearch.rb
+- lib/wukong-load/load_runner.rb
 - lib/wukong-load/loader.rb
-- lib/wukong-load/runner.rb
+- lib/wukong-load/loaders/elasticsearch.rb
+- lib/wukong-load/loaders/kafka.rb
+- lib/wukong-load/loaders/mongodb.rb
+- lib/wukong-load/loaders/sql.rb
+- lib/wukong-load/models/http_request.rb
+- lib/wukong-load/source_driver.rb
+- lib/wukong-load/source_runner.rb
 - lib/wukong-load/version.rb
 - spec/spec_helper.rb
-- spec/wukong-load/elasticsearch_spec.rb
+- spec/wukong-load/loaders/elasticsearch_spec.rb
+- spec/wukong-load/loaders/kafka_spec.rb
+- spec/wukong-load/loaders/mongodb_spec.rb
+- spec/wukong-load/loaders/sql_spec.rb
+- spec/wukong-load/models/http_request_spec.rb
 - wukong-load.gemspec
 homepage: https://github.com/infochimps-labs/wukong-load
 licenses:
@@ -80,5 +92,9 @@ specification_version: 3
 summary: Load data produced by Wukong processors and dataflows into data stores.
 test_files:
 - spec/spec_helper.rb
-- spec/wukong-load/elasticsearch_spec.rb
+- spec/wukong-load/loaders/elasticsearch_spec.rb
+- spec/wukong-load/loaders/kafka_spec.rb
+- spec/wukong-load/loaders/mongodb_spec.rb
+- spec/wukong-load/loaders/sql_spec.rb
+- spec/wukong-load/models/http_request_spec.rb
 has_rdoc: 

data/lib/wukong-load/configuration.rb

@@ -1,8 +0,0 @@
-module Wukong
-  module Load
-
-    # All local configuration for Wukong-Load lives within this object.
-    Configuration = Configliere::Param.new unless defined? Configuration
-    
-  end
-end

data/lib/wukong-load/elasticsearch.rb

@@ -1,99 +0,0 @@
-# This should be extracted into Wonderdog and inserted via the Wukong
-# plugin mechanism.
-
-require_relative('loader')
-
-module Wukong
-  module Load
-
-    # Loads data into Elasticsearch
-    class ElasticsearchLoader < Loader
-
-      field :host,          String, :default => 'localhost'
-      field :port,          Integer,:default => 9200
-      field :index,         String, :default => 'wukong'
-      field :es_type,       String, :default => 'streaming_record'
-      field :index_field,   String, :default => '_index'
-      field :es_type_field, String, :default => '_es_type'
-      field :id_field,      String, :default => '_id'
-
-      attr_accessor :connection
-
-      def setup
-        h = host.gsub(%r{^http://},'')
-        log.debug("Connecting to Elasticsearch cluster at #{h}:#{port}...")
-        begin
-          self.connection = Net::HTTP.new(h, port)
-          self.connection.use_ssl = true if host =~ /^https/
-        rescue => e
-          raise Error.new(e.message)
-        end
-      end
-
-      def load record
-        id_for(record) ? request(Net::HTTP::Put, update_path(record), record) : request(Net::HTTP::Post, create_path(record), record)
-      end
-
-      def create_path record
-        File.join('/', index_for(record).to_s, es_type_for(record).to_s)
-      end
-
-      def update_path record
-        File.join('/', index_for(record).to_s, es_type_for(record).to_s, id_for(record).to_s)
-      end
-      
-      def index_for record
-        record[index_field] || self.index
-      end
-
-      def es_type_for record
-        record[es_type_field] || self.es_type
-      end
-
-      def id_for record
-        record[id_field]
-      end
-
-      def request request_type, path, record
-        perform_request(create_request(request_type, path, record))
-      end
-
-      private
-      
-      def create_request request_type, path, record
-        request_type.new(path).tap do |req|
-          req.body = MultiJson.dump(record)
-        end
-      end
-
-      def perform_request req
-        begin
-          response = connection.request(req)
-          status   = response.code.to_i
-          if (200..201).include?(status)
-            log.info("#{req.class} #{req.path} #{status}")
-          else
-            handle_elasticsearch_error(status, response)
-          end
-        rescue => e
-          log.error("#{e.class} - #{e.message}")
-        end
-      end
-
-      def handle_elasticsearch_error response
-        begin
-          error = MultiJson.load(response.body)
-          log.error("#{response.code}: #{error['error']}")
-        rescue => e
-          log.error("Received a response code of #{status}: #{response.body}")
-        end
-      end
-        
-      register :elasticsearch_loader
-      
-    end
-  end
-end
-
-    
-    

data/lib/wukong-load/runner.rb

@@ -1,48 +0,0 @@
-module Wukong
-  module Load
-    class Runner
-
-      include Logging
-
-      def self.run settings
-        begin
-          new(settings).run
-        rescue Error => e
-          log.error(e.message)
-          exit(127)
-        end
-      end
-
-      attr_accessor :settings
-      def initialize settings
-        self.settings = settings
-      end
-
-      def args
-        settings.rest
-      end
-
-      def data_store_name
-        args.first
-      end
-
-      def processor_name
-        case data_store_name
-        when 'elasticsearch' then :elasticsearch_loader
-        when nil
-          settings.dump_help
-          exit(1)
-        else
-          raise Error.new("No loader defined for data store: #{data_store_name}")
-        end
-      end
-      
-      def run
-        EM.run do
-          StupidServer.new(processor_name, settings).run!
-        end
-      end
-      
-    end
-  end
-end

data/spec/wukong-load/elasticsearch_spec.rb

@@ -1,140 +0,0 @@
-require 'spec_helper'
-
-describe Wukong::Load::ElasticsearchLoader do
-
-  let(:record)                     { {'text' => 'hi'                                        } }
-  let(:record_with_index)          { {'text' => 'hi', '_index'          => 'custom_index'   } }
-  let(:record_with_custom_index)   { {'text' => 'hi', '_custom_index'   => 'custom_index'   } }
-  let(:record_with_es_type)        { {'text' => 'hi', '_es_type'        => 'custom_es_type' } }
-  let(:record_with_custom_es_type) { {'text' => 'hi', '_custom_es_type' => 'custom_es_type' } }
-  let(:record_with_id)             { {'text' => 'hi', '_id'             => 'the_id'         } }
-  let(:record_with_custom_id)      { {'text' => 'hi', '_custom_id'      => 'the_id'         } }
-
-  it_behaves_like 'a processor', :named => :elasticsearch_loader
-
-  context "without an Elasticsearch available" do
-    before do
-      Net::HTTP.should_receive(:new).and_raise(StandardError)
-    end
-
-    it "raises an error on setup" do
-      expect { processor(:elasticsearch_loader).setup }.to raise_error(Wukong::Error)
-    end
-  end
-
-  context "routes" do
-    context "all records" do
-      it "to a default index" do
-        proc = processor(:elasticsearch_loader)
-        proc.index_for(record).should == proc.index
-      end
-      it "to a given index" do
-        processor(:elasticsearch_loader, :index => 'custom_index').index_for(record).should == 'custom_index'
-      end
-      it "to a default type" do
-        proc = processor(:elasticsearch_loader)
-        proc.es_type_for(record).should == proc.es_type
-      end
-      it "to a given type" do
-        processor(:elasticsearch_loader, :es_type => 'custom_es_type').es_type_for(record).should == 'custom_es_type'
-      end
-    end
-    
-    context "records having a value for" do
-      it "default index field to the given index" do
-        processor(:elasticsearch_loader).index_for(record_with_index).should == 'custom_index'
-      end
-      it "given index field to the given index" do
-        processor(:elasticsearch_loader, :index_field => '_custom_index').index_for(record_with_custom_index).should == 'custom_index'
-      end
-      it "default type field to the given type" do
-        processor(:elasticsearch_loader).es_type_for(record_with_es_type).should == 'custom_es_type'
-      end
-      it "given type field to the given type" do
-        processor(:elasticsearch_loader, :es_type_field => '_custom_es_type').es_type_for(record_with_custom_es_type).should == 'custom_es_type'
-      end
-    end
-  end
-
-  context "detects IDs" do
-    it "based on the absence of a default ID field" do
-      processor(:elasticsearch_loader).id_for(record).should be_nil
-    end
-    it "based on the value of a default ID field" do
-      processor(:elasticsearch_loader).id_for(record_with_id).should == 'the_id'
-    end
-    it "based on the value of a custom ID field" do
-      processor(:elasticsearch_loader, :id_field => '_custom_id').id_for(record_with_custom_id).should == 'the_id'
-    end
-  end
-
-  context "having made a connection to the database" do
-    
-    let(:connection) { double()                         }
-    let(:log)        { double()                         }
-    subject          { processor(:elasticsearch_loader) }
-    before           do
-      Net::HTTP.should_receive(:new).and_return(connection)
-      subject.stub!(:log).and_return(log)
-    end
-    
-
-    context "sends" do
-      it "create requests on a record without an ID" do
-        subject.should_receive(:request).with(Net::HTTP::Post, '/foo/bar', kind_of(Hash))
-        subject.load({'_index' => 'foo', '_es_type' => 'bar'})
-      end
-      it "update requests on a record with an ID" do
-        subject.should_receive(:request).with(Net::HTTP::Put, '/foo/bar/1', kind_of(Hash))
-        subject.load({'_index' => 'foo', '_es_type' => 'bar', '_id' => '1'})
-      end
-    end
-
-    context "receives" do
-      
-      let(:ok) do
-        mock("Net::HTTPOK").tap do |response|
-          response.stub!(:code).and_return('200')
-          response.stub!(:body).and_return('{"ok": true}')
-        end
-      end
-      let(:created) do
-        mock("Net::HTTPCreated").tap do |response|
-          response.stub!(:code).and_return('201')
-          response.stub!(:body).and_return('{"created": true}')
-        end
-      end
-      let(:not_found) do
-        mock("Net::HTTPNotFound").tap do |response|
-          response.stub!(:code).and_return('404')
-          response.stub!(:body).and_return('{"error": "Not found"}')
-        end
-      end
-      
-      context "201 Created" do
-        before { connection.should_receive(:request).with(kind_of(Net::HTTP::Post)).and_return(created) }
-        it "by logging an INFO message" do
-          log.should_receive(:info)
-          subject.load(record)
-        end
-      end
-
-      context "200 OK" do
-        before { connection.should_receive(:request).with(kind_of(Net::HTTP::Put)).and_return(ok) }
-        it "by logging an INFO message" do
-          log.should_receive(:info)
-          subject.load(record_with_id)
-        end
-      end
-
-      context "an error response from Elasticsearch" do
-        before { connection.should_receive(:request).with(kind_of(Net::HTTP::Post)).and_return(not_found) }
-        it "by logging an ERROR message" do
-          log.should_receive(:error)
-          subject.load(record)
-        end
-      end
-      
-    end
-  end
-end