mongo-hadoop 1.0.0

data/lib/mongo-hadoop.rb

@@ -0,0 +1,5 @@
+require 'bson'
+require 'mongo-hadoop/input'
+require 'mongo-hadoop/output'
+require 'mongo-hadoop/mapper'
+require 'mongo-hadoop/reducer'

data/lib/mongo-hadoop/input.rb

@@ -0,0 +1,29 @@
+class BSONInput
+  include Enumerable
+  
+  def initialize(stream=nil)
+    @stream = stream || $stdin
+  end
+  
+  def read
+    begin
+      BSON.read_bson_document(@stream)
+    rescue NoMethodError
+      nil
+    end
+  end
+
+  def each
+    while(doc = read)
+      yield doc
+    end
+  end
+end
+
+class BSONKeyValueInput < BSONInput
+  def each
+    while(doc = read)
+      yield doc['_id'], doc
+    end
+  end
+end

data/lib/mongo-hadoop/mapper.rb

@@ -0,0 +1,29 @@
+class MongoHadoop
+  def self.map
+    input = BSONInput.new
+    output = BSONOutput.new
+
+    input.each do |doc|
+      mapped = yield doc
+      mapped = [mapped] unless mapped.respond_to?(:each)
+
+      mapped.each do |mapped|
+        output.write mapped if mapped
+      end
+    end
+  end
+
+  def self.kvmap
+    kvinput = BSONKeyValueInput.new
+    kvoutput = BSONKeyValueOutput.new
+
+    kvinput.each do |key, value|
+      mapped = yield key, value
+      mapped = [mapped] unless profiles.respond_to(:each)
+
+      mapped.each do |mapped|
+        kvoutput.write mapped if mapped
+      end
+    end
+  end
+end

data/lib/mongo-hadoop/output.rb

@@ -0,0 +1,22 @@
+class BSONOutput
+  def initialize(stream=nil)
+    @stream = stream || $stdout
+  end
+  
+  def write(doc)
+    bson_doc = BSON.serialize(doc)
+    @stream.write(bson_doc)
+    @stream.flush
+  end
+end
+
+class BSONKeyValueOutput < BSONOutput
+  def write(pair)
+    key, value = *pair
+
+    doc = value.is_a?(Hash) ? value : { :value => value }
+
+    doc['_id'] = key
+    super(doc)
+  end
+end

data/lib/mongo-hadoop/reducer.rb

@@ -0,0 +1,28 @@
+class MongoHadoop
+  def self.reduce
+    input = BSONInput.new
+    output = BSONOutput.new
+    
+    grouped = input.group_by { |doc| doc['_id'] }
+
+    grouped.each do |key, values|
+      output.write yield key, values
+    end
+  end
+
+  def self.kvreduce
+    kvinput = BSONKeyValueInput.new
+    kvoutput = BSONKeyValueOutput.new
+
+    grouped = kvinput.inject(Hash.new) do |hash, pair|
+      key, value = *pair
+      hash[key] ||= []
+      hash[key] << value
+      hash
+    end
+
+    grouped.each do |key, values|
+      kvoutput.write yield key, values
+    end
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: mongo-hadoop
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Tyler Brock
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bson
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby MongoDB Hadoop streaming support
+email: tyler.brock@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/mongo-hadoop.rb
+- lib/mongo-hadoop/input.rb
+- lib/mongo-hadoop/output.rb
+- lib/mongo-hadoop/mapper.rb
+- lib/mongo-hadoop/reducer.rb
+homepage: http://github.com/mongodb/mongo-hadoop
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: MongoDB Hadoop streaming support
+test_files: []