hadoop-rubydsl 0.0.1

data/spec/init_spec.rb

@@ -0,0 +1,56 @@
+require 'java'
+require 'init'
+
+import 'org.apache.hadoop.io.IntWritable'
+import 'org.apache.hadoop.io.Text'
+import 'org.apache.hadoop.mapred.JobConf'
+
+describe 'mapreduce init' do
+
+  before(:all) do
+    @script = create_tmp_script(<<-EOF)
+use 'LogAnalysis'
+data 'test' do
+  from 'test/inputs'
+  to 'test/outputs'
+
+  separate(" ")
+  column_name 'c0', 'c1', 'c2', 'c3'
+  topic 't1' do
+    count_uniq columns(:c1)
+  end
+end
+    EOF
+  end
+
+  before do
+    @one = IntWritable.new(1)
+    @output = mock('output')
+  end
+
+  it 'can map sucessfully' do
+    key, value  = Text.new, Text.new
+    key.set("key")
+    value.set('it should be fine')
+    @output.should_receive(:collect).once #.with(@text, @one)
+
+    map(key, value, @output, nil, @script)
+  end
+
+  it 'can reduce sucessfully' do
+    key, value = Text.new, Text.new
+    key.set("t1\tkey")
+    values = [@one, @one, @one]
+    @output.should_receive(:collect).once #.with(@text, @one)
+
+    reduce(key, values, @output, nil, @script)
+  end
+
+  it 'can set job conf' do
+    conf = JobConf.new
+    paths = setup(conf, @script)
+
+    paths[0].should == 'test/inputs'
+    paths[1].should == 'test/outputs'
+  end
+end

data/spec/log_analysis_spec.rb

@@ -0,0 +1,119 @@
+require 'init'
+require 'core'
+require 'log_analysis'
+
+include HadoopDsl::LogAnalysis
+
+describe LogAnalysisMapper do
+  before do
+    @apache_log = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326'
+  end
+  
+  it 'should separate data by space' do
+    value = 'Lorem ipsum dolor sit amet,'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+
+    mapper.column[1].value.should == 'ipsum'
+  end
+
+  it 'should separate by pattern' do
+    mapper = LogAnalysisMapper.new(nil, nil, @apache_log)
+    mapper.pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/
+
+    mapper.column[2].value.should == 'frank'
+  end
+
+  it 'should label column name by string' do
+    mapper = LogAnalysisMapper.new(nil, nil, @apache_log)
+    mapper.pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/
+    mapper.column_name 'remote_host', PASS, 'user', 'access_date', 'request', 'status', 'bytes'
+
+    mapper.column['user'].value.should == 'frank'
+  end
+
+  it 'should label column name by symbol' do
+    mapper = LogAnalysisMapper.new(nil, nil, @apache_log)
+    mapper.pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/
+    mapper.column_name :remote_host, PASS, :user, :access_date, :request, :status, :bytes
+
+    mapper.column[:user].value.should == 'frank'
+  end
+
+  it 'should count uniq column' do
+    value = 'count uniq'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+    mapper.topic('t1') { mapper.count_uniq mapper.column[1] }
+
+    mapper.emitted.first["t1\tuniq"].should == 1
+  end
+
+  it 'should sum column value' do
+    value = 'sum 123'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+    mapper.topic('t1') { mapper.sum mapper.column[1] }
+
+    mapper.emitted.first["t1"].should == 123
+  end
+
+  it 'has topic which returns label' do
+    value = 'Lorem ipsum dolor sit amet,'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+
+    topic = mapper.topic('desc', :label => 'label')
+    topic.label.should == 'label'
+  end
+  
+  it 'has topic which returns label as desc' do
+    value = 'Lorem ipsum dolor sit amet,'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+
+    topic = mapper.topic('desc')
+    topic.label.should == 'desc'
+  end
+
+  it 'has topic which returns label as desc with space' do
+    value = 'Lorem ipsum dolor sit amet,'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+
+    topic = mapper.topic('desc with space')
+    topic.label.should == 'desc_with_space'
+  end
+end
+
+describe LogAnalysisReducer do
+  it 'should count uniq in the topic' do
+    key = "t1\tuniq"
+    values = [1, 1, 1]
+    reducer = LogAnalysisReducer.new(nil, key, values)
+    reducer.separate(' ')
+    reducer.topic('t1') { reducer.count_uniq(nil) }
+
+    reducer.emitted.first["t1\tuniq"].should == 3
+  end
+
+  it 'should not count uniq of other topic' do
+    key = "t2\tuniq"
+    values = [1, 1, 1]
+    reducer = LogAnalysisReducer.new(nil, key, values)
+    reducer.separate(' ')
+    reducer.topic('t1') { reducer.count_uniq(nil) }
+
+    reducer.emitted.first.should be_nil
+  end
+
+  it 'should sum column value' do
+    key = "t1"
+    values = [123, 456, 789]
+    reducer = LogAnalysisReducer.new(nil, key, values)
+    reducer.separate(' ')
+    reducer.topic('t1') { reducer.sum(nil) }
+
+    reducer.emitted.first["t1"].should == 123+456+789
+  end
+end

data/spec/mapred_factory_spec.rb

@@ -0,0 +1,42 @@
+require File.join(File.dirname(__FILE__) , 'spec_helper')
+
+require 'mapred_factory'
+require 'log_analysis'
+
+describe 'MapRed Factory' do
+
+  before(:all) do
+    @script = create_tmp_script("use 'LogAnalysis'")
+  end
+
+  it 'can create mapper' do
+    mapper = MapperFactory.create(@script, nil, nil)
+    mapper.class.should == LogAnalysisMapper
+  end
+
+  it 'can create reducer' do
+    reducer = ReducerFactory.create(@script, nil, nil)
+    reducer.class.should == LogAnalysisReducer
+  end
+
+  it 'can create setup' do
+    s = SetupFactory.create(@script, nil)
+    s.class.should == LogAnalysisSetup
+  end
+
+  it 'can create base if not exists in specific DSL' do
+    s = SetupFactory.create(create_tmp_script("use 'WordCount'"), nil)
+    s.class.should == BaseSetup
+  end
+
+  it 'specify dsl name from script' do
+    dsl_name = MapRedFactory.dsl_name(@script)
+    dsl_name.should == 'LogAnalysis'
+  end
+
+  it 'can convert dsl name to dsl lib file and require' do
+    dsl_name = MapRedFactory.dsl_name(@script)
+    MapRedFactory.require_dsl_lib(dsl_name).should_not be_nil
+    LogAnalysisMapper
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+# spec helper
+
+require 'tempfile'
+
+def create_tmp_script(body)
+  tmp = Tempfile.new('test.rb')
+  tmp.print body
+  tmp.close
+  tmp.path
+end
+

data/spec/util_spec.rb

@@ -0,0 +1,15 @@
+require File.join(File.dirname(__FILE__) , 'spec_helper')
+
+require 'util'
+
+describe 'utilities' do
+  it 'can change camelcase str to snakecase' do
+    snake_case('CamelCaseStr').should == 'camel_case_str'
+  end
+
+  it 'can read file and get file data to string' do
+    script_body = 'This is a script body.'
+    @script = create_tmp_script(script_body)
+    read_file(@script).should == script_body
+  end
+end

data/spec/word_count_spec.rb

@@ -0,0 +1,89 @@
+require 'init'
+require 'core'
+require 'word_count'
+
+include HadoopDsl::WordCount
+
+describe WordCountMapper do
+  it 'should count uniq' do
+    value = 'Lorem ipsum Lorem sit amet,'
+    mapper = WordCountMapper.new(nil, nil, value)
+
+    mapper.count_uniq
+    mapper.emitted[0].should == {'Lorem' => 1}
+    mapper.emitted[1].should == {'ipsum' => 1}
+    mapper.emitted[2].should == {'Lorem' => 1}
+  end
+
+  it 'should count total bytes' do
+    value = 'Lorem ipsum Lorem sit amet,'
+    mapper = WordCountMapper.new(nil, nil, value)
+
+    mapper.total :bytes
+    mapper.emitted[0].should == {"#{TOTAL_PREFIX}total bytes" => 23}
+  end
+
+  it 'should count total words' do
+    value = 'Lorem ipsum Lorem sit amet,'
+    mapper = WordCountMapper.new(nil, nil, value)
+
+    mapper.total :words
+    mapper.emitted[0].should == {"#{TOTAL_PREFIX}total words" => 5}
+  end
+
+  it 'should count total lines' do
+    value = 'Lorem ipsum Lorem sit amet,'
+    mapper = WordCountMapper.new(nil, nil, value)
+
+    mapper.total :lines
+    mapper.emitted[0].should == {"#{TOTAL_PREFIX}total lines" => 1}
+  end
+
+  it 'should count total bytes, words, lines' do
+    value = 'Lorem ipsum Lorem sit amet,'
+    mapper = WordCountMapper.new(nil, nil, value)
+
+    mapper.total :bytes, :words, :lines
+    mapper.emitted[0].should == {"#{TOTAL_PREFIX}total bytes" => 23}
+    mapper.emitted[1].should == {"#{TOTAL_PREFIX}total words" => 5}
+    mapper.emitted[2].should == {"#{TOTAL_PREFIX}total lines" => 1}
+  end
+end
+
+describe WordCountReducer do
+  it 'should count uniq' do
+    key = 'Lorem'
+    values = [1, 1, 1]
+    reducer = WordCountReducer.new(nil, key, values)
+
+    reducer.count_uniq
+    reducer.emitted[0].should == {'Lorem' => 3}
+  end
+
+  it 'should count total bytes' do
+    key = "#{TOTAL_PREFIX}total bytes"
+    values = [12, 23, 45]
+    reducer = WordCountReducer.new(nil, key, values)
+
+    reducer.total :bytes
+    reducer.emitted[0].should == {"#{TOTAL_PREFIX}total bytes" => 12 + 23 + 45}
+  end
+
+  it 'should count total words' do
+    key = "#{TOTAL_PREFIX}total words"
+    values = [3, 4, 5]
+    reducer = WordCountReducer.new(nil, key, values)
+
+    reducer.total :words
+    reducer.emitted[0].should == {"#{TOTAL_PREFIX}total words" => 3 + 4 + 5}
+  end
+
+  it 'should count total lines' do
+    key = "#{TOTAL_PREFIX}total lines"
+    values = [1, 2, 3]
+    reducer = WordCountReducer.new(nil, key, values)
+
+    reducer.total :lines
+    reducer.emitted[0].should == {"#{TOTAL_PREFIX}total lines" => 6}
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification 
+name: hadoop-rubydsl
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Koichi Fujikawa
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-26 00:00:00 +09:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: jruby-on-hadoop
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Hadoop Ruby DSL
+email: fujibee@gmail.com
+executables: 
+- hadoop
+- hadoop-ruby.sh
+extensions: []
+
+extra_rdoc_files: 
+- README
+- TODO
+files: 
+- README
+- Rakefile
+- TODO
+- VERSION
+- bin/hadoop
+- bin/hadoop-ruby.sh
+- conf/hadoop-site.xml
+- examples/apachelog-v2-2.rb
+- examples/apachelog-v2.rb
+- examples/apachelog.rb
+- examples/hive_like_test.rb
+- examples/word_count_test.rb
+- hadoop-rubydsl.gemspec
+- lib/core.rb
+- lib/hive_like.rb
+- lib/init.rb
+- lib/java/.gitignore
+- lib/java/hadoop-ruby.jar
+- lib/log_analysis.rb
+- lib/mapred_factory.rb
+- lib/util.rb
+- lib/word_count.rb
+has_rdoc: true
+homepage: http://github.com/fujibee/hadoop-rubydsl
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Hadoop Ruby DSL
+test_files: 
+- spec/spec_helper.rb
+- spec/core_spec.rb
+- spec/util_spec.rb
+- spec/mapred_factory_spec.rb
+- spec/word_count_spec.rb
+- spec/hive_like_spec.rb
+- spec/log_analysis_spec.rb
+- spec/example_spec.rb
+- spec/init_spec.rb
+- examples/apachelog-v2.rb
+- examples/hive_like_test.rb
+- examples/word_count_test.rb
+- examples/apachelog-v2-2.rb
+- examples/apachelog.rb