hadoop-papyrus 0.0.6

data/lib/mapred_factory.rb

@@ -0,0 +1,54 @@
+require 'hadoop_dsl'
+
+module HadoopDsl
+  class MapRedFactory
+    def self.dsl_name(script)
+      HadoopDsl.read_file(script).each_line do |line|
+        dsl_name = $1 if line =~ /\s*dsl\s*\(?["'](\w*)["']\)?/
+        return dsl_name if dsl_name
+      end
+    end
+
+    def self.require_dsl_lib(dsl_name)
+      require HadoopDsl.snake_case(dsl_name)
+    end
+  end
+
+  class MapperFactory < MapRedFactory
+    # for cache in map loop
+    @@mapper_class = nil
+    def self.create(script, key, value)
+      # once decide in map loop
+      unless @@mapper_class
+        dsl_name = self.dsl_name(script)
+        require_dsl_lib(dsl_name)
+        @@mapper_class = eval("HadoopDsl::#{dsl_name}::#{dsl_name}Mapper")
+      end
+
+      @@mapper_class.new(script, key, value)
+    end
+  end
+
+  class ReducerFactory < MapRedFactory
+    @@reducer_class = nil
+    def self.create(script, key, values)
+      # once decide in reduce loop
+      unless @@reducer_class
+        dsl_name = self.dsl_name(script)
+        require_dsl_lib(dsl_name)
+        @@reducer_class = eval("HadoopDsl::#{dsl_name}::#{dsl_name}Reducer")
+      end
+
+      @@reducer_class.new(script, key, values)
+    end
+  end
+
+  class SetupFactory < MapRedFactory
+    def self.create(script, conf)
+      dsl_name = self.dsl_name(script)
+      require_dsl_lib(dsl_name)
+      setup_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Setup" 
+      eval(setup_class).new(script, conf) rescue HadoopDsl::BaseSetup.new(script, conf)
+    end
+  end
+end

data/lib/util.rb

@@ -0,0 +1,38 @@
+# utility functions
+require 'hadoop_dsl'
+
+module HadoopDsl
+  # file body cache
+  # reading file in map/reduce cause critical issues!
+  @@file_bodies = {}
+
+  def self.snake_case(str)
+    str.gsub(/\B[A-Z]/, '_\&').downcase
+  end
+
+  def self.read_file(file_name)
+    # use if cached
+    body = @@file_bodies[file_name] if @@file_bodies[file_name]
+
+    # read as usual
+    body = File.open(file_name).read rescue nil unless body
+
+    # read from loadpath
+    unless body
+      $:.each do |path|
+        body = File.open(File.join(path, file_name)).read rescue next
+        break
+      end
+    end
+
+    raise "cannot find file - #{file_name}" unless body
+
+    # for cache
+    @@file_bodies[file_name] = body
+    body
+  end
+
+  def self.reset_dsl_file
+    @@file_bodies = {}
+  end
+end

data/lib/word_count.rb

@@ -0,0 +1,56 @@
+require 'hadoop_dsl'
+require 'enumerator'
+
+module HadoopDsl::WordCount
+  MODEL_METHODS = []
+  TOTAL_PREFIX = "\t"
+
+  # controller
+  class WordCountMapper < HadoopDsl::BaseMapper
+    def initialize(script, key, value)
+      super(script, WordCountMapperModel.new(key, value))
+    end
+
+    # model methods
+    def_delegators :@model, *MODEL_METHODS
+
+    # emitters
+    def count_uniq
+      @model.value.split.each {|word| emit(word => 1)}
+    end
+
+    def total(*types)
+      types.each do |type|
+        case type
+        when :bytes
+          emit("#{TOTAL_PREFIX}total bytes" => @model.value.gsub(/\s/, '').length)
+        when :words
+          emit("#{TOTAL_PREFIX}total words" => @model.value.split.size)
+        when :lines
+          emit("#{TOTAL_PREFIX}total lines" => 1)
+        end
+      end
+    end
+  end
+
+  class WordCountReducer < HadoopDsl::BaseReducer
+    def initialize(script, key, values)
+      super(script, WordCountReducerModel.new(key, values))
+    end
+
+    # model methods
+    def_delegators :@model, *MODEL_METHODS
+
+    # emitters
+    def count_uniq; aggregate unless @model.total_value? end
+    def total(*types); aggregate if @model.total_value? end
+  end
+
+  # model
+  class WordCountMapperModel < HadoopDsl::BaseMapperModel
+  end
+
+  class WordCountReducerModel < HadoopDsl::BaseReducerModel
+    def total_value?; @key =~ /^#{TOTAL_PREFIX}/ end
+  end
+end

data/spec/client_spec.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'hadoop_dsl_client'
+
+describe HadoopDsl::Client do
+  before do
+    @client = HadoopDsl::Client.new(["examples/wordcount.rb", "in", "out"])
+  end
+
+  it 'can parse args' do
+    @client.files.join.should match /ruby_wrapper\.rb/
+    @client.files.join.should match /dsl_init\.rb/
+    @client.files.should include 'examples/wordcount.rb'
+    @client.inputs.should == 'in'
+    @client.outputs.should == 'out'
+  end
+
+  it 'can add dsl file into mapred args' do
+    @client.mapred_args.should ==
+      "--script dsl_init.rb in out --dslfile wordcount.rb"
+  end
+
+  it 'can add dsl lib files' do
+    lib_path = HadoopDsl.lib_path
+    @client.files.should include File.join(lib_path, 'core.rb')
+    @client.files.should include File.join(lib_path, 'log_analysis.rb')
+  end
+end

data/spec/core_spec.rb

@@ -0,0 +1,73 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'core'
+
+include HadoopDsl
+
+describe 'BaseMapRed' do
+  before(:all) do
+    @script = create_tmp_script(<<-EOF)
+from 'test/inputs'
+to 'test/outputs'
+    EOF
+  end
+
+  it 'emit key value' do
+    mapper = BaseMapper.new(@script, BaseMapperModel.new(nil, nil))
+    mapper.emit('key' => 'value')
+    mapper.emitted.should == [{'key' => 'value'}]
+  end
+
+  it 'can run BaseMapper in minimum' do
+    model = BaseMapperModel.new('key', 'value')
+    mapper = BaseMapper.new(@script, model)
+    mapper.run
+  end
+
+  it 'can run BaseReducer in minimum' do
+    model = BaseReducerModel.new('key', 'values')
+    reducer = BaseReducer.new(@script, model)
+    reducer.run
+  end
+
+  it 'can run BaseSetup in minimum' do
+    setup = BaseSetup.new(@script, nil)
+    setup.run
+  end
+
+  describe BaseMapper do
+    it 'can emit as identity' do
+      model = BaseMapperModel.new('key', 'value')
+      mapper = BaseMapper.new(@script, model)
+      mapper.identity
+
+      mapper.emitted.should == [{'key' => 'value'}] 
+    end
+  end
+
+  describe BaseReducer do
+    it 'can emit as aggregate' do
+      model = BaseReducerModel.new('key', [1, 2, 3])
+      reducer = BaseReducer.new(@script, model)
+      reducer.aggregate
+
+      reducer.emitted.should == [{'key' => 6}] 
+    end
+
+    it 'can emit as identity' do
+      model = BaseReducerModel.new('key', [1, 2, 3])
+      reducer = BaseReducer.new(@script, model)
+      reducer.identity
+
+      reducer.emitted.should == [{'key' => 1}, {'key' => 2}, {'key' => 3}] 
+    end
+  end
+
+  describe BaseSetup do
+    it 'can get paths' do
+      setup = BaseSetup.new(@script, nil)
+      setup.run
+      setup.paths[0].should == 'test/inputs'
+      setup.paths[1].should == 'test/outputs'
+    end
+  end
+end

data/spec/dsl_init_spec.rb

@@ -0,0 +1,49 @@
+require 'dsl_init'
+
+describe 'mapreduce init' do
+
+  before(:all) do
+    @script = create_tmp_script(<<-EOF)
+dsl 'LogAnalysis'
+data 'test' do
+  from 'test/inputs'
+  to 'test/outputs'
+
+  separate(" ")
+  column_name 'c0', 'c1', 'c2', 'c3'
+  topic 't1' do
+    count_uniq columns(:c1)
+  end
+end
+    EOF
+  end
+
+  before do
+    @one = 1
+    @output = mock('output')
+  end
+
+  it 'can map sucessfully' do
+    key = 'key'
+    value = 'it should be fine'
+    @output.should_receive(:collect).once #.with(@text, @one)
+
+    map(key, value, @output, nil, @script)
+  end
+
+  it 'can reduce sucessfully' do
+    key = "t1\tkey"
+    values = [@one, @one, @one]
+    @output.should_receive(:collect).once #.with(@text, @one)
+
+    reduce(key, values, @output, nil, @script)
+  end
+
+  it 'can set job conf' do
+    conf = mock('jobconf')
+    paths = setup(conf, @script)
+
+    paths[0].should == 'test/inputs'
+    paths[1].should == 'test/outputs'
+  end
+end

data/spec/example_spec.rb

@@ -0,0 +1,84 @@
+require 'log_analysis'
+require 'word_count'
+require 'hive_like'
+
+include HadoopDsl::LogAnalysis
+describe 'Aapach Log Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'log_analysis_test.rb')
+    @bot_ua = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+    @value = %Q!127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "-" "#{@bot_ua}"!
+  end
+
+  it 'can run example by mapper' do
+    mapper = LogAnalysisMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.first.should == {"ua\t#{@bot_ua}" => 1}
+  end
+
+  it 'can run example by reducer' do
+    reducer = LogAnalysisReducer.new(@script, "ua\tChrome", [1, 1, 1])
+    reducer.run
+    reducer.emitted.first["ua\tChrome"].should == 3
+  end
+end
+
+include HadoopDsl::WordCount
+describe 'Word Count Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'word_count_test.rb')
+    @value = 'Lorem ipsum ipsum Lorem sit amet,'
+  end
+
+  it 'can run example by mapper' do
+    mapper = WordCountMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.size.should == 9
+    mapper.emitted.each do |e|
+      case e.keys.first
+      when 'Lorem'
+        e.values.first.should == 1
+      when 'total words'
+        e.values.first.should == 6
+      end
+    end
+  end
+
+  it 'can run example by reducer' do
+    reducer = WordCountReducer.new(@script, "Lorem", [1, 1, 1])
+    reducer.run
+    reducer.emitted.first["Lorem"].should == 3
+  end
+end
+
+include HadoopDsl::HiveLike
+describe 'Hive Like Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'hive_like_test.rb')
+    @value = 'apple, 3, 100'
+  end
+
+  it 'can run setup' do
+    conf = mock('conf')
+    conf.should_receive(:output_key_class=).once
+    conf.should_receive(:output_value_class=).once
+
+    setup = HiveLikeSetup.new(@script, conf)
+    setup.run
+    setup.paths[0].should == 'hive-like/items.txt'
+  end
+
+  it 'can run example by mapper' do
+    mapper = HiveLikeMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.size.should == 1
+    mapper.emitted.first['items'].should == '3, 100, apple'
+  end
+
+  it 'can run example by reducer' do
+    values = ['v1', 'v2', 'v3']
+    reducer = HiveLikeReducer.new(@script, "items", values)
+    reducer.run
+    reducer.emitted.first["items"].should == 'v1'
+  end
+end

data/spec/hive_like_spec.rb

@@ -0,0 +1,57 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'hive_like'
+
+include HadoopDsl::HiveLike
+
+describe HiveLikeSetup do
+  it 'should load data' do
+    script = create_tmp_script(%Q!load_data "hive-like/inputs", items;!)
+    conf = mock('conf')
+    conf.should_receive(:output_key_class=).once
+    conf.should_receive(:output_value_class=).once
+
+    setup = HiveLikeSetup.new(script, conf)
+    setup.run
+    setup.paths[0].should == 'hive-like/inputs'
+    setup.paths[1].should == 'hive-like/outputs'
+  end
+end
+
+describe HiveLikeMapper do
+  before do
+    @value = 'apple, 3, 100'
+  end
+
+  it 'should create table' do
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    mapper.create_table('items', 'item', 'STRING', 'quantity', 'INT', 'price', 'INT');
+    mapper.table.name.should == 'items'
+    mapper.table.column(0).should == 'item'
+    mapper.table.column(1).should == 'quantity'
+  end
+
+  it 'should select' do
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    mapper.create_table('items', 'item', 'STRING', 'quantity', 'INT', 'price', 'INT');
+    mapper.select("item", "quantity", "price", "from", "items")
+    mapper.emitted.first.should == {'items' => 'apple, 3, 100'}
+  end
+
+  it 'should pre process script body' do
+    body = "select foo, bar from table;\n"
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    processed = mapper.pre_process(body)
+    processed.should == %Q!select("foo", "bar", "from", "table")\n!
+  end
+end
+
+describe HiveLikeReducer do
+  it 'should select as identity' do
+    key = 'Lorem'
+    values = [1, 1, 1]
+    reducer = HiveLikeReducer.new(nil, key, values)
+
+    reducer.select
+    reducer.emitted[0].should == {'Lorem' => 1}
+  end
+end