hadoop-rubydsl 0.0.1

data/lib/hive_like.rb

@@ -0,0 +1,122 @@
+require 'core'
+require 'enumerator'
+
+module HadoopDsl::HiveLike
+  include HadoopDsl
+  
+  AVAILABLE_METHODS = [:select, :create_table, :table]
+
+  # common
+  module HiveLikeMapRed
+    def pre_process(body)
+      processed = ""
+      body.each do |line|
+        next if line =~ /^#/
+        if line =~ /^(\w*)\s+(.*);$/
+          method = $1
+          args = sprit_and_marge_args($2)
+          processed << "#{method}(#{args})\n"
+        else 
+          processed << line + "\n"
+        end
+      end
+      processed
+    end
+
+    def sprit_and_marge_args(raw)
+      raw.gsub(/[\(\)]/, ' ').split.map do |s|
+        stripped = s.gsub(/[\s,"']/, '')
+        %Q!"#{stripped}"!
+      end.join(", ")
+    end
+  end
+
+  # controller
+  class HiveLikeSetup < BaseSetup
+    def load_data(inputs, table)
+      @from = inputs
+      @to = inputs.gsub(/#{File.basename(inputs)}$/, 'outputs')
+    end
+    
+    def output_format
+      @conf.output_key_class = Text
+      @conf.output_value_class = Text
+    end
+
+    # might not need but occur error if not exists
+    def select(*args) end
+
+    include HiveLikeMapRed
+  end
+
+  class HiveLikeMapper < BaseMapper
+    def initialize(script, key, value)
+      super(script, HiveLikeMapperModel.new(key, value))
+    end
+
+    include HiveLikeMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  class HiveLikeReducer < BaseReducer
+    def initialize(script, key, values)
+      super(script, HiveLikeReducerModel.new(key, values))
+    end
+
+    include HiveLikeMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  # model
+  class HiveLikeMapperModel < BaseMapperModel
+    attr_reader :table
+
+    def initialize(key, value)
+      super(key, value)
+    end
+
+    # emitters
+    def create_table(name, *column_and_type)
+      @table = Table.new(name)
+      column_and_type.each_with_index do |column, index|
+        next if index % 2 != 0 # type
+        @table.columns << column_and_type[index]
+      end
+    end
+
+    def select(*args)
+      from_index = args.index('from')
+      if from_index
+        values = args[0...from_index].map do |column|
+          splitted = @value.split(/[,\s]+/)
+          splitted[@table.columns.index(column)]
+        end
+        @controller.emit(args[from_index + 1] => values.join(", "))
+      end
+    end
+
+    class Table
+      attr_reader :name, :columns
+
+      def initialize(name)
+        @name = name
+        @columns = []
+      end
+
+      def column(index) @columns[index] end
+    end
+  end
+
+  class HiveLikeReducerModel < BaseReducerModel
+    def initialize(key, values)
+      super(key, values)
+    end
+
+    # emitters
+    def select(*args) identity end
+  end
+end

data/lib/init.rb

@@ -0,0 +1,60 @@
+require 'core'
+require 'java'
+require 'mapred_factory'
+
+import 'org.apache.hadoop.io.IntWritable'
+import 'org.apache.hadoop.io.Text'
+
+include HadoopDsl
+
+# Hadoop IO types
+HadoopDsl::Text = Text
+HadoopDsl::IntWritable = IntWritable
+
+def map(key, value, output, reporter, script)
+  mapper = MapperFactory.create(script, key.to_string, value.to_string)
+  mapper.run
+
+  write(output, mapper)
+end
+
+def reduce(key, values, output, reporter, script)
+  ruby_values = values.map {|v| to_ruby(v)}
+  reducer = ReducerFactory.create(script, key.to_string, ruby_values)
+  reducer.run
+
+  write(output, reducer)
+end
+
+def setup(conf, script)
+  setup = SetupFactory.create(script, conf)
+  setup.run
+
+  setup.paths.to_java
+end
+
+private
+
+def write(output, controller)
+  controller.emitted.each do |e|
+    e.each do |k, v|
+      output.collect(to_hadoop(k), to_hadoop(v))
+    end
+  end
+end
+
+def to_ruby(value)
+  case value
+  when IntWritable then value.get
+  when Text then value.to_string
+  else raise "no match class: #{value.class}"
+  end
+end
+
+def to_hadoop(value)
+  case value
+  when Integer then IntWritable.new(value)
+  when String then t = Text.new; t.set(value); t
+  else raise "no match class: #{value.class}"
+  end
+end

data/lib/java/.gitignore

@@ -0,0 +1 @@
+jruby-complete-*.jar

data/lib/log_analysis.rb

@@ -0,0 +1,165 @@
+require 'core'
+require 'enumerator'
+
+module HadoopDsl::LogAnalysis
+  include HadoopDsl
+  
+  KEY_SEP = "\t"
+  PREFIX = 'col'
+  PASS = nil
+  AVAILABLE_METHODS = [:separate, :pattern, :column_name, :column, :topic, :value, :count_uniq, :sum]
+
+  # common
+  module LogAnalysisMapRed
+    # entry point
+    def data(description = '', &block) yield end
+
+    def each_line(&block) yield end
+  end
+
+  # controller
+  class LogAnalysisSetup < BaseSetup
+    def initialize(script, conf)
+      super(script, conf)
+    end
+
+    include LogAnalysisMapRed
+  end
+
+  class LogAnalysisMapper < BaseMapper
+    def initialize(script, key, value)
+      super(script, LogAnalysisMapperModel.new(key, value))
+    end
+
+    include LogAnalysisMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  class LogAnalysisReducer < BaseReducer
+    def initialize(script, key, values)
+      super(script, LogAnalysisReducerModel.new(key, values))
+    end
+
+    include LogAnalysisMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  # model
+  class LogAnalysisMapperModel < BaseMapperModel
+    def initialize(key, value)
+      super(key, value)
+      @columns = ColumnArray.new
+      @topics = []
+    end
+
+    def column; @columns end
+
+    def topic(desc, options = {}, &block)
+      @topics << @current_topic = Topic.new(desc, options[:label])
+      yield if block_given?
+      @current_topic
+    end
+
+    def separate(sep)
+      parts = @value.split(sep)
+      create_or_replace_columns_with(parts) {|column, value| column.value = value}
+    end
+
+    def pattern(re)
+      if @value =~ re
+        md = Regexp.last_match
+        create_or_replace_columns_with(md.captures) {|column, value| column.value = value}
+      end
+    end
+
+    # column names by String converted to Symbol
+    def column_name(*names)
+      sym_names = names.map {|name| name.is_a?(String) ? name.to_sym : name }
+      create_or_replace_columns_with(sym_names) {|column, name| column.name = name}
+    end
+
+    def create_or_replace_columns_with(array, &block)
+      columns = array.enum_for(:each_with_index).map do |p, i|
+        c = @columns[i] ? @columns[i] : Column.new(i)
+        yield c, p
+        c
+      end
+      @columns = ColumnArray.new(columns)
+    end
+
+    # emitters
+    def count_uniq(column)
+      @controller.emit([@current_topic.label, KEY_SEP, column.value].join => 1)
+    end
+
+    def sum(column)
+      @controller.emit([@current_topic.label].join => column.value.to_i)
+    end
+
+    class ColumnArray < Array
+      def [](key)
+        case key
+        when Integer then at(key)
+        when Symbol then (select {|c| c.name == key}).first
+        when String then (select {|c| c.name == key.to_sym}).first
+        end
+      end
+    end
+
+    class Column
+      attr_reader :index
+      attr_accessor :value, :name
+
+      def initialize(index, value = nil)
+        @index, @value = index, value
+      end
+    end
+
+    class Topic
+      def initialize(desc, label = nil)
+        @desc, @label = desc, label
+      end
+
+      def label
+        @label || @desc.gsub(/\s/, '_')
+      end
+    end
+  end
+
+  class LogAnalysisReducerModel < BaseReducerModel
+    def initialize(key, values)
+      super(key, values)
+      if key =~ /(\w*)#{KEY_SEP}?(.*)/
+        @topic = Topic.new($1, values)
+      end
+    end
+
+    def topic(desc, options = {}, &block)
+      @current_topic = Topic.new(options[:label] || desc.gsub(/\s/, '_'), nil)
+      yield if block_given?
+      @current_topic
+    end
+
+    def count_uniq(column)
+      aggregate if @topic == @current_topic
+    end
+
+    def sum(column)
+      aggregate if @topic == @current_topic
+    end
+
+    class Topic
+      attr_reader :label, :values
+      
+      def initialize(label, values)
+        @label, @values = label, values
+      end
+
+      def ==(rh) self.label == rh.label end
+    end
+  end
+end

data/lib/mapred_factory.rb

@@ -0,0 +1,43 @@
+require 'util'
+
+module HadoopDsl
+  class MapRedFactory
+    def self.dsl_name(script)
+      read_file(script).each_line do |line|
+        dsl_name = $1 if line =~ /^use\s*'(\w*)'/
+        return dsl_name
+      end
+    end
+
+    def self.require_dsl_lib(dsl_name)
+      require snake_case(dsl_name)
+    end
+  end
+
+  class MapperFactory < MapRedFactory
+    def self.create(script, key, value)
+      dsl_name = self.dsl_name(script)
+      require_dsl_lib(dsl_name)
+      mapper_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Mapper" 
+      return eval(mapper_class).new(script, key, value)
+    end
+  end
+
+  class ReducerFactory < MapRedFactory
+    def self.create(script, key, values)
+      dsl_name = self.dsl_name(script)
+      require_dsl_lib(dsl_name)
+      reducer_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Reducer" 
+      return eval(reducer_class).new(script, key, values)
+    end
+  end
+
+  class SetupFactory < MapRedFactory
+    def self.create(script, conf)
+      dsl_name = self.dsl_name(script)
+      require_dsl_lib(dsl_name)
+      setup_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Setup" 
+      eval(setup_class).new(script, conf) rescue HadoopDsl::BaseSetup.new(script, conf)
+    end
+  end
+end

data/lib/util.rb

@@ -0,0 +1,11 @@
+# utility functions
+
+module HadoopDsl
+  def snake_case(str)
+    str.gsub(/\B[A-Z]/, '_\&').downcase
+  end
+
+  def read_file(file_name)
+    File.open(file_name).read
+  end
+end

data/lib/word_count.rb

@@ -0,0 +1,76 @@
+require 'core'
+require 'enumerator'
+
+module HadoopDsl::WordCount
+  include HadoopDsl
+  
+  AVAILABLE_METHODS = [:count_uniq, :total]
+  TOTAL_PREFIX = "\t"
+
+  # common
+  module WordCountMapRed
+    # entry point
+    def data(description = '', &block) yield end
+  end
+
+  # controller
+  class WordCountMapper < BaseMapper
+    def initialize(script, key, value)
+      super(script, WordCountMapperModel.new(key, value))
+    end
+
+    include WordCountMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  class WordCountReducer < BaseReducer
+    def initialize(script, key, values)
+      super(script, WordCountReducerModel.new(key, values))
+    end
+
+    include WordCountMapRed
+
+    # model methods
+    def_delegators :@model, *AVAILABLE_METHODS
+  end
+
+  # model
+  class WordCountMapperModel < BaseMapperModel
+    def initialize(key, value)
+      super(key, value)
+    end
+
+    # emitters
+    def count_uniq
+      @value.split.each {|word| @controller.emit(word => 1)}
+    end
+
+    def total(*types)
+      types.each do |type|
+        case type
+        when :bytes
+          @controller.emit("#{TOTAL_PREFIX}total bytes" => @value.gsub(/\s/, '').length)
+        when :words
+          @controller.emit("#{TOTAL_PREFIX}total words" => @value.split.size)
+        when :lines
+          @controller.emit("#{TOTAL_PREFIX}total lines" => 1)
+        end
+      end
+    end
+  end
+
+  class WordCountReducerModel < BaseReducerModel
+    def initialize(key, values)
+      super(key, values)
+    end
+
+    # emitters
+    def count_uniq; aggregate unless total_value? end
+    def total(*types); aggregate if total_value? end
+
+    private
+    def total_value?; @key =~ /^#{TOTAL_PREFIX}/ end
+  end
+end

data/spec/core_spec.rb

@@ -0,0 +1,73 @@
+require 'init'
+require 'core'
+
+include HadoopDsl
+
+describe 'BaseMapRed' do
+  before(:all) do
+    @script = create_tmp_script(<<-EOF)
+from 'test/inputs'
+to 'test/outputs'
+    EOF
+  end
+
+  it 'emit key value' do
+    mapper = BaseMapper.new(@script, BaseMapperModel.new(nil, nil))
+    mapper.emit('key' => 'value')
+    mapper.emitted.should == [{'key' => 'value'}]
+  end
+
+  it 'can run BaseMapper in minimum' do
+    model = BaseMapperModel.new('key', 'value')
+    mapper = BaseMapper.new(@script, model)
+    mapper.run
+  end
+
+  it 'can run BaseReducer in minimum' do
+    model = BaseReducerModel.new('key', 'values')
+    reducer = BaseReducer.new(@script, model)
+    reducer.run
+  end
+
+  it 'can run BaseSetup in minimum' do
+    setup = BaseSetup.new(@script, nil)
+    setup.run
+  end
+
+  describe BaseMapper do
+    it 'can emit as identity' do
+      model = BaseMapperModel.new('key', 'value')
+      mapper = BaseMapper.new(@script, model)
+      model.identity
+
+      mapper.emitted.should == [{'key' => 'value'}] 
+    end
+  end
+
+  describe BaseReducer do
+    it 'can emit as aggregate' do
+      model = BaseReducerModel.new('key', [1, 2, 3])
+      reducer = BaseReducer.new(@script, model)
+      model.aggregate
+
+      reducer.emitted.should == [{'key' => 6}] 
+    end
+
+    it 'can emit as identity' do
+      model = BaseReducerModel.new('key', [1, 2, 3])
+      reducer = BaseReducer.new(@script, model)
+      model.identity
+
+      reducer.emitted.should == [{'key' => 1}, {'key' => 2}, {'key' => 3}] 
+    end
+  end
+
+  describe BaseSetup do
+    it 'can get paths' do
+      setup = BaseSetup.new(@script, nil)
+      setup.run
+      setup.paths[0].should == 'test/inputs'
+      setup.paths[1].should == 'test/outputs'
+    end
+  end
+end

data/spec/example_spec.rb

@@ -0,0 +1,82 @@
+require 'log_analysis'
+require 'word_count'
+
+include HadoopDsl::LogAnalysis
+describe 'Aapach Log Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'apachelog-v2.rb')
+    @value = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326'
+  end
+
+  it 'can run example by mapper' do
+    mapper = LogAnalysisMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.first["user\tfrank"].should == 1
+  end
+
+  it 'can run example by reducer' do
+    reducer = LogAnalysisReducer.new(@script, "user\tfrank", [1, 1, 1])
+    reducer.run
+    reducer.emitted.first["user\tfrank"].should == 3
+  end
+end
+
+include HadoopDsl::WordCount
+describe 'Word Count Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'word_count_test.rb')
+    @value = 'Lorem ipsum ipsum Lorem sit amet,'
+  end
+
+  it 'can run example by mapper' do
+    mapper = WordCountMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.size.should == 9
+    mapper.emitted.each do |e|
+      case e.keys.first
+      when 'Lorem'
+        e.values.first.should == 1
+      when 'total words'
+        e.values.first.should == 6
+      end
+    end
+  end
+
+  it 'can run example by reducer' do
+    reducer = WordCountReducer.new(@script, "Lorem", [1, 1, 1])
+    reducer.run
+    reducer.emitted.first["Lorem"].should == 3
+  end
+end
+
+include HadoopDsl::HiveLike
+describe 'Hive Like Example' do
+  before(:all) do
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'hive_like_test.rb')
+    @value = 'apple, 3, 100'
+  end
+
+  it 'can run setup' do
+    conf = mock('conf')
+    conf.should_receive(:output_key_class=).once
+    conf.should_receive(:output_value_class=).once
+
+    setup = HiveLikeSetup.new(@script, conf)
+    setup.run
+    setup.paths[0].should == 'hive-like/items.txt'
+  end
+
+  it 'can run example by mapper' do
+    mapper = HiveLikeMapper.new(@script, nil, @value)
+    mapper.run
+    mapper.emitted.size.should == 1
+    mapper.emitted.first['items'].should == '3, 100, apple'
+  end
+
+  it 'can run example by reducer' do
+    values = ['v1', 'v2', 'v3']
+    reducer = HiveLikeReducer.new(@script, "items", values)
+    reducer.run
+    reducer.emitted.first["items"].should == 'v1'
+  end
+end

data/spec/hive_like_spec.rb

@@ -0,0 +1,58 @@
+require 'init'
+require 'core'
+require 'hive_like'
+
+include HadoopDsl::HiveLike
+
+describe HiveLikeSetup do
+  it 'should load data' do
+    script = create_tmp_script(%Q!load_data "hive-like/inputs", items;!)
+    conf = mock('conf')
+    conf.should_receive(:output_key_class=).once
+    conf.should_receive(:output_value_class=).once
+
+    setup = HiveLikeSetup.new(script, conf)
+    setup.run
+    setup.paths[0].should == 'hive-like/inputs'
+    setup.paths[1].should == 'hive-like/outputs'
+  end
+end
+
+describe HiveLikeMapper do
+  before do
+    @value = 'apple, 3, 100'
+  end
+
+  it 'should create table' do
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    mapper.create_table('items', 'item', 'STRING', 'quantity', 'INT', 'price', 'INT');
+    mapper.table.name.should == 'items'
+    mapper.table.column(0).should == 'item'
+    mapper.table.column(1).should == 'quantity'
+  end
+
+  it 'should select' do
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    mapper.create_table('items', 'item', 'STRING', 'quantity', 'INT', 'price', 'INT');
+    mapper.select("item", "quantity", "price", "from", "items")
+    mapper.emitted.first.should == {'items' => 'apple, 3, 100'}
+  end
+
+  it 'should pre process script body' do
+    body = "select foo, bar from table;\n"
+    mapper = HiveLikeMapper.new(nil, nil, @value)
+    processed = mapper.pre_process(body)
+    processed.should == %Q!select("foo", "bar", "from", "table")\n!
+  end
+end
+
+describe HiveLikeReducer do
+  it 'should select as identity' do
+    key = 'Lorem'
+    values = [1, 1, 1]
+    reducer = HiveLikeReducer.new(nil, key, values)
+
+    reducer.select
+    reducer.emitted[0].should == {'Lorem' => 1}
+  end
+end