hadoop-rubydsl 0.0.3 → 0.0.4

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/bin/hrd

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'hadoop-dsl'
+require 'hadoop_dsl_client'
 
 HadoopDsl::Client.new(ARGV).run

data/examples/hive_like_test.rb

@@ -1,4 +1,4 @@
-use 'HiveLike'
+dsl 'HiveLike'
 
 # hive-like/items.txt
 # apple, 3, 100

data/examples/{apachelog-v2-2.rb → log_analysis_test.rb}

@@ -1,4 +1,4 @@
-use 'LogAnalysis'
+dsl 'LogAnalysis'
 
 data 'apache log on test2' do
   from 'apachelog/inputs'
@@ -14,5 +14,28 @@ data 'apache log on test2' do
     topic 'ua counts', :label => 'ua' do
       count_uniq column[:ua]
     end
+
+    topic 'count bot', :label => 'bot' do
+      ua = column[:ua].value
+      bot = ua if ua =~ /bot/i
+      count_uniq bot
+    end
+
+    topic 'ua counts group by path' do
+      request = column[:request].value
+      path = request.split(/\s+/)[1]
+      group_by path
+      count_uniq column[:ua]
+    end
+
+    topic 'ua counts by daily' do
+      group_date_by column[:access_date], :daily
+      count_uniq column[:ua]
+    end
+
+#    topic 'total bytes' do
+#      select_date column[:access_date], BY_MONTHLY
+#      sum column[:bytes].to_kilobytes # / 1024
+#    end
   end
 end

data/examples/word_count_test.rb

@@ -1,4 +1,4 @@
-use 'WordCount'
+dsl 'WordCount'
 
 from 'wc/inputs'
 to 'wc/outputs'

data/hadoop-rubydsl.gemspec

@@ -5,38 +5,34 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hadoop-rubydsl}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = %q{2010-01-04}
+  s.date = %q{2010-01-13}
   s.description = %q{Hadoop Ruby DSL}
   s.email = %q{fujibee@gmail.com}
   s.executables = ["hrd", "hadoop-hudson.sh", "hadoop-ruby.sh"]
   s.extra_rdoc_files = [
-    "README.rdoc",
-     "TODO"
+    "README.rdoc"
   ]
   s.files = [
     ".gitignore",
      "README.rdoc",
      "Rakefile",
-     "TODO",
      "VERSION",
      "bin/hadoop-hudson.sh",
      "bin/hadoop-ruby.sh",
      "bin/hrd",
      "conf/hadoop-site.xml",
-     "examples/apachelog-v2-2.rb",
-     "examples/apachelog-v2.rb",
-     "examples/apachelog.rb",
      "examples/hive_like_test.rb",
+     "examples/log_analysis_test.rb",
      "examples/word_count_test.rb",
      "hadoop-rubydsl.gemspec",
-     "lib/client.rb",
      "lib/core.rb",
      "lib/dsl_init.rb",
-     "lib/hadoop-dsl.rb",
+     "lib/hadoop_dsl.rb",
+     "lib/hadoop_dsl_client.rb",
      "lib/hive_like.rb",
      "lib/log_analysis.rb",
      "lib/mapred_factory.rb",
@@ -59,11 +55,9 @@ Gem::Specification.new do |s|
      "spec/hive_like_spec.rb",
      "spec/log_analysis_spec.rb",
      "spec/example_spec.rb",
-     "examples/apachelog-v2.rb",
      "examples/hive_like_test.rb",
-     "examples/word_count_test.rb",
-     "examples/apachelog-v2-2.rb",
-     "examples/apachelog.rb"
+     "examples/log_analysis_test.rb",
+     "examples/word_count_test.rb"
   ]
 
   if s.respond_to? :specification_version then

data/lib/core.rb

@@ -1,10 +1,33 @@
-require 'util'
+require 'hadoop_dsl'
 require 'forwardable'
 
 module HadoopDsl
+  # common
+  module DslElement
+    # all DSL statements without def is processed here
+    def method_missing(method_name, *args)
+      yield if block_given?
+      self
+    end
+  end
+
   # controller
+  module DslController
+    include DslElement
+
+    def run
+      body = pre_process(HadoopDsl.read_file(@script))
+      eval(body, binding, @script)
+    end
+
+    def pre_process(body)
+      body # do nothing
+    end
+  end
+
   class BaseMapRed
     extend Forwardable
+    include DslController
 
     attr_reader :emitted
 
@@ -14,66 +37,54 @@ module HadoopDsl
       @emitted = []
     end
 
-    def run
-      body = pre_process(read_file(@script))
-      eval(body, binding, @script)
-    end
-
-    def pre_process(body)
-      body # do nothing
-    end
-
     def emit(hash) @emitted << hash end
 
-    # all DSL statements without def is processed here
-    def method_missing(method_name, *args) self end
+  private
+    def key; @model.key end
   end
 
   class BaseSetup
+    include DslController
+
     def initialize(script, conf)
       @script, @conf = script, conf
       output_format
     end
 
-    def run
-      body = pre_process(read_file(@script))
-      eval(body, binding, @script)
-    end
-
-    def pre_process(body)
-      body # do nothing
-    end
-
-    # do nothing
-    def output_format; end
-
+    def output_format; end # do nothing
     def paths; [@from, @to] end
-
     def from(path) @from = path end
     def to(path) @to = path end
-
-    # all DSL statements without def is processed here
-    def method_missing(method_name, *args) self end
   end
 
   class BaseMapper < BaseMapRed
-    def initialize(script, model)
-      super(script, model)
+    # common functions
+    def identity
+      emit(@model.key => @model.value)
     end
+
+  private
+    def value; @model.values end
   end
 
   class BaseReducer < BaseMapRed
-    def initialize(script, model)
-      super(script, model)
+    # common functions
+    def aggregate
+      emit(@model.key => @model.values.inject {|ret, i| ret + i})
+    end
+
+    def identity
+      @model.values.each {|v| emit(@model.key => v)}
     end
+
+  private
+    def values; @model.values end
   end
 
   # model
   class BaseModel
+    include DslElement
     attr_accessor :controller
-
-    # all DSL statements without def is processed here
-    def method_missing(method_name, *args) self end
   end
 
   class BaseMapperModel < BaseModel
@@ -82,11 +93,6 @@ module HadoopDsl
     def initialize(key, value)
       @key, @value = key, value
     end
-
-    # common functions
-    def identity
-      @controller.emit(@key => @value)
-    end
   end
 
   class BaseReducerModel < BaseModel
@@ -95,14 +101,5 @@ module HadoopDsl
     def initialize(key, values)
       @key, @values = key, values
     end
-
-    # common functions
-    def aggregate
-      @controller.emit(@key => @values.inject {|ret, i| ret + i})
-    end
-
-    def identity
-      @values.each {|v| @controller.emit(@key => v)}
-    end
   end
 end

data/lib/dsl_init.rb

@@ -1,16 +1,7 @@
-require 'core'
-require 'java'
-require 'mapred_factory'
-
-import 'org.apache.hadoop.io.IntWritable'
-import 'org.apache.hadoop.io.Text'
+require 'hadoop_dsl'
 
 include HadoopDsl
 
-# Hadoop IO types
-HadoopDsl::Text = Text
-HadoopDsl::IntWritable = IntWritable
-
 def map(key, value, output, reporter, script)
   mapper = MapperFactory.create(script, key, value)
   mapper.run

data/lib/hadoop_dsl.rb

@@ -0,0 +1,14 @@
+require 'util'
+require 'mapred_factory'
+require 'core'
+
+# for jruby
+if defined? JRUBY_VERSION
+  require 'java'
+  import 'org.apache.hadoop.io.IntWritable'
+  import 'org.apache.hadoop.io.Text'
+
+  # Hadoop IO types
+  HadoopDsl::Text = Text
+  HadoopDsl::IntWritable = IntWritable
+end

data/lib/{client.rb → hadoop_dsl_client.rb}

@@ -1,4 +1,14 @@
+require 'jruby-on-hadoop'
+
 module HadoopDsl
+  def self.lib_path
+    File.expand_path(File.dirname(__FILE__))
+  end
+
+  def self.dsl_init_script
+    File.join(lib_path, "dsl_init.rb")
+  end
+
   class Client < JRubyOnHadoop::Client
     def parse_args
       super

data/lib/hive_like.rb

@@ -1,11 +1,6 @@
-require 'core'
-require 'enumerator'
+require 'hadoop_dsl'
 
 module HadoopDsl::HiveLike
-  include HadoopDsl
-  
-  AVAILABLE_METHODS = [:select, :create_table, :table]
-
   # common
   module HiveLikeMapRed
     def pre_process(body)
@@ -17,7 +12,7 @@ module HadoopDsl::HiveLike
           args = sprit_and_marge_args($2)
           processed << "#{method}(#{args})\n"
         else 
-          processed << line + "\n"
+          processed << line + "\n" if line
         end
       end
       processed
@@ -32,15 +27,15 @@ module HadoopDsl::HiveLike
   end
 
   # controller
-  class HiveLikeSetup < BaseSetup
+  class HiveLikeSetup < HadoopDsl::BaseSetup
     def load_data(inputs, table)
       @from = inputs
       @to = inputs.gsub(/#{File.basename(inputs)}$/, 'outputs')
     end
     
     def output_format
-      @conf.output_key_class = Text
-      @conf.output_value_class = Text
+      @conf.output_key_class = HadoopDsl::Text
+      @conf.output_value_class = HadoopDsl::Text
     end
 
     # might not need but occur error if not exists
@@ -49,37 +44,43 @@ module HadoopDsl::HiveLike
     include HiveLikeMapRed
   end
 
-  class HiveLikeMapper < BaseMapper
+  class HiveLikeMapper < HadoopDsl::BaseMapper
     def initialize(script, key, value)
       super(script, HiveLikeMapperModel.new(key, value))
     end
 
     include HiveLikeMapRed
 
-    # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
+    def_delegators :@model, :create_table, :table
+
+    # emitters
+    def select(*args)
+      from_index = args.index('from')
+      if from_index
+        values = args[0...from_index].map do |column|
+          splitted = @model.value.split(/[,\s]+/)
+          splitted[@model.table.columns.index(column)]
+        end
+        emit(args[from_index + 1] => values.join(", "))
+      end
+    end
   end
 
-  class HiveLikeReducer < BaseReducer
+  class HiveLikeReducer < HadoopDsl::BaseReducer
     def initialize(script, key, values)
       super(script, HiveLikeReducerModel.new(key, values))
     end
 
     include HiveLikeMapRed
 
-    # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
+    # emitters
+    def select(*args) identity end
   end
 
   # model
-  class HiveLikeMapperModel < BaseMapperModel
+  class HiveLikeMapperModel < HadoopDsl::BaseMapperModel
     attr_reader :table
 
-    def initialize(key, value)
-      super(key, value)
-    end
-
-    # emitters
     def create_table(name, *column_and_type)
       @table = Table.new(name)
       column_and_type.each_with_index do |column, index|
@@ -88,17 +89,6 @@ module HadoopDsl::HiveLike
       end
     end
 
-    def select(*args)
-      from_index = args.index('from')
-      if from_index
-        values = args[0...from_index].map do |column|
-          splitted = @value.split(/[,\s]+/)
-          splitted[@table.columns.index(column)]
-        end
-        @controller.emit(args[from_index + 1] => values.join(", "))
-      end
-    end
-
     class Table
       attr_reader :name, :columns
 
@@ -111,12 +101,6 @@ module HadoopDsl::HiveLike
     end
   end
 
-  class HiveLikeReducerModel < BaseReducerModel
-    def initialize(key, values)
-      super(key, values)
-    end
-
-    # emitters
-    def select(*args) identity end
+  class HiveLikeReducerModel < HadoopDsl::BaseReducerModel
   end
 end

data/lib/log_analysis.rb

@@ -1,55 +1,123 @@
-require 'core'
+require 'hadoop_dsl'
 require 'enumerator'
 
 module HadoopDsl::LogAnalysis
-  include HadoopDsl
-  
   KEY_SEP = "\t"
   PREFIX = 'col'
   PASS = nil
-  AVAILABLE_METHODS = [:separate, :pattern, :column_name, :column, :topic, :value, :count_uniq, :sum]
+  MODEL_METHODS = [:column, :value]
 
-  # common
-  module LogAnalysisMapRed
-    # entry point
-    def data(description = '', &block) yield end
+  # controller
+  class LogAnalysisMapper < HadoopDsl::BaseMapper
+    def initialize(script, key, value)
+      super(script, LogAnalysisMapperModel.new(key, value))
+    end
 
-    def each_line(&block) yield end
-  end
+    # model methods
+    def_delegators :@model, *MODEL_METHODS
+   
+    def topic(desc, options = {}, &block)
+      @model.create_topic(desc, options)
+      yield if block_given?
+      current_topic
+    end
 
-  # controller
-  class LogAnalysisSetup < BaseSetup
-    def initialize(script, conf)
-      super(script, conf)
+    def separate(sep)
+      parts = value.split(sep)
+      @model.create_or_replace_columns_with(parts) {|column, value| column.value = value}
     end
 
-    include LogAnalysisMapRed
-  end
+    def pattern(re)
+      if value =~ re
+        md = Regexp.last_match
+        @model.create_or_replace_columns_with(md.captures) {|column, value| column.value = value}
+      end
+    end
 
-  class LogAnalysisMapper < BaseMapper
-    def initialize(script, key, value)
-      super(script, LogAnalysisMapperModel.new(key, value))
+    # column names by String converted to Symbol
+    def column_name(*names)
+      sym_names = names.map {|name| name.is_a?(String) ? name.to_sym : name }
+      @model.create_or_replace_columns_with(sym_names) {|column, name| column.name = name}
     end
 
-    include LogAnalysisMapRed
+    def group_by(column_or_value)
+      case column_or_value
+      when LogAnalysisMapperModel::Column
+        column = column_or_value
+        current_topic.key_elements << column.value
+      else
+        value = column_or_value
+        current_topic.key_elements << value
+      end
+    end
 
-    # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
+    def group_date_by(column, term)
+      require 'time'
+      time = parse_time(column.value)
+      time_key = case term
+                 when :daily then time.strftime('%Y%m%d') 
+                 when :monthly then time.strftime('%Y%m') 
+                 when :yearly then time.strftime('%Y') 
+                 end
+      current_topic.key_elements << time_key
+    end
+
+    # emitters
+    def count_uniq(column_or_value)
+      uniq_key =
+        case column_or_value
+        when LogAnalysisMapperModel::Column
+          column = column_or_value
+          column.value
+        else column_or_value # value
+        end
+      current_topic.key_elements << uniq_key
+      emit(current_topic.key => 1)
+    end
+
+    def sum(column)
+      emit(current_topic.key => column.value.to_i)
+    end
+
+  private
+    def current_topic; @model.current_topic end
+
+    def parse_time(str)
+      begin Time.parse(str)
+      rescue
+        # apachelog pattern ex) "10/Oct/2000:13:55:36 -0700"
+        Time.parse($1) if str =~ /^(\d*\/\w*\/\d*):/
+      end
+    end
   end
 
-  class LogAnalysisReducer < BaseReducer
+  class LogAnalysisReducer < HadoopDsl::BaseReducer
     def initialize(script, key, values)
       super(script, LogAnalysisReducerModel.new(key, values))
     end
 
-    include LogAnalysisMapRed
-
     # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
+    def_delegators :@model, *MODEL_METHODS
+
+    def topic(desc, options = {}, &block)
+      @model.create_topic(desc, options)
+      yield if block_given?
+      @model.current_topic
+    end
+
+    def count_uniq(column)
+      aggregate if @model.topic == @model.current_topic
+    end
+
+    def sum(column)
+      aggregate if @model.topic == @model.current_topic
+    end
   end
 
   # model
-  class LogAnalysisMapperModel < BaseMapperModel
+  class LogAnalysisMapperModel < HadoopDsl::BaseMapperModel
+    attr_reader :current_topic
+
     def initialize(key, value)
       super(key, value)
       @columns = ColumnArray.new
@@ -58,28 +126,8 @@ module HadoopDsl::LogAnalysis
 
     def column; @columns end
 
-    def topic(desc, options = {}, &block)
+    def create_topic(desc, options)
       @topics << @current_topic = Topic.new(desc, options[:label])
-      yield if block_given?
-      @current_topic
-    end
-
-    def separate(sep)
-      parts = @value.split(sep)
-      create_or_replace_columns_with(parts) {|column, value| column.value = value}
-    end
-
-    def pattern(re)
-      if @value =~ re
-        md = Regexp.last_match
-        create_or_replace_columns_with(md.captures) {|column, value| column.value = value}
-      end
-    end
-
-    # column names by String converted to Symbol
-    def column_name(*names)
-      sym_names = names.map {|name| name.is_a?(String) ? name.to_sym : name }
-      create_or_replace_columns_with(sym_names) {|column, name| column.name = name}
     end
 
     def create_or_replace_columns_with(array, &block)
@@ -91,15 +139,6 @@ module HadoopDsl::LogAnalysis
       @columns = ColumnArray.new(columns)
     end
 
-    # emitters
-    def count_uniq(column)
-      @controller.emit([@current_topic.label, KEY_SEP, column.value].join => 1)
-    end
-
-    def sum(column)
-      @controller.emit([@current_topic.label].join => column.value.to_i)
-    end
-
     class ColumnArray < Array
       def [](key)
         case key
@@ -120,17 +159,28 @@ module HadoopDsl::LogAnalysis
     end
 
     class Topic
+      attr_reader :key_elements
+
       def initialize(desc, label = nil)
         @desc, @label = desc, label
+        @key_elements = []
       end
 
       def label
         @label || @desc.gsub(/\s/, '_')
       end
+
+      def key
+        without_label =
+          @key_elements.size > 0 ? @key_elements.join(KEY_SEP) : nil
+        [label, without_label].compact.join(KEY_SEP)
+      end
     end
   end
 
-  class LogAnalysisReducerModel < BaseReducerModel
+  class LogAnalysisReducerModel < HadoopDsl::BaseReducerModel
+    attr_reader :topic, :current_topic
+
     def initialize(key, values)
       super(key, values)
       if key =~ /(\w*)#{KEY_SEP}?(.*)/
@@ -138,18 +188,8 @@ module HadoopDsl::LogAnalysis
       end
     end
 
-    def topic(desc, options = {}, &block)
+    def create_topic(desc, options)
       @current_topic = Topic.new(options[:label] || desc.gsub(/\s/, '_'), nil)
-      yield if block_given?
-      @current_topic
-    end
-
-    def count_uniq(column)
-      aggregate if @topic == @current_topic
-    end
-
-    def sum(column)
-      aggregate if @topic == @current_topic
     end
 
     class Topic

data/lib/mapred_factory.rb

@@ -1,16 +1,16 @@
-require 'util'
+require 'hadoop_dsl'
 
 module HadoopDsl
   class MapRedFactory
     def self.dsl_name(script)
-      read_file(script).each_line do |line|
-        dsl_name = $1 if line =~ /^use\s*'(\w*)'/
-        return dsl_name
+      HadoopDsl.read_file(script).each_line do |line|
+        dsl_name = $1 if line =~ /\s*dsl\s*\(?["'](\w*)["']\)?/
+        return dsl_name if dsl_name
       end
     end
 
     def self.require_dsl_lib(dsl_name)
-      require snake_case(dsl_name)
+      require HadoopDsl.snake_case(dsl_name)
     end
   end
 

data/lib/util.rb

@@ -1,18 +1,18 @@
 # utility functions
+require 'hadoop_dsl'
 
 module HadoopDsl
-  def snake_case(str)
+  def self.snake_case(str)
     str.gsub(/\B[A-Z]/, '_\&').downcase
   end
 
-  def read_file(file_name)
+  def self.read_file(file_name)
     # read as usual
     body = File.open(file_name).read rescue nil
     return body if body
 
     # read from loadpath
     $:.each do |path|
-      p path
       body = File.open(File.join(path, file_name)).read rescue next
       return body if body
     end

data/lib/word_count.rb

@@ -1,76 +1,56 @@
-require 'core'
+require 'hadoop_dsl'
 require 'enumerator'
 
 module HadoopDsl::WordCount
-  include HadoopDsl
-  
-  AVAILABLE_METHODS = [:count_uniq, :total]
+  MODEL_METHODS = []
   TOTAL_PREFIX = "\t"
 
-  # common
-  module WordCountMapRed
-    # entry point
-    def data(description = '', &block) yield end
-  end
-
   # controller
-  class WordCountMapper < BaseMapper
+  class WordCountMapper < HadoopDsl::BaseMapper
     def initialize(script, key, value)
       super(script, WordCountMapperModel.new(key, value))
     end
 
-    include WordCountMapRed
-
-    # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
-  end
-
-  class WordCountReducer < BaseReducer
-    def initialize(script, key, values)
-      super(script, WordCountReducerModel.new(key, values))
-    end
-
-    include WordCountMapRed
-
     # model methods
-    def_delegators :@model, *AVAILABLE_METHODS
-  end
-
-  # model
-  class WordCountMapperModel < BaseMapperModel
-    def initialize(key, value)
-      super(key, value)
-    end
+    def_delegators :@model, *MODEL_METHODS
 
     # emitters
     def count_uniq
-      @value.split.each {|word| @controller.emit(word => 1)}
+      @model.value.split.each {|word| emit(word => 1)}
     end
 
     def total(*types)
       types.each do |type|
         case type
         when :bytes
-          @controller.emit("#{TOTAL_PREFIX}total bytes" => @value.gsub(/\s/, '').length)
+          emit("#{TOTAL_PREFIX}total bytes" => @model.value.gsub(/\s/, '').length)
         when :words
-          @controller.emit("#{TOTAL_PREFIX}total words" => @value.split.size)
+          emit("#{TOTAL_PREFIX}total words" => @model.value.split.size)
         when :lines
-          @controller.emit("#{TOTAL_PREFIX}total lines" => 1)
+          emit("#{TOTAL_PREFIX}total lines" => 1)
         end
       end
     end
   end
 
-  class WordCountReducerModel < BaseReducerModel
-    def initialize(key, values)
-      super(key, values)
+  class WordCountReducer < HadoopDsl::BaseReducer
+    def initialize(script, key, values)
+      super(script, WordCountReducerModel.new(key, values))
     end
 
+    # model methods
+    def_delegators :@model, *MODEL_METHODS
+
     # emitters
-    def count_uniq; aggregate unless total_value? end
-    def total(*types); aggregate if total_value? end
+    def count_uniq; aggregate unless @model.total_value? end
+    def total(*types); aggregate if @model.total_value? end
+  end
+
+  # model
+  class WordCountMapperModel < HadoopDsl::BaseMapperModel
+  end
 
-    private
+  class WordCountReducerModel < HadoopDsl::BaseReducerModel
     def total_value?; @key =~ /^#{TOTAL_PREFIX}/ end
   end
 end

data/spec/client_spec.rb

@@ -1,4 +1,5 @@
-require 'hadoop-dsl'
+require File.join(File.dirname(__FILE__), 'spec_helper')
+require 'hadoop_dsl_client'
 
 describe HadoopDsl::Client do
   before do

data/spec/core_spec.rb

@@ -1,4 +1,4 @@
-require 'dsl_init'
+require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'core'
 
 include HadoopDsl
@@ -38,7 +38,7 @@ to 'test/outputs'
     it 'can emit as identity' do
       model = BaseMapperModel.new('key', 'value')
       mapper = BaseMapper.new(@script, model)
-      model.identity
+      mapper.identity
 
       mapper.emitted.should == [{'key' => 'value'}] 
     end
@@ -48,7 +48,7 @@ to 'test/outputs'
     it 'can emit as aggregate' do
       model = BaseReducerModel.new('key', [1, 2, 3])
       reducer = BaseReducer.new(@script, model)
-      model.aggregate
+      reducer.aggregate
 
       reducer.emitted.should == [{'key' => 6}] 
     end
@@ -56,7 +56,7 @@ to 'test/outputs'
     it 'can emit as identity' do
       model = BaseReducerModel.new('key', [1, 2, 3])
       reducer = BaseReducer.new(@script, model)
-      model.identity
+      reducer.identity
 
       reducer.emitted.should == [{'key' => 1}, {'key' => 2}, {'key' => 3}] 
     end

data/spec/dsl_init_spec.rb

@@ -4,7 +4,7 @@ describe 'mapreduce init' do
 
   before(:all) do
     @script = create_tmp_script(<<-EOF)
-use 'LogAnalysis'
+dsl 'LogAnalysis'
 data 'test' do
   from 'test/inputs'
   to 'test/outputs'

data/spec/example_spec.rb

@@ -1,23 +1,25 @@
 require 'log_analysis'
 require 'word_count'
+require 'hive_like'
 
 include HadoopDsl::LogAnalysis
 describe 'Aapach Log Example' do
   before(:all) do
-    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'apachelog-v2.rb')
-    @value = '127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326'
+    @script = File.join(File.dirname(__FILE__), '..', 'examples', 'log_analysis_test.rb')
+    @bot_ua = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+    @value = %Q!127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "-" "#{@bot_ua}"!
   end
 
   it 'can run example by mapper' do
     mapper = LogAnalysisMapper.new(@script, nil, @value)
     mapper.run
-    mapper.emitted.first["user\tfrank"].should == 1
+    mapper.emitted.first.should == {"ua\t#{@bot_ua}" => 1}
   end
 
   it 'can run example by reducer' do
-    reducer = LogAnalysisReducer.new(@script, "user\tfrank", [1, 1, 1])
+    reducer = LogAnalysisReducer.new(@script, "ua\tChrome", [1, 1, 1])
     reducer.run
-    reducer.emitted.first["user\tfrank"].should == 3
+    reducer.emitted.first["ua\tChrome"].should == 3
   end
 end
 

data/spec/hive_like_spec.rb

@@ -1,4 +1,4 @@
-require 'core'
+require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'hive_like'
 
 include HadoopDsl::HiveLike

data/spec/log_analysis_spec.rb

@@ -1,4 +1,4 @@
-require 'core'
+require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'log_analysis'
 
 include HadoopDsl::LogAnalysis
@@ -39,13 +39,22 @@ describe LogAnalysisMapper do
     mapper.column[:user].value.should == 'frank'
   end
 
-  it 'should count uniq column' do
+  it 'should count uniq by column' do
     value = 'count uniq'
     mapper = LogAnalysisMapper.new(nil, nil, value)
     mapper.separate(' ')
     mapper.topic('t1') { mapper.count_uniq mapper.column[1] }
 
-    mapper.emitted.first["t1\tuniq"].should == 1
+    mapper.emitted.should == [{"t1\tuniq" => 1}]
+  end
+
+  it 'should count uniq by value' do
+    value = 'count uniq'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+    mapper.topic('t1') { mapper.count_uniq 'orig value' }
+
+    mapper.emitted.should == [{"t1\torig value" => 1}]
   end
 
   it 'should sum column value' do
@@ -83,6 +92,54 @@ describe LogAnalysisMapper do
     topic = mapper.topic('desc with space')
     topic.label.should == 'desc_with_space'
   end
+
+  it 'can group date monthly' do
+    value = '2010/1/1 newyearday'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+    mapper.column_name 'date', 'holiday'
+
+    ['yearly', 'monthly', 'daily'].each do |term|
+      mapper.topic(term) do
+        mapper.group_date_by mapper.column[:date], term.to_sym
+        mapper.count_uniq mapper.column[:holiday]
+      end
+    end
+    mapper.emitted.should ==
+      [
+        {"yearly\t2010\tnewyearday" => 1},
+        {"monthly\t201001\tnewyearday" => 1},
+        {"daily\t20100101\tnewyearday" => 1}
+      ]
+  end
+
+  it 'can group by' do
+    value = '1 sub_2 bingo!'
+    mapper = LogAnalysisMapper.new(nil, nil, value)
+    mapper.separate(' ')
+    mapper.column_name 'id', 'sub_id', 'data'
+
+    mapper.topic('test') do
+      mapper.group_by mapper.column[:sub_id]
+      mapper.count_uniq mapper.column[:data]
+    end
+    mapper.emitted.should == [{"test\tsub_2\tbingo!" => 1}]
+  end
+end
+
+Topic = LogAnalysisMapperModel::Topic
+describe Topic do
+  it 'can get key with label' do
+    t = Topic.new('label')
+    t.key.should == 'label'
+  end
+
+  it 'can get key with label and elements' do
+    t = Topic.new('label')
+    t.key_elements << 'e1'
+    t.key_elements << 'e2'
+    t.key.should == "label\te1\te2"
+  end
 end
 
 describe LogAnalysisReducer do

data/spec/mapred_factory_spec.rb

@@ -1,31 +1,33 @@
 require File.join(File.dirname(__FILE__) , 'spec_helper')
-
 require 'mapred_factory'
-require 'log_analysis'
 
-describe 'MapRed Factory' do
+include HadoopDsl
 
+describe 'MapRed Factory' do
   before(:all) do
-    @script = create_tmp_script("use 'LogAnalysis'")
+    @script = create_tmp_script("dsl 'LogAnalysis'")
   end
 
   it 'can create mapper' do
     mapper = MapperFactory.create(@script, nil, nil)
-    mapper.class.should == LogAnalysisMapper
+    mapper.class.should == LogAnalysis::LogAnalysisMapper
   end
 
   it 'can create reducer' do
     reducer = ReducerFactory.create(@script, nil, nil)
-    reducer.class.should == LogAnalysisReducer
+    reducer.class.should == LogAnalysis::LogAnalysisReducer
   end
 
   it 'can create setup' do
-    s = SetupFactory.create(@script, nil)
-    s.class.should == LogAnalysisSetup
+    conf = mock('conf')
+    conf.should_receive(:output_key_class=).once
+    conf.should_receive(:output_value_class=).once
+    s = SetupFactory.create(create_tmp_script("dsl 'HiveLike'"), conf)
+    s.class.should == HiveLike::HiveLikeSetup
   end
 
   it 'can create base if not exists in specific DSL' do
-    s = SetupFactory.create(create_tmp_script("use 'WordCount'"), nil)
+    s = SetupFactory.create(create_tmp_script("dsl 'WordCount'"), nil)
     s.class.should == BaseSetup
   end
 
@@ -37,6 +39,24 @@ describe 'MapRed Factory' do
   it 'can convert dsl name to dsl lib file and require' do
     dsl_name = MapRedFactory.dsl_name(@script)
     MapRedFactory.require_dsl_lib(dsl_name).should_not be_nil
-    LogAnalysisMapper
+    LogAnalysis::LogAnalysisMapper
+  end
+
+  it 'can create mapper if statement has double quote' do
+    script = create_tmp_script(%Q!dsl "LogAnalysis"!)
+    mapper = MapperFactory.create(script, nil, nil)
+    mapper.class.should == LogAnalysis::LogAnalysisMapper
+  end
+
+  it 'can create mapper if exists more space' do
+    script = create_tmp_script(%Q!  dsl   "LogAnalysis"   !)
+    mapper = MapperFactory.create(script, nil, nil)
+    mapper.class.should == LogAnalysis::LogAnalysisMapper
+  end
+
+  it 'can create mapper if exists bracket' do
+    script = create_tmp_script(%Q!  dsl ("LogAnalysis")   !)
+    mapper = MapperFactory.create(script, nil, nil)
+    mapper.class.should == LogAnalysis::LogAnalysisMapper
   end
 end

data/spec/spec_helper.rb

@@ -1,5 +1,4 @@
 # spec helper
-
 require 'tempfile'
 
 def create_tmp_script(body)

data/spec/util_spec.rb

@@ -1,19 +1,18 @@
 require File.join(File.dirname(__FILE__) , 'spec_helper')
-
 require 'util'
 
 describe 'utilities' do
   it 'can change camelcase str to snakecase' do
-    snake_case('CamelCaseStr').should == 'camel_case_str'
+    HadoopDsl.snake_case('CamelCaseStr').should == 'camel_case_str'
   end
 
   it 'can read file and get file data to string' do
     script_body = 'This is a script body.'
     @script = create_tmp_script(script_body)
-    read_file(@script).should == script_body
+    HadoopDsl.read_file(@script).should == script_body
   end
 
   it 'raise error if no file in loadpath' do
-    lambda { read_file('not_exists_on_loadpath') }.should raise_error
+    lambda { HadoopDsl.read_file('not_exists_on_loadpath') }.should raise_error
   end
 end

data/spec/word_count_spec.rb

@@ -1,4 +1,4 @@
-require 'core'
+require File.join(File.dirname(__FILE__), 'spec_helper')
 require 'word_count'
 
 include HadoopDsl::WordCount

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hadoop-rubydsl
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors: 
 - Koichi Fujikawa
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-04 00:00:00 +09:00
+date: 2010-01-13 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -32,27 +32,23 @@ extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
-- TODO
 files: 
 - .gitignore
 - README.rdoc
 - Rakefile
-- TODO
 - VERSION
 - bin/hadoop-hudson.sh
 - bin/hadoop-ruby.sh
 - bin/hrd
 - conf/hadoop-site.xml
-- examples/apachelog-v2-2.rb
-- examples/apachelog-v2.rb
-- examples/apachelog.rb
 - examples/hive_like_test.rb
+- examples/log_analysis_test.rb
 - examples/word_count_test.rb
 - hadoop-rubydsl.gemspec
-- lib/client.rb
 - lib/core.rb
 - lib/dsl_init.rb
-- lib/hadoop-dsl.rb
+- lib/hadoop_dsl.rb
+- lib/hadoop_dsl_client.rb
 - lib/hive_like.rb
 - lib/log_analysis.rb
 - lib/mapred_factory.rb
@@ -97,8 +93,6 @@ test_files:
 - spec/hive_like_spec.rb
 - spec/log_analysis_spec.rb
 - spec/example_spec.rb
-- examples/apachelog-v2.rb
 - examples/hive_like_test.rb
+- examples/log_analysis_test.rb
 - examples/word_count_test.rb
-- examples/apachelog-v2-2.rb
-- examples/apachelog.rb

data/TODO

@@ -1,2 +0,0 @@
-* entire error handling
-* "use" method not allowed double quote..

data/examples/apachelog-v2.rb

@@ -1,25 +0,0 @@
-use 'LogAnalysis'
-
-data 'apache log on test1' do
-  from 'apachlog/inputs'
-  to 'apachlog/outputs'
-
-  each_line do
-    pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/
-    column_name 'remote_host', 'pass', 'user', 'access_date', 'request', 'status', 'bytes' # 各カラムにラベルをつける
-
-    topic 'which users?', :label => 'user' do
-      count_uniq column[:user]
-    end
-
-#    topic 'access date by monthly' do
-#      select_date column[:access_date], BY_MONTHLY
-#      count column[:access_date]
-#    end
-#
-#    topic 'total bytes' do
-#      select_date column[:access_date], BY_MONTHLY
-#      sum column[:bytes].to_kilobytes # / 1024
-#    end
-  end
-end

data/examples/apachelog.rb

@@ -1,15 +0,0 @@
-# Apache log analysis
-#
-# example target data:
-#   127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
-#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb2.gif HTTP/1.0" 200 2326
-#   127.0.0.1 - frank2 [10/Oct/2000:13:55:36 -0700] "GET /apache_pb3.gif HTTP/1.0" 404 2326
-
-use 'LogAnalysis'
-
-data.pattern /(.*) (.*) (.*) (\[.*\]) (".*") (\d*) (\d*)/
-column[2].count_uniq
-column[3].count_uniq
-column[4].count_uniq
-column[5].count_uniq
-column[6].sum

data/lib/hadoop-dsl.rb

@@ -1,12 +0,0 @@
-require 'jruby-on-hadoop'
-require 'client'
-
-module HadoopDsl
-  def self.lib_path
-    File.expand_path(File.dirname(__FILE__))
-  end
-
-  def self.dsl_init_script
-    File.join(lib_path, "dsl_init.rb")
-  end
-end