hadoop-rubydsl 0.0.4 → 0.0.5

data/README.rdoc

@@ -26,7 +26,7 @@ You can get Hadoop job results in your hdfs wc/outputs/part-*
 == Examples
 
 Word Count DSL script
- use 'WordCount'
+ dsl 'WordCount'
 
  from 'wc/inputs'
  to 'wc/outputs'
@@ -35,7 +35,7 @@ Word Count DSL script
  total :bytes, :words, :lines
 
 Log Analysis DSL script
- use 'LogAnalysis'
+ dsl 'LogAnalysis'
  
  data 'apache log on test2' do
    from 'apachelog/inputs'

data/VERSION

@@ -1 +1 @@
-0.0.4
+0.0.5

data/examples/log_analysis_test.rb

@@ -23,14 +23,16 @@ data 'apache log on test2' do
 
     topic 'ua counts group by path' do
       request = column[:request].value
-      path = request.split(/\s+/)[1]
-      group_by path
+      if request
+        path = request.split(/\s+/)[1]
+        group_by path
+      end
       count_uniq column[:ua]
     end
 
     topic 'ua counts by daily' do
-      group_date_by column[:access_date], :daily
-      count_uniq column[:ua]
+#      group_date_by column[:access_date], :daily
+#      count_uniq column[:ua]
     end
 
 #    topic 'total bytes' do

data/hadoop-rubydsl.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{hadoop-rubydsl}
-  s.version = "0.0.4"
+  s.version = "0.0.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = %q{2010-01-13}
+  s.date = %q{2010-01-28}
   s.description = %q{Hadoop Ruby DSL}
   s.email = %q{fujibee@gmail.com}
   s.executables = ["hrd", "hadoop-hudson.sh", "hadoop-ruby.sh"]

data/lib/core.rb

@@ -5,8 +5,9 @@ module HadoopDsl
   # common
   module DslElement
     # all DSL statements without def is processed here
-    def method_missing(method_name, *args)
-      yield if block_given?
+    def method_missing(name, *args)
+      # if block given, labeled for non-local exit
+      catch name do; yield end if block_given?
       self
     end
   end

data/lib/log_analysis.rb

@@ -9,6 +9,8 @@ module HadoopDsl::LogAnalysis
 
   # controller
   class LogAnalysisMapper < HadoopDsl::BaseMapper
+    @@reg_cache = {}
+
     def initialize(script, key, value)
       super(script, LogAnalysisMapperModel.new(key, value))
     end
@@ -27,10 +29,16 @@ module HadoopDsl::LogAnalysis
       @model.create_or_replace_columns_with(parts) {|column, value| column.value = value}
     end
 
-    def pattern(re)
+    def pattern(reg_str)
+      # try to get RE from cache
+      cached = @@reg_cache[reg_str] 
+      re = cached ? @@reg_cache[reg_str] : Regexp.new(reg_str)
+      @@reg_cache[reg_str] ||= re # new cache
+
       if value =~ re
         md = Regexp.last_match
         @model.create_or_replace_columns_with(md.captures) {|column, value| column.value = value}
+      else throw :each_line # non-local exit
       end
     end
 

data/lib/mapred_factory.rb

@@ -15,20 +15,31 @@ module HadoopDsl
   end
 
   class MapperFactory < MapRedFactory
+    # for cache in map loop
+    @@mapper_class = nil
     def self.create(script, key, value)
-      dsl_name = self.dsl_name(script)
-      require_dsl_lib(dsl_name)
-      mapper_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Mapper" 
-      return eval(mapper_class).new(script, key, value)
+      # once decide in map loop
+      unless @@mapper_class
+        dsl_name = self.dsl_name(script)
+        require_dsl_lib(dsl_name)
+        @@mapper_class = eval("HadoopDsl::#{dsl_name}::#{dsl_name}Mapper")
+      end
+
+      @@mapper_class.new(script, key, value)
     end
   end
 
   class ReducerFactory < MapRedFactory
+    @@reducer_class = nil
     def self.create(script, key, values)
-      dsl_name = self.dsl_name(script)
-      require_dsl_lib(dsl_name)
-      reducer_class = "HadoopDsl::#{dsl_name}::#{dsl_name}Reducer" 
-      return eval(reducer_class).new(script, key, values)
+      # once decide in reduce loop
+      unless @@reducer_class
+        dsl_name = self.dsl_name(script)
+        require_dsl_lib(dsl_name)
+        @@reducer_class = eval("HadoopDsl::#{dsl_name}::#{dsl_name}Reducer")
+      end
+
+      @@reducer_class.new(script, key, values)
     end
   end
 

data/lib/util.rb

@@ -2,21 +2,37 @@
 require 'hadoop_dsl'
 
 module HadoopDsl
+  # file body cache
+  # reading file in map/reduce cause critical issues!
+  @@file_bodies = {}
+
   def self.snake_case(str)
     str.gsub(/\B[A-Z]/, '_\&').downcase
   end
 
   def self.read_file(file_name)
+    # use if cached
+    body = @@file_bodies[file_name] if @@file_bodies[file_name]
+
     # read as usual
-    body = File.open(file_name).read rescue nil
-    return body if body
+    body = File.open(file_name).read rescue nil unless body
 
     # read from loadpath
-    $:.each do |path|
-      body = File.open(File.join(path, file_name)).read rescue next
-      return body if body
+    unless body
+      $:.each do |path|
+        body = File.open(File.join(path, file_name)).read rescue next
+        break
+      end
     end
 
-    raise "cannot find file - #{file_name}"
+    raise "cannot find file - #{file_name}" unless body
+
+    # for cache
+    @@file_bodies[file_name] = body
+    body
+  end
+
+  def self.reset_dsl_file
+    @@file_bodies = {}
   end
 end

data/spec/log_analysis_spec.rb

@@ -23,6 +23,15 @@ describe LogAnalysisMapper do
     mapper.column[2].value.should == 'frank'
   end
 
+  it 'should non-local exit if cannot separate by pattern' do
+    mapper = LogAnalysisMapper.new(nil, nil, @apache_log + " a")
+    mapper.each_line do
+      mapper.pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)$/
+      fail 'should not be reached'
+    end
+    mapper.column[0].should be_nil
+  end
+
   it 'should label column name by string' do
     mapper = LogAnalysisMapper.new(nil, nil, @apache_log)
     mapper.pattern /(.*) (.*) (.*) \[(.*)\] (".*") (\d*) (\d*)/

data/spec/mapred_factory_spec.rb

@@ -59,4 +59,16 @@ describe 'MapRed Factory' do
     mapper = MapperFactory.create(script, nil, nil)
     mapper.class.should == LogAnalysis::LogAnalysisMapper
   end
+
+  it 'can create mapper from class name cache' do
+    mapper = MapperFactory.create(@script, nil, nil)
+    mapper2 = MapperFactory.create(@script, nil, nil)
+    mapper.class.should == mapper2.class
+  end
+
+  it 'can create reducer from class name cache' do
+    reducer = ReducerFactory.create(@script, nil, nil)
+    reducer2 = ReducerFactory.create(@script, nil, nil)
+    reducer.class.should == reducer2.class
+  end
 end

data/spec/util_spec.rb

@@ -2,17 +2,33 @@ require File.join(File.dirname(__FILE__) , 'spec_helper')
 require 'util'
 
 describe 'utilities' do
+  before do
+    HadoopDsl.reset_dsl_file
+    @script_body = 'This is a script body.'
+    @script = create_tmp_script(@script_body)
+  end
+
   it 'can change camelcase str to snakecase' do
     HadoopDsl.snake_case('CamelCaseStr').should == 'camel_case_str'
   end
 
   it 'can read file and get file data to string' do
-    script_body = 'This is a script body.'
-    @script = create_tmp_script(script_body)
-    HadoopDsl.read_file(@script).should == script_body
+    HadoopDsl.read_file(@script).should == @script_body
   end
 
   it 'raise error if no file in loadpath' do
     lambda { HadoopDsl.read_file('not_exists_on_loadpath') }.should raise_error
   end
+
+  it 'can load from cache if script is loaded' do
+    HadoopDsl.read_file(@script).should == @script_body
+    File.delete(@script)
+    HadoopDsl.read_file(@script).should == @script_body
+  end
+
+  it 'can load from each cache even if one script is loaded' do
+    HadoopDsl.read_file(@script).should == @script_body
+    another_script = create_tmp_script("another")
+    HadoopDsl.read_file(another_script).should == "another"
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: hadoop-rubydsl
 version: !ruby/object:Gem::Version 
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors: 
 - Koichi Fujikawa
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-13 00:00:00 +09:00
+date: 2010-01-28 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency