td 0.10.65 → 0.10.66

data/ChangeLog

@@ -1,3 +1,10 @@
+
+== 2013-01-16 version 0.10.66
+
+* td-client v0.8.42
+* query, sched:create, sched:update subcommands support -R, --retry option
+
+
 == 2012-12-27 version 0.10.65
 
 * td-client v0.8.40

data/lib/td/command/import.rb

@@ -206,8 +206,8 @@ module Command
           record = {}
 
           cap = m.captures
-          @names.each_with_index {|name,i|
-            if value = cap[i]
+          @names.each_with_index {|name,cap_i|
+            if value = cap[cap_i]
               if name == "time"
                 value = parse_time(value).to_i
               end
@@ -275,7 +275,7 @@ module Command
 
   class MessagePackParser
     def initialize(time_key)
-      require 'json'
+      require 'msgpack'
       @time_key = time_key
     end
 

data/lib/td/command/job.rb

@@ -119,6 +119,7 @@ module Command
     puts "Status       : #{job.status}"
     puts "Type         : #{job.type}"
     puts "Priority     : #{job_priority_name_of(job.priority)}"
+    puts "Retry limit  : #{job.retry_limit}"
     puts "Result       : #{job.result_url}"
     puts "Database     : #{job.db_name}"
     puts "Query        : #{job.query}"

data/lib/td/command/query.rb

@@ -12,6 +12,7 @@ module Command
     result_user = nil
     result_ask_password = false
     priority = nil
+    retry_limit = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -46,6 +47,9 @@ module Command
         raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
       end
     }
+    op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
+      retry_limit = i
+    }
 
     sql = op.cmd_parse
 
@@ -68,7 +72,7 @@ module Command
     # local existance check
     get_database(client, db_name)
 
-    job = client.query(db_name, sql, result_url, priority)
+    job = client.query(db_name, sql, result_url, priority, retry_limit)
 
     $stderr.puts "Job #{job.job_id} is queued."
     $stderr.puts "Use '#{$prog} job:show #{job.job_id}' to show the status."

data/lib/td/command/sched.rb

@@ -32,6 +32,7 @@ module Command
     result_user = nil
     result_ask_password = false
     priority = nil
+    retry_limit = nil
 
     op.on('-d', '--database DB_NAME', 'use the database (required)') {|s|
       db_name = s
@@ -57,6 +58,9 @@ module Command
         raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
       end
     }
+    op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
+      retry_limit = i
+    }
 
     name, cron, sql = op.cmd_parse
 
@@ -76,7 +80,7 @@ module Command
     get_database(client, db_name)
 
     begin
-      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority)
+      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority, :retry_limit=>retry_limit)
     rescue AlreadyExistsError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' already exists."
@@ -111,6 +115,7 @@ module Command
     timezone = nil
     delay = nil
     priority = nil
+    retry_limit = nil
 
     op.on('-s', '--schedule CRON', 'change the schedule') {|s|
       cron = s
@@ -136,6 +141,10 @@ module Command
         raise "unknown priority #{s.inspect} should be -2 (very-low), -1 (low), 0 (normal), 1 (high) or 2 (very-high)"
       end
     }
+    op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
+      retry_limit = i
+    }
+
 
     name = op.cmd_parse
 
@@ -147,6 +156,7 @@ module Command
     params['timezone'] = timezone if timezone
     params['delay'] = delay.to_s if delay
     params['priority'] = priority.to_s if priority
+    params['retry_limit'] = retry_limit.to_s if retry_limit
 
     if params.empty?
       $stderr.puts op.to_s
@@ -209,6 +219,7 @@ module Command
       puts "Next         : #{s.next_time}"
       puts "Result       : #{s.result_url}"
       puts "Priority     : #{job_priority_name_of(s.priority)}"
+      puts "Retry limit  : #{s.retry_limit}"
       puts "Database     : #{s.database}"
       puts "Query        : #{s.query}"
     end

data/lib/td/file_reader.rb

@@ -1,6 +1,16 @@
 
 module TreasureData
+  # json and msgpack format supports array types with columns
+  #
+  # - when --column-header option
+  # ["a", "b", "c"] # first line is header
+  # ["v", 10, true] # array types, e.g. generate {"a" => "v", "b" => 10, "c" => true}
+  # ...
+  # - when --columns a,b,c
+  # ["v", 10, true] # array types
+  # ...
   class FileReader
+    require 'time'
     require 'zlib'
 
     class DecompressIOFilter
@@ -67,20 +77,36 @@ module TreasureData
       end
     end
 
-    # TODO
-    #class QuotedDelimiterParsingReader
-    #  def initialize(io, error, opts)
-    #    require 'strscan'
-    #    @io = io
-    #    @error = error
-    #    @delimiter_expr = opts[:delimiter_expr]
-    #    @quote_char = opts[:quote_char]
-    #    @escape_char = opts[:escape_char]
-    #  end
-
-    #  def forward
-    #  end
-    #end
+    # TODO: encoding handling
+    class SeparatedValueParsingReader
+      def initialize(io, error, opts)
+        if encoding = opts[:encoding]
+          io.set_encoding(encoding, :invalid => :replace, :undef => :replace) if io.respond_to?(:set_encoding)
+        end
+
+        # csv module is pure Ruby implementation.
+        # So this may cause slow performance in large dataset.
+        csv_opts = {
+          :col_sep => opts[:delimiter_expr],
+          :row_sep => $/,
+          :skip_blanks => true
+        }
+        csv_opts[:quote_char] = opts[:quote_char] if opts[:quote_char]
+        begin
+          require 'fastercsv'
+          @io = FasterCSV.new(io, csv_opts)
+        rescue LoadError => e
+          require 'csv'
+          @io = CSV.new(io, csv_opts)
+        end
+        @error = error
+        # @escape_char = opts[:escape_char]
+      end
+
+      def forward
+        @io.readline
+      end
+    end
 
     class JSONParser
       def initialize(reader, error, opts)
@@ -102,34 +128,68 @@ module TreasureData
       end
     end
 
-    # TODO
-    #class ApacheParser
-    #  REGEXP = /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/
-    #
-    #  def initialize(reader, error, opts)
-    #    @reader = reader
-    #  end
-    #
-    #  def forward
-    #    while true
-    #      m = REGEXP.match(@reader.forward_row)
-    #      if m
-    #        h = {
-    #          'host' => m[1],
-    #          'user' => m[2],
-    #          'time' => m[3],
-    #          'method' => m[4],
-    #          'path' => m[5],
-    #          'code' => m[6],
-    #          'size' => m[7].to_i,
-    #          'referer' => m[8],
-    #          'agent' => m[9],
-    #        }
-    #        return h
-    #      end
-    #    end
-    #  end
-    #end
+    # TODO: Support user defined format like in_tail
+    module RegexpParserMixin
+      def initialize(reader, error, opts)
+        @reader = reader
+        @error = error
+      end
+
+      def forward
+        while true
+          line = @reader.forward_row
+          begin
+            m = @regexp.match(line)
+            unless m
+              @error.call("invalid #{@format} format", line)
+              next
+            end
+
+            return m.captures
+          rescue
+            @error.call("skipped: #{$!}", line)
+            next
+          end
+        end
+      end      
+    end
+
+    # ApacheParser and SyslogParser is a port of old table:import's parsers
+
+    class ApacheParser
+      # 1.8 don't have named capture, so need column names.
+      COLUMNS = ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent']
+      TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
+
+      include RegexpParserMixin
+
+      def initialize(reader, error, opts)
+        super
+
+        # e.g. 127.0.0.1 - - [23/Oct/2011:08:20:01 -0700] "GET / HTTP/1.0" 200 492 "-" "Wget/1.12 (linux-gnu)"
+        @format = 'apache'
+        @regexp = /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/
+      end
+    end
+
+    class SyslogParser
+      # This parser doesn't consider nil value.
+      # But td platform removes the key, which has nil value, in data import.
+      # So this is not critical in table:import.
+
+      COLUMNS = ['time', 'host', 'ident', 'pid', 'message']
+      TIME_FORMAT = "%b %d %H:%M:%S"
+
+      include RegexpParserMixin
+
+      def initialize(reader, error, opts)
+        super
+
+        # e.g. Dec 20 12:41:44 localhost kernel:10000 [4843680.692840] e1000e: eth2 NIC Link is Down
+        @format = 'syslog'
+        @regexp = /^([^ ]* [^ ]* [^ ]*) ([^ ]*) ([a-zA-Z0-9_\/\.\-]*)(?:\[([0-9]+)\])?[^\:]*\: *(.*)$/
+      end
+    end
 
     class AutoTypeConvertParserFilter
       def initialize(parser, error, opts)
@@ -173,7 +233,6 @@ module TreasureData
 
     class TimeParserFilter
       def initialize(parser, error, opts)
-        require 'time'
         @parser = parser
         @error = error
         @time_column = opts[:time_column]
@@ -300,6 +359,7 @@ module TreasureData
         if s.to_i.to_s == s
           @opts[:time_value] = s.to_i
         else
+          require 'time'
           @opts[:time_value] = Time.parse(s).to_i
         end
       }
@@ -319,12 +379,17 @@ module TreasureData
       when 'tsv'
         @format = 'text'
         @opts[:delimiter_expr] = /\t/
-      #when 'apache'
-      #  @format = 'apache'
-      #  @opts[:column_names] = ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent']
-      #  @opts[:null_expr] = /\A(?:\-|)\z/
-      #  @opts[:time_column] = 'time'
-      #  @opts[:time_format] = '%d/%b/%Y:%H:%M:%S %z'
+      when 'apache'
+        @format = name
+        @opts[:column_names] = ApacheParser::COLUMNS
+        @opts[:null_expr] = /\A(?:\-|)\z/
+        @opts[:time_column] = 'time'
+        @opts[:time_format] = ApacheParser::TIME_FORMAT
+      when 'syslog'
+        @format = name
+        @opts[:column_names] = SyslogParser::COLUMNS
+        @opts[:time_column] = 'time'
+        @opts[:time_format] = SyslogParser::TIME_FORMAT
       when 'msgpack'
         @format = 'msgpack'
       when 'json'
@@ -360,7 +425,30 @@ module TreasureData
           end
         }
 
-      #when 'apache'
+      when 'apache', 'syslog'
+        Proc.new {|io,error|
+          io = DecompressIOFilter.filter(io, error, opts)
+          reader = LineReader.new(io, error, opts)
+          parser = if @format == 'apache'
+                     ApacheParser.new(reader, error, opts)
+                   else
+                     SyslogParser.new(reader, error, opts)
+                   end
+          if opts[:column_names]
+            column_names = opts[:column_names]
+          else
+            raise "--columns option is required"
+          end
+          unless opts[:all_string]
+            parser = AutoTypeConvertParserFilter.new(parser, error, opts)
+          end
+          parser = HashBuilder.new(parser, error, column_names)
+          if opts[:time_value]
+            parser = SetTimeParserFilter.new(parser, error, opts)
+          else
+            parser = TimeParserFilter.new(parser, error, opts)
+          end
+        }
 
       when 'json'
         Proc.new {|io,error|

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.65'
+VERSION = '0.10.66'
 
 end

data/spec/file_reader/filter_spec.rb

@@ -0,0 +1,236 @@
+require 'spec_helper'
+require 'file_reader/shared_context'
+
+require 'stringio'
+require 'td/file_reader'
+
+include TreasureData
+
+describe 'FileReader filters' do
+  include_context 'error_proc'
+
+  let :delimiter do
+    "\t"
+  end
+
+  let :dataset do
+    [
+      ['hoge', 12345, true,  'null', Time.now.to_s],
+      ['foo',  34567, false, 'null', Time.now.to_s],
+      ['piyo', 56789, true,  nil,    Time.now.to_s],
+    ]
+  end
+
+  let :lines do
+    dataset.map { |data| data.map(&:to_s).join(delimiter) }
+  end
+
+  let :parser do
+    io = StringIO.new(lines.join("\n"))
+    reader = FileReader::LineReader.new(io, error, {})
+    FileReader::DelimiterParser.new(reader, error, :delimiter_expr => delimiter)
+  end
+
+  describe FileReader::AutoTypeConvertParserFilter do
+    let :options do
+      { 
+        :null_expr => /\A(?:nil||\-|\\N)\z/i,
+        :true_expr => /\A(?:true)\z/i,
+        :false_expr => /\A(?:false)\z/i,
+      }
+    end
+
+    it 'initialize' do
+      filter = FileReader::AutoTypeConvertParserFilter.new(parser, error, options)
+      filter.should_not be_nil
+    end
+
+    context 'after initialization' do
+      let :filter do
+        FileReader::AutoTypeConvertParserFilter.new(parser, error, options)
+      end
+
+      it 'forward returns one converted line' do
+        filter.forward.should == dataset[0]
+      end
+
+      it 'feeds all lines' do
+        begin
+          i = 0
+          while line = filter.forward
+            line.should == dataset[i]
+            i += 1
+          end
+        rescue
+        end
+      end
+    end
+  end
+
+  describe FileReader::HashBuilder do
+    let :columns do
+      ['str', 'num', 'bool', 'null', 'log_at']
+    end
+
+    let :built_dataset do
+      # [{"str" => "hoge", "num" => "12345", "bool" => "true" , "null" =>"null", "log_at" => "2012-12-26 05:14:09 +0900"}, ...]
+      dataset.map { |data| Hash[columns.zip(data.map(&:to_s))]}
+    end
+
+    it 'initialize' do
+      builder = FileReader::HashBuilder.new(parser, error, columns)
+      builder.should_not be_nil
+    end
+
+    context 'after initialization' do
+      let :builder do
+        FileReader::HashBuilder.new(parser, error, columns)
+      end
+
+      it 'forward returns one converted line' do
+        builder.forward.should == built_dataset[0]
+      end
+
+      it 'feeds all lines' do
+        begin
+          i = 0
+          while line = builder.forward
+            line.should == built_dataset[i]
+            i += 1
+          end
+        rescue
+        end
+      end
+
+      describe FileReader::TimeParserFilter do
+        it "can't be initialized without :time_column option" do
+          expect { 
+            FileReader::TimeParserFilter.new(parser, error, {})
+          }.to raise_error(Exception, /--time-column/)
+        end
+
+        it 'initialize' do
+          filter = FileReader::TimeParserFilter.new(builder, error, :time_column => 'log_at')
+          filter.should_not be_nil
+        end
+
+        context 'after initialization' do
+          let :timed_dataset do
+            require 'time'
+            built_dataset.each { |data| data['time'] = Time.parse(data['log_at']).to_i }
+          end
+
+          let :filter do
+            FileReader::TimeParserFilter.new(builder, error, :time_column => 'log_at')
+          end
+
+          it 'forward returns one parse line with parsed log_at' do
+            filter.forward.should == timed_dataset[0]
+          end
+
+          it 'feeds all lines' do
+            begin
+              i = 0
+              while line = filter.forward
+                line.should == timed_dataset[i]
+                i += 1
+              end
+            rescue
+            end
+          end
+
+          context 'missing log_at column lines' do
+            let :columns do
+              ['str', 'num', 'bool', 'null', 'created_at']
+            end
+
+            let :error_pattern do
+              /^time column 'log_at' is missing/
+            end
+
+            it 'feeds all lines' do
+              i = 0
+              begin
+                while line = filter.forward
+                  i += 1
+                end
+              rescue RSpec::Expectations::ExpectationNotMetError => e
+                fail
+              rescue
+                i.should == 0
+              end
+            end
+          end
+
+          context 'invalid time format' do
+            let :error_pattern do
+              /^invalid time format/
+            end
+
+            [{:time_column => 'log_at', :time_format => "%d"},
+             {:time_column => 'str'}].each { |options|
+              let :filter do
+                FileReader::TimeParserFilter.new(builder, error, options)
+              end
+
+              it 'feeds all lines' do
+                i = 0
+                begin
+                  while line = filter.forward
+                    i += 1
+                  end
+                rescue RSpec::Expectations::ExpectationNotMetError => e
+                  fail
+                rescue
+                  i.should == 0
+                end
+              end
+            }
+          end
+        end
+      end
+
+      describe FileReader::SetTimeParserFilter do
+        it "can't be initialized without :time_value option" do
+          expect { 
+            FileReader::SetTimeParserFilter.new(parser, error, {})
+          }.to raise_error(Exception, /--time-value/)
+        end
+
+        it 'initialize' do
+          filter = FileReader::SetTimeParserFilter.new(builder, error, :time_value => Time.now.to_i)
+          filter.should_not be_nil
+        end
+
+        context 'after initialization' do
+          let :time_value do
+            Time.now.to_i
+          end
+
+          let :timed_dataset do
+            built_dataset.each { |data| data['time'] = time_value }
+          end
+
+          let :filter do
+            FileReader::SetTimeParserFilter.new(builder, error, :time_value => time_value)
+          end
+
+          it 'forward returns one converted line with time' do
+            filter.forward.should == timed_dataset[0]
+          end
+
+          it 'feeds all lines' do
+            begin
+              i = 0
+              while line = filter.forward
+                line.should == timed_dataset[i]
+                i += 1
+              end
+            rescue
+            end
+          end
+        end
+      end
+    end
+  end
+end