td 0.10.44 → 0.10.45

data/ChangeLog

@@ -1,4 +1,11 @@
 
+== 2012-09-12 version 0.10.45
+
+* bulk_import:prepare_parts supports --true and --false type conversion options
+* bulk_import:prepare_parts supports --time-value option
+* bulk_import:prepare_parts convers /null/i to NULL by default
+
+
 == 2012-09-04 version 0.10.44
 
 * Added password:change subcommand

data/lib/td/file_reader.rb

@@ -10,8 +10,8 @@ module TreasureData
         @u = MessagePack::Unpacker.new(@io)
       end
 
-      def next
-        @u.next
+      def forward
+        @u.forward
       end
     end
 
@@ -25,7 +25,7 @@ module TreasureData
         @error = error
       end
 
-      def next_row
+      def forward_row
         @io.readline($/).chomp
       end
     end
@@ -34,15 +34,11 @@ module TreasureData
       def initialize(reader, error, opts)
         @reader = reader
         @delimiter_expr = opts[:delimiter_expr]
-        @null_expr = opts[:null_expr]
       end
 
-      def next
-        row = @reader.next_row
+      def forward
+        row = @reader.forward_row
         array = row.split(@delimiter_expr)
-        array.map! {|x|
-          @null_expr =~ x ? nil : x
-        }
       end
     end
 
@@ -57,7 +53,7 @@ module TreasureData
     #    @escape_char = opts[:escape_char]
     #  end
 
-    #  def next
+    #  def forward
     #  end
     #end
 
@@ -67,9 +63,9 @@ module TreasureData
         @error = error
       end
 
-      def next
+      def forward
         while true
-          line = @reader.next_row
+          line = @reader.forward_row
           begin
             return JSON.parse(line)
           rescue
@@ -88,9 +84,9 @@ module TreasureData
     #    @reader = reader
     #  end
     #
-    #  def next
+    #  def forward
     #    while true
-    #      m = REGEXP.match(@reader.next_row)
+    #      m = REGEXP.match(@reader.forward_row)
     #      if m
     #        h = {
     #          'host' => m[1],
@@ -110,16 +106,29 @@ module TreasureData
     #end
 
     class AutoTypeConvertParserFilter
-      def initialize(parser, error)
+      def initialize(parser, error, opts)
         @parser = parser
+        @null_expr = opts[:null_expr]
+        @true_expr = opts[:true_expr]
+        @false_expr = opts[:false_expr]
       end
 
-      def next
-        array = @parser.next
+      def forward
+        array = @parser.forward
         array.map! {|s|
-          # nil.to_i == 0 != nil.to_s
-          i = s.to_i
-          i.to_s == s ? i : s
+          if !s.is_a?(String)
+            s
+          elsif s =~ @null_expr
+            nil
+          elsif s =~ @true_expr
+            true
+          elsif s =~ @false_expr
+            false
+          else
+            # nil.to_i == 0 != nil.to_s
+            i = s.to_i
+            i.to_s == s ? i : s
+          end
         }
       end
     end
@@ -130,8 +139,8 @@ module TreasureData
         @columns = columns
       end
 
-      def next
-        array = @parser.next
+      def forward
+        array = @parser.forward
         Hash[@columns.zip(array)]
       end
     end
@@ -143,14 +152,14 @@ module TreasureData
         @error = error
         @time_column = opts[:time_column]
         unless @time_column
-          raise '-t, --time-column NAME option is required'
+          raise '--time-column or --time-value option is required'
         end
         @time_format = opts[:time_format]
       end
 
-      def next
+      def forward
         while true
-          row = @parser.next
+          row = @parser.forward
           tval = row[@time_column]
 
           unless tval
@@ -186,12 +195,31 @@ module TreasureData
       end
     end
 
+    class SetTimeParserFilter
+      def initialize(parser, error, opts)
+        @parser = parser
+        @error = error
+        @time_value = opts[:time_value]
+        unless @time_value
+          raise '--time-column or --time-value option is required'
+        end
+      end
+
+      def forward
+        row = @parser.forward
+        row['time'] = @time_value
+        row
+      end
+    end
+
     def initialize
       @format = "text"
       @default_opts = {
         :delimiter_expr => /\t|,/,
         #:line_delimiter_expr => /\r?\n/,
-        :null_expr => /\A(?:\\N|\-|)\z/,
+        :null_expr => /\A(?:null||\-|\\N)\z/i,
+        :true_expr => /\A(?:true)\z/i,
+        :false_expr => /\A(?:false)\z/i,
         #:quote_char => "\"",
       }
       @opts = {}
@@ -211,15 +239,21 @@ module TreasureData
       op.on('-H', '--column-header', 'first line includes column names', TrueClass) {|b|
         @opts[:column_header] = b
       }
-      op.on('-d', '--delimiter REGEX', "delimiter between columns (default: #{@default_opts[:delimiter_expr].inspect[1..-2]})") {|s|
+      op.on('-d', '--delimiter REGEX', "delimiter between columns (default: #{@default_opts[:delimiter_expr].to_s})") {|s|
         @opts[:delimiter_expr] = Regexp.new(s)
       }
       #op.on('-D', '--line-delimiter REGEX', "delimiter between rows (default: #{@default_opts[:line_delimiter_expr].inspect[1..-2]})") {|s|
       #  @opts[:line_delimiter_expr] = Regexp.new(s)
       #}
-      op.on('-N', '--null REGEX', "null expression (default: #{@default_opts[:null_expr].inspect[1..-2]})") {|s|
+      op.on('--null REGEX', "null expression for the automatic type conversion (default: #{@default_opts[:null_expr].to_s})") {|s|
         @opts[:null_expr] = Regexp.new(s)
       }
+      op.on('--true REGEX', "true expression for the automatic type conversion (default: #{@default_opts[:true_expr].to_s})") {|s|
+        @opts[:true_expr] = Regexp.new(s)
+      }
+      op.on('--false REGEX', "false expression for the automatic type conversion (default: #{@default_opts[:false_expr].to_s})") {|s|
+        @opts[:false_expr] = Regexp.new(s)
+      }
       # TODO
       #op.on('-E', '--escape CHAR', "escape character (default: no escape character)") {|s|
       #  @opts[:escape_char] = s
@@ -230,12 +264,19 @@ module TreasureData
       op.on('-S', '--all-string', 'disable automatic type conversion', TrueClass) {|b|
         @opts[:all_string] = b
       }
-      op.on('-t', '--time-column NAME', 'name of the time column (default: auto detect)') {|s|
+      op.on('-t', '--time-column NAME', 'name of the time column') {|s|
         @opts[:time_column] = s
       }
       op.on('-T', '--time-format FORMAT', 'strftime(3) format of the time column') {|s|
         @opts[:time_format] = s
       }
+      op.on('--time-value TIME', 'value of the time column') {|s|
+        if s.to_i.to_s == s
+          @opts[:time_value] = s.to_i
+        else
+          @opts[:time_value] = Time.parse(s).to_i
+        end
+      }
       op.on('-e', '--encoding NAME', "text encoding") {|s|
         @opts[:encoding] = s
       }
@@ -275,17 +316,21 @@ module TreasureData
           reader = LineReader.new(io, error, opts)
           parser = DelimiterParser.new(reader, error, opts)
           if opts[:column_header]
-            column_names = parser.next
+            column_names = parser.forward
           elsif opts[:column_names]
             column_names = opts[:column_names]
           else
             raise "--column-header or --columns option is required"
           end
           unless opts[:all_string]
-            parser = AutoTypeConvertParserFilter.new(parser, error)
+            parser = AutoTypeConvertParserFilter.new(parser, error, opts)
           end
           parser = HashBuilder.new(parser, error, column_names)
-          parser = TimeParserFilter.new(parser, error, opts)
+          if opts[:time_value]
+            parser = SetTimeParserFilter.new(parser, error, opts)
+          else
+            parser = TimeParserFilter.new(parser, error, opts)
+          end
         }
 
       #when 'apache'
@@ -295,28 +340,36 @@ module TreasureData
           reader = LineReader.new(io, error, opts)
           parser = JSONParser.new(reader, error, opts)
           if opts[:column_header]
-            column_names = parser.next
+            column_names = parser.forward
           elsif opts[:column_names]
             column_names = opts[:column_names]
           end
           if column_names
             parser = HashBuilder.new(parser, error, column_names)
           end
-          parser = TimeParserFilter.new(parser, error, opts)
+          if opts[:time_value]
+            parser = SetTimeParserFilter.new(parser, error, opts)
+          else
+            parser = TimeParserFilter.new(parser, error, opts)
+          end
         }
 
       when 'msgpack'
         Proc.new {|io,error|
           parser = MessagePackParsingReader.new(io, error, opts)
           if opts[:column_header]
-            column_names = parser.next
+            column_names = parser.forward
           elsif opts[:column_names]
             column_names = opts[:column_names]
           end
           if column_names
             parser = HashBuilder.new(parser, error, column_names)
           end
-          parser = TimeParserFilter.new(parser, error, opts)
+          if opts[:time_value]
+            parser = SetTimeParserFilter.new(parser, error, opts)
+          else
+            parser = TimeParserFilter.new(parser, error, opts)
+          end
         }
       end
     end
@@ -325,7 +378,7 @@ module TreasureData
       factory = compose_factory
       parser = factory.call(io, error)
       begin
-        while record = parser.next
+        while record = parser.forward
           block.call(record)
         end
       rescue EOFError

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.44'
+VERSION = '0.10.45'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.44
+  version: 0.10.45
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-04 00:00:00.000000000 Z
+date: 2012-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack