td 0.10.38 → 0.10.39

data/ChangeLog

@@ -1,4 +1,9 @@
 
+== 2012-08-07 version 0.10.39
+
+* Added bulk_import:prepare_parts and bulk_import:upload_parts subcommands
+
+
 == 2012-08-06 version 0.10.38
 
 * *:list and *:show: subcommands show organization name if it's set

data/lib/td/command/acl.rb

@@ -35,7 +35,7 @@ module Command
 
     client.grant_access_control(subject, action, scope, grant_option)
 
-    $stderr.puts "Access control [#{subject} #{action} #{scope}] is created #{grant_option? ? 'with' : 'without'} grant option."
+    $stderr.puts "Access control [#{subject} #{action} #{scope}] is created #{grant_option ? 'with' : 'without'} grant option."
   end
 
   def acl_revoke(op)

data/lib/td/command/bulk_import.rb

@@ -85,17 +85,48 @@ module Command
   end
 
   def bulk_import_upload_part(op)
-    name, part_name, path = op.cmd_parse
+    retry_limit = 10
+    retry_wait = 1
 
-    client = get_client
+    name, part_name, path = op.cmd_parse
 
-    File.open(path, "rb") {|is|
-      client.bulk_import_upload_part(name, part_name, is, is.size)
+    File.open(path, "rb") {|io|
+      bulk_import_upload_impl(name, part_name, io, io.size, retry_limit, retry_wait)
     }
 
     $stderr.puts "Part '#{part_name}' is uploaded."
   end
 
+  def bulk_import_upload_parts(op)
+    retry_limit = 10
+    retry_wait = 1
+    suffix_count = 0
+    part_prefix = ""
+
+    op.on('-P', '--prefix NAME', 'add prefix to parts name') {|s|
+      part_prefix = s
+    }
+    op.on('-s', '--use-suffix COUNT', 'use COUNT number of . (dots) in the source file name to the parts name', Integer) {|i|
+      suffix_count = i
+    }
+
+    name, *files = op.cmd_parse
+
+    files.each {|ifname|
+      basename = File.basename(ifname)
+      part_name = part_prefix + basename.split('.')[0..suffix_count].join('.')
+
+      File.open(ifname, "rb") {|io|
+        size = io.size
+        $stderr.puts "Uploading '#{ifname}' -> '#{part_name}'... (#{size} bytes)"
+
+        bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
+      }
+    }
+
+    $stderr.puts "done."
+  end
+
   def bulk_import_delete_part(op)
     name, part_name = op.cmd_parse
 
@@ -203,6 +234,105 @@ module Command
     $stderr.puts "Bulk import session '#{name}' is unfrozen."
   end
 
+
+  PART_SPLIT_SIZE = 16*1024*1024
+
+  def bulk_import_prepare_part(op)
+    outdir = nil
+    split_size_kb = PART_SPLIT_SIZE / 1024  # kb
+
+    require 'td/file_reader'
+    reader = FileReader.new
+    reader.init_optparse(op)
+
+    op.on('-s', '--split-size SIZE_IN_KB', "size of each parts (default: #{split_size_kb})", Integer) {|i|
+      split_size_kb = i
+    }
+    op.on('-o', '--output DIR', 'output directory') {|s|
+      outdir = s
+    }
+
+    *files = op.cmd_parse
+
+    unless outdir
+      $stderr.puts "-o, --output DIR option is required."
+      exit 1
+    end
+
+    split_size = split_size_kb * 1024
+
+    require 'fileutils'
+    FileUtils.mkdir_p(outdir)
+
+    require 'json'
+    require 'msgpack'
+    require 'zlib'
+
+    error = Proc.new {|reason,data|
+      begin
+        $stderr.puts "#{reason}: #{data.to_json}"
+      rescue
+        $stderr.puts "#{reason}"
+      end
+    }
+
+    files.each {|ifname|
+      $stderr.puts "Processing #{ifname}..."
+      record_num = 0
+
+      basename = File.basename(ifname).split('.').first
+      File.open(ifname) {|io|
+        of_index = 0
+        out = nil
+        zout = nil
+        begin
+          reader.parse(io, error) {|record|
+            if zout == nil
+              ofname = "#{basename}_#{of_index}.msgpack.gz"
+              $stderr.puts "  Preparing part \"#{basename}_#{of_index}\"..."
+              out = File.open("#{outdir}/#{ofname}", 'wb')
+              zout = Zlib::GzipWriter.new(out)
+
+              t = record['time']
+              $stderr.puts "  sample: #{Time.at(t).utc} #{record.to_json}"
+            end
+
+            zout.write(record.to_msgpack)
+            record_num += 1
+
+            if out.size > split_size
+              zout.close
+              of_index += 1
+              out = nil
+              zout = nil
+            end
+          }
+        ensure
+          if zout
+            zout.close
+            zout = nil
+          end
+        end
+        $stderr.puts "  #{ifname}: #{record_num} entries."
+      }
+    }
+  end
+
+  private
+  def bulk_import_upload_impl(name, part_name, io, size, retry_limit, retry_wait)
+    begin
+      client = get_client
+      client.bulk_import_upload_part(name, part_name, io, size)
+    rescue
+      if retry_limit > 0
+        retry_limit -= 1
+        $stderr.puts "#{$!}; retrying '#{part_name}'..."
+        sleep retry_wait
+        retry
+      end
+      raise
+    end
+  end
 end
 end
 

data/lib/td/command/list.rb

@@ -228,7 +228,9 @@ module List
   add_list 'bulk_import:list', %w[], 'List bulk import sessions', 'bulk_import:list'
   add_list 'bulk_import:show', %w[name], 'Show list of uploaded parts', 'bulk_import:show'
   add_list 'bulk_import:create', %w[name db table], 'Create a new bulk import session to the the table', 'bulk_import:create logs_201201 example_db event_logs'
+  add_list 'bulk_import:prepare_part', %w[files_], 'Convert files into part file format', 'bulk_import:prepare_part logs/*.csv --format csv --columns time,uid,price,count --time-column "time" -o parts/'
   add_list 'bulk_import:upload_part', %w[name id path.msgpack.gz], 'Upload or re-upload a file into a bulk import session', 'bulk_import:upload_part logs_201201 01h data-201201-01.msgpack.gz'
+  add_list 'bulk_import:upload_parts', %w[name files_], 'Upload or re-upload files into a bulk import session', 'bulk_import:upload_parts parts/* --prefix logs_'
   add_list 'bulk_import:delete_part', %w[name id], 'Delete a uploaded file from a bulk import session', 'bulk_import:delete_part logs_201201 01h'
   add_list 'bulk_import:perform', %w[name], 'Start to validate and convert uploaded files', 'bulk_import:perform logs_201201'
   add_list 'bulk_import:error_records', %w[name], 'Show records which did not pass validations', 'bulk_import:error_records logs_201201'

data/lib/td/file_reader.rb

@@ -0,0 +1,324 @@
+
+module TreasureData
+  class FileReader
+
+    class MessagePackParsingReader
+      def initialize(io, error, opts)
+        require 'msgpack'
+        @io = io
+        @error = error
+        @u = MessagePack::Unpacker.new(@io)
+      end
+
+      def next
+        @u.next
+      end
+    end
+
+    class LineReader
+      def initialize(io, error, opts)
+        if encoding = opts[:encoding]
+          io.external_encoding = encoding
+        end
+        #@delimiter = opts[:line_delimiter_expr] || /\r?\n/
+        @io = io
+        @error = error
+      end
+
+      def next_row
+        @io.readline($/).chomp
+      end
+    end
+
+    class DelimiterParser
+      def initialize(reader, error, opts)
+        @reader = reader
+        @delimiter_expr = opts[:delimiter_expr]
+        @null_expr = opts[:null_expr]
+        # TODO
+        #@escape_char = opts[:escape_char]
+        #@quote_char = opts[:quote_char]
+      end
+
+      def next
+        row = @reader.next_row
+        array = row.split(@delimiter_expr)
+        array.map! {|x|
+          @null_expr =~ x ? nil : x
+        }
+      end
+    end
+
+    class JSONParser
+      def initialize(reader, error, opts)
+        @reader = reader
+        @error = error
+      end
+
+      def next
+        while true
+          line = @reader.next_row
+          begin
+            return JSON.parse(line)
+          rescue
+            @error.call("invalid json format: #{$!}", line)
+            next
+          end
+        end
+      end
+    end
+
+    #class ApacheParser
+    #  REGEXP = /^([^ ]*) [^ ]* ([^ ]*) \[([^\]]*)\] "(\S+)(?: +([^ ]*) +\S*)?" ([^ ]*) ([^ ]*)(?: "([^\"]*)" "([^\"]*)")?$/
+    #
+    #  def initialize(reader, error, opts)
+    #    @reader = reader
+    #  end
+    #
+    #  def next
+    #    while true
+    #      m = REGEXP.match(@reader.next_row)
+    #      if m
+    #        h = {
+    #          'host' => m[1],
+    #          'user' => m[2],
+    #          'time' => m[3],
+    #          'method' => m[4],
+    #          'path' => m[5],
+    #          'code' => m[6],
+    #          'size' => m[7].to_i,
+    #          'referer' => m[8],
+    #          'agent' => m[9],
+    #        }
+    #        return h
+    #      end
+    #    end
+    #  end
+    #end
+
+    class AutoTypeConvertParserFilter
+      def initialize(parser, error)
+        @parser = parser
+      end
+
+      def next
+        array = @parser.next
+        array.map! {|s|
+          # nil.to_i == 0 != nil.to_s
+          i = s.to_i
+          i.to_s == s ? i : s
+        }
+      end
+    end
+
+    class HashBuilder
+      def initialize(parser, error, columns)
+        @parser = parser
+        @columns = columns
+      end
+
+      def next
+        array = @parser.next
+        # FIXME error handling
+        Hash[@columns.zip(array)]
+      end
+    end
+
+    class TimeParserFilter
+      def initialize(parser, error, opts)
+        require 'time'
+        @parser = parser
+        @error = error
+        @time_column = opts[:time_column]
+        unless @time_column
+          raise '-t, --time-column NAME option is required'
+        end
+        @time_format = opts[:time_format]
+      end
+
+      def next
+        while true
+          row = @parser.next
+          tval = row[@time_column]
+
+          unless tval
+            @error.call("time column '#{@time_column}' is missing", row)
+            next
+          end
+
+          begin
+            if tf = @time_format
+              row['time'] = parse_time(tval, tf).to_i
+            elsif tval.is_a?(Integer)
+                row['time'] = tval
+            else
+              row['time'] = Time.parse(tval).to_i
+            end
+            return row
+
+          rescue
+            @error.call("invalid time format '#{tval}': #{$!}", row)
+            next
+          end
+        end
+      end
+
+      if Time.respond_to?(:strptime)
+        def parse_time(value, format)
+          Time.strptime(value, format)
+        end
+      else
+        def parse_time(value, format)
+          Time.parse(DateTime.strptime(value, format).to_s)
+        end
+      end
+    end
+
+    def initialize
+      @format = "text"
+      @default_opts = {
+        :delimiter_expr => /\t|,/,
+        #:line_delimiter_expr => /\r?\n/,
+        :null_expr => /\A(?:\\N|\-|)\z/,
+        #:quote_char => "\"",
+      }
+      @opts = {}
+      @parser_class = nil
+    end
+
+    attr_reader :default_opts, :opts
+    attr_accessor :parser_class
+
+    def init_optparse(op)
+      op.on('-f', '--format NAME', "source file format") {|s|
+        set_format_template(s)
+      }
+      op.on('-h', '--columns NAME,NAME,...', 'column names') {|s|
+        @opts[:column_names] = s.split(',')
+      }
+      op.on('-H', '--column-header', 'first line includes column names', TrueClass) {|b|
+        @opts[:column_header] = b
+      }
+      op.on('-d', '--delimiter REGEX', "delimiter between columns (default: #{@default_opts[:delimiter_expr].inspect[1..-2]}") {|s|
+        @opts[:delimiter_expr] = Regexp.new(s)
+      }
+      #op.on('-D', '--line-delimiter REGEX', "delimiter between rows (default: #{@default_opts[:line_delimiter_expr].inspect[1..-2]})") {|s|
+      #  @opts[:line_delimiter_expr] = Regexp.new(s)
+      #}
+      op.on('-N', '--null REGEX', "null expression (default: #{@default_opts[:null_expr].inspect[1..-2]}") {|s|
+        @opts[:null_expr] = Regexp.new(s)
+      }
+      # TODO
+      #op.on('-E', '--escape CHAR', "escape character (default: no escape character)") {|s|
+      #  @opts[:escape_char] = s
+      #}
+      #op.on('-Q', '--quote CHAR', "quote character (default: #{@default_opts[:quote_char]}") {|s|
+      #  @opts[:quote_char] = s
+      #}
+      op.on('-S', '--all-string', 'disable automatic type conversion', TrueClass) {|b|
+        @opts[:all_string] = b
+      }
+      op.on('-t', '--time-column NAME', 'name of the time column (default: auto detect)') {|s|
+        @opts[:time_column] = s
+      }
+      op.on('-T', '--time-format FORMAT', 'strftime(3) format of the time column') {|s|
+        @opts[:time_format] = s
+      }
+      op.on('-e', '--encoding NAME', "text encoding") {|s|
+        @opts[:encoding] = s
+      }
+      op.on('-C', '--compress NAME', 'compression format name [plain, gzip] (default: auto detect)') {|s|
+        @opts[:compress] = s
+      }
+    end
+
+    def set_format_template(name)
+      case name
+      when 'csv'
+        @format = 'text'
+        @opts[:delimiter_expr] = /,/
+      when 'tsv'
+        @format = 'text'
+        @opts[:delimiter_expr] = /\t/
+      #when 'apache'
+      #  @format = 'apache'
+      #  @opts[:column_names] = ['host', 'user', 'time', 'method', 'path', 'code', 'size', 'referer', 'agent']
+      #  @opts[:null_expr] = /\A(?:\-|)\z/
+      #  @opts[:time_column] = 'time'
+      #  @opts[:time_format] = '%d/%b/%Y:%H:%M:%S %z'
+      when 'msgpack'
+        @format = 'msgpack'
+      when 'json'
+        @format = 'json'
+      else
+        raise "Unknown format: #{name}"
+      end
+    end
+
+    def compose_factory
+      opts = @default_opts.merge(@opts)
+      case @format
+      when 'text'
+        Proc.new {|io,error|
+          reader = LineReader.new(io, error, opts)
+          parser = DelimiterParser.new(reader, error, opts)
+          if opts[:column_header]
+            column_names = parser.next
+          elsif opts[:column_names]
+            column_names = opts[:column_names]
+          else
+            raise "--column-header or --columns option is required"
+          end
+          unless opts[:all_string]
+            parser = AutoTypeConvertParserFilter.new(parser, error)
+          end
+          parser = HashBuilder.new(parser, error, column_names)
+          parser = TimeParserFilter.new(parser, error, opts)
+        }
+
+      #when 'apache'
+
+      when 'json'
+        Proc.new {|io,error|
+          reader = LineReader.new(io, error, opts)
+          parser = JSONParser.new(reader, error, opts)
+          if opts[:column_header]
+            column_names = parser.next
+          elsif opts[:column_names]
+            column_names = opts[:column_names]
+          end
+          if column_names
+            parser = HashBuilder.new(parser, error, column_names)
+          end
+          parser = TimeParserFilter.new(parser, error, opts)
+        }
+
+      when 'msgpack'
+        Proc.new {|io,error|
+          parser = MessagePackParsingReader.new(io, error, opts)
+          if opts[:column_header]
+            column_names = parser.next
+          elsif opts[:column_names]
+            column_names = opts[:column_names]
+          end
+          if column_names
+            parser = HashBuilder.new(parser, error, column_names)
+          end
+          parser = TimeParserFilter.new(parser, error, opts)
+        }
+      end
+    end
+
+    def parse(io, error, &block)
+      factory = compose_factory
+      parser = factory.call(io, error)
+      begin
+        while record = parser.next
+          block.call(record)
+        end
+      rescue EOFError
+      end
+    end
+
+  end
+end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.38'
+VERSION = '0.10.39'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.38
+  version: 0.10.39
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-20 00:00:00.000000000 Z
+date: 2012-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -163,6 +163,7 @@ files:
 - lib/td/compat_gzip_reader.rb
 - lib/td/config.rb
 - lib/td/distribution.rb
+- lib/td/file_reader.rb
 - lib/td/version.rb
 - ChangeLog
 - README.rdoc