td 0.11.2 → 0.11.3

data/lib/td/command/table.rb

@@ -11,6 +11,7 @@ module Command
     COLLECTION ITEMS KEYS LINES STORED SEQUENCEFILE TEXTFILE INPUTFORMAT OUTPUTFORMAT LOCATION TABLESAMPLE BUCKET OUT
     OF CAST ADD REPLACE COLUMNS RLIKE REGEXP TEMPORARY FUNCTION EXPLAIN EXTENDED SERDE WITH SERDEPROPERTIES LIMIT SET TBLPROPERTIES
   ]
+  KEY_NUM_LIMIT = 512
 
   def table_create(op)
     type = nil
@@ -485,7 +486,10 @@ module Command
           client.create_database(db_name)
           $stderr.puts "Database '#{db_name}' is created."
         rescue AlreadyExistsError
+          # do nothing
         end
+      rescue ForbiddenError
+        # do nothing
       end
 
       API.validate_table_name(table_name)
@@ -510,7 +514,7 @@ module Command
         parser = MessagePackParser.new(time_key)
       end
 
-    else
+    else  # apache, syslog
       regexp, names, time_format = IMPORT_TEMPLATES[format]
       if !regexp || !names || !time_format
         $stderr.puts "Unknown format '#{format}'"
@@ -519,26 +523,40 @@ module Command
       parser = TextParser.new(names, regexp, time_format)
     end
 
-    get_table(client, db_name, table_name)
+    begin
+      db = client.database(db_name)
+    rescue ForbiddenError => e
+      puts "Warning: database and table validation skipped - #{e.message}"
+    else
+      begin
+        table = db.table(table_name)
+      rescue ForbiddenError => e
+        puts "Warning: table validation skipped - #{e.message}"
+      end
+    end
 
     require 'zlib'
 
-    files = paths.map {|path|
-      if path == '-'
-        $stdin
-      elsif path =~ /\.gz$/
-        require 'td/compat_gzip_reader'
-        Zlib::GzipReader.open(path)
-      else
-        File.open(path)
-      end
-    }
+    begin
+      files = paths.map {|path|
+        if path == '-'
+          $stdin
+        elsif path =~ /\.gz$/
+          require 'td/compat_gzip_reader'
+          Zlib::GzipReader.open(path)
+        else
+          File.open(path)
+        end
+      }
+    rescue Errno::ENOENT => e
+      raise ImportError, e.message
+    end
 
     require 'msgpack'
     require 'tempfile'
     #require 'thread'
 
-    files.zip(paths).each {|file,path|
+    files.zip(paths).each {|file, path|
       import_log_file(file, path, client, db_name, table_name, parser)
     }
 
@@ -568,10 +586,11 @@ module Command
 
       n += 1
       x += 1
-      if n % 10000 == 0
+      if n % 10000 == 0   # by records imported
         puts "  imported #{n} entries from #{path}..."
 
-      elsif out.pos > 1024*1024  # TODO size
+      # TODO size
+      elsif out.pos > 1024 * 1024   # by 1 MB chunks
         puts "  imported #{n} entries from #{path}..."
         begin
           writer.finish
@@ -592,6 +611,7 @@ module Command
       end
     }
 
+    # if there is anything parse but not imported yet
     if x != 0
       writer.finish
       size = out.pos
@@ -602,6 +622,11 @@ module Command
       client.import(db_name, table_name, "msgpack.gz", out, size)
     end
 
+    # throw an exception if no record is imported
+    if n == 0
+      raise ImportError, "no valid record to import from #{path}"
+    end
+
     puts "  imported #{n} entries from #{path}."
     $stderr.puts normalized_message if has_bignum
   ensure
@@ -661,75 +686,64 @@ module Command
     end
   end
 
-  class JsonParser
+  # Generic class for both JSON and MessagePack parsers to
+  # reduce code duplication
+  class StructuredParser
+    def sanitize_record(record, &block)
+      unless record.is_a?(Hash)
+        raise "record must be a Hash"
+      end
+
+      time = record[@time_key]
+      unless time
+        raise "record doesn't have '#{@time_key}' column"
+      end
+
+      if record.size > KEY_NUM_LIMIT
+        raise "record contains too many keys (#{record.size}, max allowed #{KEY_NUM_LIMIT})"
+      end
+
+      case time
+      when Integer
+        # do nothing
+      else
+        time = Time.parse(time.to_s).to_i
+      end
+      record['time'] = time
+
+      block.call(record)
+    end
+    protected :sanitize_record
+  end
+
+  class JsonParser < StructuredParser
     def initialize(time_key)
       require 'json'
       @time_key = time_key
     end
 
     def call(file, path, &block)
-      i = 0
       file.each_line {|line|
-        i += 1
         begin
           record = JSON.parse(line)
-
-          unless record.is_a?(Hash)
-            raise "record must be a Hash"
-          end
-
-          time = record[@time_key]
-          unless time
-            raise "record doesn't have '#{@time_key}' column"
-          end
-
-          case time
-          when Integer
-            # do nothing
-          else
-            time = Time.parse(time.to_s).to_i
-          end
-          record['time'] = time
-
-          block.call(record)
-
+          sanitize_record(record, &block)
         rescue
-          $stderr.puts "  skipped: #{$!}: #{line.dump}"
+          $stderr.puts "  skipped: #{$!}: #{record.to_json}"
         end
       }
     end
   end
 
-  class MessagePackParser
+  class MessagePackParser < StructuredParser
     def initialize(time_key)
       require 'msgpack'
       @time_key = time_key
     end
 
     def call(file, path, &block)
-      i = 0
       MessagePack::Unpacker.new(file).each {|record|
-        i += 1
         begin
-          unless record.is_a?(Hash)
-            raise "record must be a Hash"
-          end
-
-          time = record[@time_key]
-          unless time
-            raise "record doesn't have '#{@time_key}' column"
-          end
-
-          case time
-          when Integer
-            # do nothing
-          else
-            time = Time.parse(time.to_s).to_i
-          end
-          record['time'] = time
-
-          block.call(record)
-
+          sanitize_record(record, &block)
         rescue
           $stderr.puts "  skipped: #{$!}: #{record.to_json}"
         end

data/lib/td/updater.rb

@@ -103,7 +103,7 @@ module Updater
     when on_mac?
       'pkg'
     else
-      raise_error "Non supported environment"
+      raise_error "Environment not supported"
     end
   end
 
@@ -132,7 +132,8 @@ module Updater
     when Net::HTTPSuccess then response.body
     when Net::HTTPRedirection then fetch(response['Location'])
     else
-      raise "An error occurred when fetching from '#{url}'."
+      raise Command::UpdateError,
+            "An error occurred when fetching from '#{url}'."
       response.error!
     end
   end
@@ -141,6 +142,10 @@ module Updater
     ENV['TD_TOOLBELT_UPDATE_ROOT'] || "http://toolbelt.treasuredata.com"
   end
 
+  def maven_repo_root
+    ENV['TD_TOOLBELT_JARUPDATE_ROOT'] || "http://central.maven.org"
+  end
+
   def self.version_endpoint
     "#{endpoint_root}/version.#{package_category}"
   end
@@ -252,8 +257,8 @@ module Updater
   #
 
   # locate the root of the td package which is 3 folders up from the location of this file
-  def jarfile_dest_path
-    File.join(Updater.home_directory, ".td", "java")
+  def self.jarfile_dest_path
+    File.join(home_directory, ".td", "java")
   end
 
   private
@@ -280,10 +285,13 @@ module Updater
         return true
       elsif response.class == Net::HTTPFound || \
             response.class == Net::HTTPRedirection
-        puts "redirect '#{url}' to '#{response['Location']}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+        unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          puts "redirect '#{url}' to '#{response['Location']}'... "
+        end
         return stream_fetch(response['Location'], binfile, &progress)
       else
-        raise "An error occurred when fetching from '#{uri}' " +
+        raise Command::UpdateError,
+              "An error occurred when fetching from '#{uri}' " +
               "(#{response.class.to_s}: #{response.message})."
         return false
       end
@@ -296,23 +304,25 @@ module Updater
     require 'open-uri'
     require 'fileutils'
 
-    maven_repo = "http://maven.treasure-data.com/com/treasure_data/td-import"
+    maven_repo = "#{maven_repo_root}/maven2/com/treasuredata/td-import"
 
     begin
       xml = Updater.fetch("#{maven_repo}/maven-metadata.xml")
     rescue Exception => exc
       raise Command::UpdateError,
-            "There was a problem accessing the remote XML resource '#{maven_repo}/maven-metadata.xml' " +
-            "(#{exc.class.to_s}: #{exc.message})"
+            "There was a problem accessing the remote XML resource " +
+            "'#{maven_repo}/maven-metadata.xml' (#{exc.class.to_s}: #{exc.message})"
     end
     if xml.nil? || xml.empty?
       raise Command::UpdateError,
-            "The remote XML resource '#{maven_repo}/maven-metadata.xml' returned an empty file."
+            "The remote XML resource '#{maven_repo}/maven-metadata.xml' " +
+            "returned an empty file."
     end
 
     # read version and update date from the xml file
     doc = REXML::Document.new(xml)
-    updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
+    updated = Time.strptime(REXML::XPath.match(doc,
+      '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
     version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
 
     # Convert into UTF to compare time correctly
@@ -320,8 +330,8 @@ module Updater
     last_updated = existent_jar_updated_time
 
     if updated > last_updated
-      FileUtils.mkdir_p(jarfile_dest_path) unless File.exists?(jarfile_dest_path)
-      Dir.chdir jarfile_dest_path
+      FileUtils.mkdir_p(Updater.jarfile_dest_path) unless File.exists?(Updater.jarfile_dest_path)
+      Dir.chdir Updater.jarfile_dest_path
 
       File.open('VERSION', 'w') {|f|
         if hourly
@@ -334,17 +344,18 @@ module Updater
         f.print "#{version} #{updated}"
       }
 
+      status = nil
       indicator = Command::TimeBasedDownloadProgressIndicator.new(
         "Updating td-import.jar", Time.new.to_i, 2)
-      binfile = File.open 'td-import.jar.new', 'wb'
-      status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
-        indicator.update
+      File.open('td-import.jar.new', 'wb') {|binfile|
+        status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
+          indicator.update
+        }
       }
-      binfile.close
       indicator.finish()
 
       if status
-        puts "Installed td-import.jar v#{version} in '#{jarfile_dest_path}'.\n"
+        puts "Installed td-import.jar v#{version} in '#{Updater.jarfile_dest_path}'.\n"
         File.rename 'td-import.jar.new', 'td-import.jar'
       else
         puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
@@ -379,12 +390,12 @@ module Updater
 
   private
   def last_jar_autoupdate_timestamp
-    File.join(jarfile_dest_path, "td-import-java.version")
+    File.join(Updater.jarfile_dest_path, "td-import-java.version")
   end
 
   private
   def existent_jar_updated_time
-    files = find_files("td-import-java.version", [jarfile_dest_path])
+    files = Command.find_files("td-import-java.version", [Updater.jarfile_dest_path])
     if files.empty?
       return Time.at(0)
     end
@@ -399,47 +410,5 @@ module Updater
     time
   end
 
-  #
-  # Helpers
-  #
-  def find_files(glob, locations)
-    files = []
-    locations.each {|loc|
-      files = Dir.glob("#{loc}/#{glob}")
-      break unless files.empty?
-    }
-    files
-  end
-
-  def find_version_file
-    version = find_files('VERSION', [jarfile_dest_path])
-    if version.empty?
-      $stderr.puts "Cannot find VERSION file in '#{jarfile_dest_path}'."
-      exit 10
-    end
-    version.first
-  end
-
-  def find_td_import_jar
-    jar = find_files('td-import.jar', [jarfile_dest_path])
-    if jar.empty?
-      $stderr.puts "Cannot find td-import.jar in '#{jarfile_dest_path}'."
-      exit 10
-    end
-    jar.first
-  end
-
-  def find_logging_property
-    installed_path = File.join(File.expand_path('../..', File.dirname(__FILE__)), 'java')
-
-    config = find_files("logging.properties", [installed_path])
-    if config.empty?
-      puts "Cannot find 'logging.properties' file in '#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
-      []
-    else
-      config.first
-    end
-  end
-
 end # module Updater
 end # module TreasureData

data/lib/td/version.rb

@@ -1,3 +1,3 @@
 module TreasureData
-  TOOLBELT_VERSION = '0.11.2'
+  TOOLBELT_VERSION = '0.11.3'
 end

data/spec/td/common_spec.rb

@@ -0,0 +1,192 @@
+require 'spec_helper'
+require 'td/command/common'
+
+module TreasureData::Command
+  describe 'humanize_bytesize' do
+    describe 'for values < 1024' do
+      values = [0, 1, 10, 1023]
+      values.each {|v|
+        it "uses B as label and has no suffix (#{v})" do
+          TreasureData::Command::humanize_bytesize(v, 1).should == "#{v} B"
+        end
+      }
+    end
+
+    describe 'for 1024' do
+      it 'uses kB and does not have a suffix' do
+        TreasureData::Command::humanize_bytesize(1024).should == "1 kB"
+      end
+    end
+    describe 'for values between 1025 and (1024^2 - 1)' do
+      base = 1024
+      values = [
+        [base + 1, "1.0"],
+        [base + 2, "1.0"],
+        [base * 1024 - 1, "1023.9"]
+      ]
+      values.each {|val, exp|
+        it "uses kB as label and has a suffix (#{val})" do
+          result = TreasureData::Command::humanize_bytesize(val, 1)
+          expect(result).to eq("#{exp} kB")
+        end
+      }
+    end
+
+    describe 'for 1024^2' do
+      it 'uses MB and does not have a suffix' do
+        TreasureData::Command::humanize_bytesize(1024 ** 2).should == "1 MB"
+      end
+    end
+    describe 'for values between (1024^2 + 1) and (1024^3 - 1)' do
+      base = 1024 ** 2
+      values = [
+        [base + 1, "1.0"],
+        [base + 2, "1.0"],
+        [base * 1024 - 1, "1023.9"]
+      ]
+      values.each {|val, exp|
+        it "uses MB as label and has a suffix (#{val})" do
+          result = TreasureData::Command::humanize_bytesize(val, 1)
+          expect(result).to eq("#{exp} MB")
+        end
+      }
+    end
+
+    describe 'for 1024^3' do
+      it 'uses GB and does not have a suffix' do
+        TreasureData::Command::humanize_bytesize(1024 ** 3).should == "1 GB"
+      end
+    end
+    describe 'for values between (1024^3 + 1) and (1024^4 - 1)' do
+      base = 1024 ** 3
+      values = [
+        [base + 1, "1.0"],
+        [base + 2, "1.0"],
+        [base * 1024 - 1, "1023.9"]
+      ]
+      values.each {|val, exp|
+        it "uses GB as label and has a suffix (#{val})" do
+          result = TreasureData::Command::humanize_bytesize(val, 1)
+          expect(result).to eq("#{exp} GB")
+        end
+      }
+    end
+
+    describe 'for 1024^4' do
+      it 'uses TB and does not have a suffix' do
+        TreasureData::Command::humanize_bytesize(1024 ** 4).should == "1 TB"
+      end
+    end
+    describe 'for values between (1024^4 + 1) and (1024^5 - 1)' do
+      base = 1024 ** 4
+      values = [
+        [base + 1, "1.0"],
+        [base + 2, "1.0"],
+        [base * 1024 - 1, "1023.9"]
+      ]
+      values.each {|val, exp|
+        it "uses TB as label and has a suffix (#{val})" do
+          result = TreasureData::Command::humanize_bytesize(val, 1)
+          expect(result).to eq("#{exp} TB")
+        end
+      }
+    end
+
+    describe 'for 1024^5' do
+      it 'uses TB and does not have a suffix' do
+        TreasureData::Command::humanize_bytesize(1024 ** 5).should == "1024 TB"
+      end
+    end
+    describe 'for values between (1024^5 + 1) and (1024^6 - 1)' do
+      base = 1024 ** 5
+      values = [
+        [base + 1, "1024.0"],
+        [base + 2, "1024.0"],
+        [base * 1024 - 1, "1048575.9"]
+      ]
+      values.each {|val, exp|
+        it "uses TB as label and has a suffix (#{val})" do
+          result = TreasureData::Command::humanize_bytesize(val, 1)
+          expect(result).to eq("#{exp} TB")
+        end
+      }
+    end
+
+    describe 'shows 1 digit' do
+      it 'without second function argument' do
+        values = [1024 + 1024 / 2, "1.5"]
+        val, exp = values
+        result = TreasureData::Command::humanize_bytesize(val)
+        expect(result).to eq("#{exp} kB")
+      end
+    end
+    describe 'shows the correct number of digits specified by the second argument' do
+      (0...5).each {|i|
+        it "when = #{i}" do
+          val = 1024 + 1024 / 2
+          if i == 0
+            exp = 1.to_s
+          else
+            exp = sprintf "%.*f", i, 1.5
+          end
+          result = TreasureData::Command::humanize_bytesize(val, i)
+          expect(result).to eq("#{exp} kB")
+        end
+      }
+    end
+  end
+
+  describe 'SizeBasedDownloadProgressIndicator' do
+    it "shows in 1% increments with default 'perc_step'" do
+      size = 200
+      indicator = TreasureData::Command::SizeBasedDownloadProgressIndicator.new("Downloading", size)
+      size_increments = 2
+      curr_size = 0
+      while (curr_size += size_increments) < size do
+        indicator.update(size_increments)
+        sleep(0.05)
+      end
+      indicator.finish
+    end
+  end
+
+  describe 'TimeBasedDownloadProgressIndicator' do
+    it "increments about every 2 seconds with default 'periodicity'" do
+      start_time = Time.now.to_i
+      indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time)
+      end_time = start_time + 10
+      last_time = start_time
+      while (curr_time = Time.now.to_i) < end_time do
+        ret = indicator.update
+        if ret == true
+          diff = curr_time - last_time
+          diff.should be >= 2
+          diff.should be < 3
+          last_time = curr_time
+        end
+        sleep(0.5)
+      end
+      indicator.finish
+    end
+
+    periodicities = [1, 2, 5]
+    periodicities.each {|periodicity|
+      it "increments about every #{periodicity} seconds with 'periodicity' = #{periodicity}" do
+        start_time = Time.now.to_i
+        indicator = TreasureData::Command::TimeBasedDownloadProgressIndicator.new("Downloading", start_time, periodicity)
+        end_time = start_time + 10
+        last_time = start_time
+        while (curr_time = Time.now.to_i) < end_time do
+          ret = indicator.update
+          if ret == true
+            (curr_time - last_time).should be >= periodicity
+            (curr_time - last_time).should be < (periodicity + 1)
+            last_time = curr_time
+          end
+          sleep(0.5)
+        end
+        indicator.finish
+      end
+    }
+  end
+end

data/spec/td/updater_spec.rb

@@ -2,15 +2,21 @@ require 'spec_helper'
 require 'td/updater'
 
 module TreasureData::Updater
+
   describe 'without the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
+    let :default_toolbelt_url do
+      "http://toolbelt.treasuredata.com"
+    end
+
     describe 'endpoints methods' do
       it 'use the default root path' do
-        TreasureData::Updater.endpoint_root.should == TreasureData::Updater::DEFAULT_TOOLBELT_URL
-        TreasureData::Updater.version_endpoint.should =~ Regexp.new(TreasureData::Updater::DEFAULT_TOOLBELT_URL)
-        TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(TreasureData::Updater::DEFAULT_TOOLBELT_URL)
+        TreasureData::Updater.endpoint_root.should == default_toolbelt_url
+        TreasureData::Updater.version_endpoint.should =~ Regexp.new(default_toolbelt_url)
+        TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(default_toolbelt_url)
       end
     end
   end
+
   describe 'with the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
     before do
       ENV['TD_TOOLBELT_UPDATE_ROOT'] = 'https://0.0.0.0:5000/'

data/td.gemspec

@@ -15,12 +15,13 @@ Gem::Specification.new do |gem|
   gem.test_files            = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables           = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths         = ['lib']
+  gem.required_ruby_version = '>= 1.9'
 
   gem.add_dependency "msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"]
   gem.add_dependency "yajl-ruby", "~> 1.1"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.62"
+  gem.add_dependency "td-client", "~> 0.8.63"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"