td 0.10.99 → 0.11.1

data/lib/td/updater.rb

@@ -4,8 +4,11 @@ require "shellwords"
 require "zip/zip"
 
 module TreasureData
-  # This architecture is based on Heroku command
   module Updater
+    #
+    # Toolbelt upgrade
+    #
+
     def self.raise_error(message)
       # TODO: Replace better Exception class
       raise RuntimeError.new(message)
@@ -53,13 +56,12 @@ module TreasureData
           return File.join(td_gem, "/lib/td/version.rb")
         end
       }
-
       nil
     end
 
     def self.client_version_from_path(path)
       if version_file = get_client_version_file(path)
-        File.read(version_file).match(/VERSION = '([^']+)'/)[1]
+        File.read(version_file).match(/TOOLBELT_VERSION = '([^']+)'/)[1]
       else
         '0.0.0'
       end
@@ -95,7 +97,7 @@ module TreasureData
     end
 
     def self.package_category
-      case 
+      case
       when on_windows?
         'exe'
       when on_mac?
@@ -113,29 +115,35 @@ module TreasureData
       # Net::HTTP.get_response can't get response from HTTPS endpoint.
       # So we use following code to avoid above issues.
       u = URI(uri)
-      response = if u.scheme == 'https'
-                   http = Net::HTTP.new(u.host, u.port)        
-                   http.use_ssl = true
-                   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-                   http.request(Net::HTTP::Get.new(u.path))
-                 else
-                   Net::HTTP.get_response(u)
-                 end
+      response =
+        if u.scheme == 'https'
+          http = Net::HTTP.new(u.host, u.port)
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.request(Net::HTTP::Get.new(u.path))
+        else
+          Net::HTTP.get_response(u)
+        end
 
       case response
       when Net::HTTPSuccess then response.body
       when Net::HTTPRedirection then fetch(response['Location'])
       else
+        raise "An error occurred when fetching from '#{uri}'."
         response.error!
       end
     end
 
+    def self.endpoint_root
+      ENV['TD_TOOLBELT_UPDATE_ROOT'] || "http://toolbelt.treasuredata.com"
+    end
+
     def self.version_endpoint
-      "http://toolbelt.treasure-data.com/version.#{package_category}"
+      "#{endpoint_root}/version.#{package_category}"
     end
 
     def self.update_package_endpoint
-      "http://toolbelt.treasure-data.com/td-update-#{package_category}.zip"
+      "#{endpoint_root}/td-update-#{package_category}.zip"
     end
 
     def self.update(autoupdate = false)
@@ -149,10 +157,33 @@ module TreasureData
 
         if compare_versions(latest_version, latest_local_version) > 0
           Dir.mktmpdir do |download_dir|
+
+            # initialize the progress indicator
+            base_msg = "Downloading updated toolbelt package"
+            print base_msg + " " * 10
+            start_time = last_time = Time.new.to_i
+
+            # downloading the update compressed file
             File.open("#{download_dir}/td-update.zip", "wb") do |file|
-              file.print fetch(update_package_endpoint)
+              endpoint = update_package_endpoint
+              puts "\npackage '#{endpoint}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+              stream_fetch(endpoint, file) {
+
+                # progress indicator
+                if (time = Time.now.to_i) - last_time > 2
+                  msg = "\r" + base_msg + ": #{time - start_time}s elapsed"
+                  # TODO parse 'time - start_time' with humanize_time from common.rb
+                  #      once this method is not static
+                  print msg + " " * 10
+                  last_time = time
+                end
+              }
             end
 
+            # clear progress indicator with this final message
+            puts "\r" + base_msg + "...done" + " " * 20
+
+            print "Unpacking updated toolbelt package..."
             Zip::ZipFile.open("#{download_dir}/td-update.zip") do |zip|
               zip.each do |entry|
                 target = File.join(download_dir, entry.to_s)
@@ -160,6 +191,7 @@ module TreasureData
                 zip.extract(entry, target) { true }
               end
             end
+            print "done\n"
 
             FileUtils.rm "#{download_dir}/td-update.zip"
 
@@ -200,16 +232,10 @@ module TreasureData
         load('td/updater.rb') # reload updated updater
       end
 
-      background_update!
-    end
-
-    def self.last_autoupdate_path
-      File.join(home_directory, ".td", "autoupdate.last")
-    end
-
-    def self.background_update!
-      if File.exists?(last_autoupdate_path)
-        return if (Time.now.to_i - File.mtime(last_autoupdate_path).to_i) < 60 * 60 * 1 # every 1 hours
+      # check every hour if the toolbelt can be updated.
+      # => If so, update in the background
+      if File.exists?(last_toolbelt_autoupdate_timestamp)
+        return if (Time.now.to_i - File.mtime(last_toolbelt_autoupdate_timestamp).to_i) < 60 * 60 * 1 # every 1 hours
       end
       log_path = File.join(home_directory, '.td', 'autoupdate.log')
       FileUtils.mkdir_p File.dirname(log_path)
@@ -223,8 +249,184 @@ module TreasureData
         spawn(td_binary, 'update', :err => log_file, :out => log_file)
       end
       Process.detach(pid)
-      FileUtils.mkdir_p File.dirname(last_autoupdate_path)
-      FileUtils.touch last_autoupdate_path
+      FileUtils.mkdir_p File.dirname(last_toolbelt_autoupdate_timestamp)
+      FileUtils.touch last_toolbelt_autoupdate_timestamp
+    end
+
+    def self.last_toolbelt_autoupdate_timestamp
+      File.join(home_directory, ".td", "autoupdate.last")
+    end
+
+    #
+    # td-import.jar upgrade
+    #
+
+    # locate the root of the td package which is 3 folders up from the location of this file
+    def jarfile_dest_path
+      File.join(Updater.home_directory, ".td", "java")
+    end
+
+    private
+    def self.stream_fetch(url, binfile, &progress)
+      require 'net/http'
+
+      uri = URI(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.scheme == 'https'
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      http.start
+      request = Net::HTTP::Get.new(uri.to_s)
+      http.request request do |response|
+        if response.class == Net::HTTPOK
+          # print a . every tick_period seconds
+          response.read_body do |chunk|
+            binfile.write chunk
+            progress.call #unless progress.nil?
+          end
+          return true
+        elsif response.class == Net::HTTPFound || \
+              response.class == Net::HTTPRedirection
+          puts "redirect '#{url}' to '#{response['Location']}'... " unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          return stream_fetch(response['Location'], binfile, &progress)
+        else
+          raise "An error occurred when fetching from '#{uri}'."
+          return false
+        end
+      end
     end
-  end
-end
+
+    private
+    def jar_update(hourly = false)
+      maven_repo = "http://maven.treasure-data.com/com/treasure_data/td-import"
+
+      require 'rexml/document'
+      require 'open-uri'
+      require 'fileutils'
+
+      doc = REXML::Document.new(open("#{maven_repo}/maven-metadata.xml") { |f| f.read })
+      updated = Time.strptime(REXML::XPath.match(doc, '/metadata/versioning/lastUpdated').first.text, "%Y%m%d%H%M%S")
+      version = REXML::XPath.match(doc, '/metadata/versioning/release').first.text
+
+      # Convert into UTF to compare time correctly
+      updated = (updated + updated.gmt_offset).utc unless updated.gmt?
+      last_updated = existent_jar_updated_time
+
+      if updated > last_updated
+        FileUtils.mkdir_p(jarfile_dest_path) unless File.exists?(jarfile_dest_path)
+        Dir.chdir jarfile_dest_path
+
+        File.open('VERSION', 'w') { |f| f.print "#{version} via import:jar_update" }
+        File.open('td-import-java.version', 'w') { |f| f.print "#{version} #{updated}" }
+
+        # initialize the progress indicator
+        base_msg = "Updating td-import.jar"
+        start_time = last_time = Time.new.to_i
+        print base_msg + " " * 10
+
+        binfile = File.open 'td-import.jar.new', 'wb'
+        status = Updater.stream_fetch("#{maven_repo}/#{version}/td-import-#{version}-jar-with-dependencies.jar", binfile) {
+
+          # progress indicator
+          if (time = Time.now.to_i) - last_time > 2
+            msg = "\r" + base_msg + ": #{humanize_time(time - start_time)} elapsed"
+            # TODO parse 'time - start_time' with humanize_time from common.rb
+            #      once this method is not static
+            print msg + " " * 10
+            last_time = time
+          end
+        }
+        binfile.close
+
+        # clear progress indicator with this final message
+        puts "\r" + base_msg + "...done" + " " * 20
+
+        if status
+          puts "Installed td-import.jar v#{version} in '#{jarfile_dest_path}'.\n"
+          File.rename 'td-import.jar.new', 'td-import.jar'
+        else
+          puts "Update of td-import.jar failed." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+          File.delete 'td-import.jar.new' if File.exists? 'td-import.jar.new'
+        end
+      else
+        puts 'Installed td-import.jar is already at the latest version.' unless hourly
+      end
+    end
+
+    def check_n_update_jar(hourly = false)
+      if hourly && \
+         File.exists?(last_jar_autoupdate_timestamp) && \
+         (Time.now - File.mtime(last_jar_autoupdate_timestamp)).to_i < (60 * 60 * 1) # every hour
+        return
+      end
+      jar_update(hourly)
+      FileUtils.touch File.join(jarfile_dest_path, "td-import-java.version")
+    end
+
+    private
+    def last_jar_autoupdate_timestamp
+      File.join(jarfile_dest_path, "td-import-java.version")
+    end
+
+    private
+    def existent_jar_updated_time
+      files = find_files("td-import-java.version", [jarfile_dest_path])
+      if files.empty?
+        return Time.at(0)
+      end
+      content = File.open(files.first).read
+      index = content.index(' ')
+      time = nil
+      if index.nil?
+        time = Time.at(0).utc
+      else
+        time = Time.parse(content[index+1..-1].strip).utc
+      end
+      time
+    end
+
+    #
+    # Helpers
+    #
+    def find_files(glob, locations)
+      files = []
+      locations.each {|loc|
+        files = Dir.glob("#{loc}/#{glob}")
+        break unless files.empty?
+      }
+      files
+    end
+
+    def find_version_file
+      version = find_files('VERSION', [jarfile_dest_path])
+      if version.empty?
+        $stderr.puts "Cannot find VERSION file in '#{jarfile_dest_path}'."
+        exit 10
+      end
+      version.first
+    end
+
+    def find_td_import_jar
+      jar = find_files('td-import.jar', [jarfile_dest_path])
+      if jar.empty?
+        $stderr.puts "Cannot find td-import.jar in '#{jarfile_dest_path}'."
+        exit 10
+      end
+      jar.first
+    end
+
+    def find_logging_property
+      installed_path = File.join(File.expand_path('../..', File.dirname(__FILE__)), 'java')
+
+      config = find_files("logging.properties", [installed_path])
+      if config.empty?
+        puts "Cannot find 'logging.properties' file in '#{installed_path}'." unless ENV['TD_TOOLBELT_DEBUG'].nil?
+        []
+      else
+        config.first
+      end
+    end
+
+  end # end of module Updater
+end # end of module TreasureData

data/lib/td/version.rb

@@ -1,5 +1,3 @@
 module TreasureData
-
-VERSION = '0.10.99'
-
+  TOOLBELT_VERSION = '0.11.1'
 end

data/spec/td/updater_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'td/updater'
+
+module TreasureData::Updater
+  describe 'without the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
+    describe 'endpoints methods' do
+      it 'use the default root path' do
+        TreasureData::Updater.endpoint_root.should == TreasureData::Updater::DEFAULT_TOOLBELT_URL
+        TreasureData::Updater.version_endpoint.should =~ Regexp.new(TreasureData::Updater::DEFAULT_TOOLBELT_URL)
+        TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(TreasureData::Updater::DEFAULT_TOOLBELT_URL)
+      end
+    end
+  end
+  describe 'with the TD_TOOLBELT_UPDATE_ROOT environment variable defined' do
+    before do
+      ENV['TD_TOOLBELT_UPDATE_ROOT'] = 'https://0.0.0.0:5000/'
+    end
+    describe 'endpoints methods' do
+      it 'use the custom root path' do
+        TreasureData::Updater.endpoint_root.should == ENV['TD_TOOLBELT_UPDATE_ROOT']
+        TreasureData::Updater.version_endpoint.should =~ Regexp.new(ENV['TD_TOOLBELT_UPDATE_ROOT'])
+        TreasureData::Updater.update_package_endpoint.should =~ Regexp.new(ENV['TD_TOOLBELT_UPDATE_ROOT'])
+      end
+    end
+    after do
+      ENV.delete 'TD_TOOLBELT_UPDATE_ROOT'
+    end
+  end
+end

data/spec/td/version_spec.rb

@@ -6,7 +6,7 @@ module TreasureData::Command
       stderr, stdout = execute_td("--version")
       stderr.should == ""
       stdout.should == <<-STDOUT
-td #{TreasureData::VERSION}
+#{TreasureData::TOOLBELT_VERSION}
 STDOUT
     end
   end

data/td.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |gem|
   gem.description = "CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing"
   gem.homepage    = "http://treasure-data.com/"
   gem.summary     = "CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing"
-  gem.version     = TreasureData::VERSION
+  gem.version     = TreasureData::TOOLBELT_VERSION
   gem.authors     = ["Treasure Data, Inc."]
   gem.email       = "support@treasure-data.com"
   gem.has_rdoc    = false
@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1.0"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.6.1"
-  gem.add_dependency "td-client", "~> 0.8.57"
+  gem.add_dependency "td-client", "~> 0.8.60"
   gem.add_dependency "td-logger", "~> 0.3.21"
   gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.99
-  prerelease: 
+  version: 0.11.1
 platform: ruby
 authors:
 - Treasure Data, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-01 00:00:00.000000000 Z
+date: 2014-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -37,7 +35,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -60,7 +57,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -68,7 +64,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -76,7 +71,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: hirb
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -84,7 +78,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -92,7 +85,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -100,7 +92,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -108,23 +99,20 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: td-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.57
+        version: 0.8.60
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.57
+        version: 0.8.60
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -132,7 +120,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -140,7 +127,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -148,7 +134,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -156,7 +141,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -164,7 +148,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -172,7 +155,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -180,7 +162,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -188,7 +169,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -196,7 +176,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -214,9 +193,8 @@ files:
 - README.rdoc
 - Rakefile
 - bin/td
-- build/update-td-import-java.sh
 - contrib/completion/_td
-- contrib/completion/td-comletion.bash
+- contrib/completion/td-completion.bash
 - data/sample_apache.json
 - data/sample_apache_gen.rb
 - dist/exe.rake
@@ -230,10 +208,7 @@ files:
 - dist/resources/pkg/postinstall
 - dist/resources/pkg/ruby-2.0.0-p0.pkg
 - dist/resources/pkg/td
-- java/VERSION
 - java/logging.properties
-- java/td-import-java.version
-- java/td-import.jar
 - lib/td.rb
 - lib/td/command/account.rb
 - lib/td/command/acl.rb
@@ -274,37 +249,31 @@ files:
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
 - spec/td/helpers_spec.rb
+- spec/td/updater_spec.rb
 - spec/td/version_spec.rb
 - td.gemspec
 homepage: http://treasure-data.com/
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 687370063067596647
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 687370063067596647
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: CLI to manage data on Treasure Data, the Hadoop-based cloud data warehousing
 test_files:
 - spec/file_reader/filter_spec.rb
@@ -315,4 +284,5 @@ test_files:
 - spec/file_reader_spec.rb
 - spec/spec_helper.rb
 - spec/td/helpers_spec.rb
+- spec/td/updater_spec.rb
 - spec/td/version_spec.rb