td 0.10.75 → 0.10.76

data/ChangeLog

@@ -1,3 +1,11 @@
+== 2013-04-22 version 0.10.76
+
+* Add auto update feature for td-toolbelt
+* Add -T to specify query type
+* Include Ruby binary for Mac TD toolbelt
+* td-client v0.8.48
+
+
 == 2013-04-09 version 0.10.75
 
 * Fix unparsed regexp pattern in Ruby 1.8

data/README.rdoc

@@ -33,11 +33,11 @@ You need to authorize the account, before executing any other commands.
 
 == Mac OS X
 
-Disable RVM or rbenv and use system ruby (/usr/bin/ruby).
+Disable RVM or rbenv and use ruby.pkg's ruby (/usr/local/td/ruby/bin/ruby).
 And then run following commands:
 
-    $ sudo gem install bundler
-    $ rake pkg:build
+    $ /usr/local/td/ruby/bin/gem install bundler rubyzip
+    $ /usr/local/td/ruby/bin/rake pkg:build
 
 == Windows
 
@@ -52,7 +52,7 @@ Then run following commands on MinGW Shell:
 
     $ mingw-get install msys-vim
     $ mingw-get install msys-wget
-    $ gem install bundler
+    $ gem install bundler rubyzip
     $ bundle install  # don't use "--path" option
     $ rake exe:build  # don't use "bundle exec"
 

data/Rakefile

@@ -1,5 +1,6 @@
 require 'rubygems'
 require 'bundler'
+require 'zip/zip'
 Bundler::GemHelper.install_tasks
 
 task :default => :build
@@ -49,7 +50,8 @@ def install_use_gems(target_dir)
     end
   }
   FileUtils.mv Dir.glob("#{target_dir}/gems/*"), target_dir
-  %W(bin cache doc gems specifications).each { |dir|
+  FileUtils.rm_f Dir.glob("#{target_dir}/*.gem")
+  %W(bin cache doc gems specifications build_info).each { |dir|
     FileUtils.remove_dir("#{target_dir}/#{dir}", true)
   }
 end
@@ -119,6 +121,15 @@ def download_resource(url)
   path
 end
 
+def zip_files(zip_name, target_dir)
+  Zip::ZipFile.open(zip_name, Zip::ZipFile::CREATE) do |zip|
+    Dir["#{target_dir}/**/*"].each do |file|
+      zipped_path = file[target_dir.length + 1..-1]
+      zip.add(zipped_path, file) { true }
+    end
+  end
+end
+
 Dir[File.expand_path("../dist/**/*.rake", __FILE__)].each do |rake|
   import rake
 end

data/bin/td

@@ -5,6 +5,13 @@ gem 'td-client'
 here = File.dirname(__FILE__)
 $LOAD_PATH << File.expand_path(File.join(here, '..', 'lib'))
 
+require 'td/updater'
+TreasureData::Updater.disable(<<EOS
+`td update` is only available from Treasure Data Toolbelt.
+Download and install from http://toolbelt.treasure-data.com
+EOS
+)
+
 require 'td/command/runner'
 TreasureData::Command::Runner.new.run ARGV
 

data/dist/exe.rake

@@ -23,6 +23,8 @@ task 'exe:build' => :build do
       install_resource 'exe/td-cmd.bat', 'td-cmd.bat', 0755
     end
 
+    zip_files(project_root_path('pkg/td-update-exe.zip'), 'td')
+
     # create td.iss and run Inno Setup
     install_erb_resource 'exe/td.iss', 'td.iss', 0644, variables
 
@@ -38,6 +40,7 @@ end
 desc "clean Windows exe package"
 task "exe:clean" do
   FileUtils.rm_rf build_dir_path('exe')
-  FileUtils.rm_rf project_root_path("exe/td-#{version}.exe")
+  FileUtils.rm_rf project_root_path("pkg/td-#{version}.exe")
+  FileUtils.rm_rf project_root_path("pkg/td-update-exe.zip")
 end
 

data/dist/pkg.rake

@@ -16,6 +16,8 @@ task 'pkg:build' => :build do
       sh "pax -wz -x cpio . > ../bundle/td-client.pkg/Payload"
     end
 
+    zip_files(project_root_path('pkg/td-update-pkg.zip'), 'td-client.build')
+
     # crete ./bundle/td-client.pkg/Bom
     sh "mkbom -s td-client.build bundle/td-client.pkg/Bom"
 
@@ -34,6 +36,9 @@ task 'pkg:build' => :build do
     # create ./bundle/Distribution
     install_erb_resource('pkg/Distribution.erb', 'bundle/Distribution', 0644, variables)
 
+    sh "pkgutil --expand #{project_root_path('dist/resources/pkg/ruby-2.0.0-p0.pkg')} ruby"
+    mv "ruby/ruby-2.0.0-p0.pkg", "bundle/ruby.pkg"
+
     # create td-a.b.c.pkg
     sh "pkgutil --flatten bundle td-#{version}.pkg"
     FileUtils.cp "td-#{version}.pkg", project_root_path("pkg/td-#{version}.pkg")
@@ -44,5 +49,6 @@ desc "clean Mac OS X pkg package"
 task "pkg:clean" do
   FileUtils.rm_rf build_dir_path('pkg')
   FileUtils.rm_rf project_root_path("pkg/td-#{version}.pkg")
+  FileUtils.rm_rf project_root_path("pkg/td-update-pkg.zip")
 end
 

data/dist/resources/exe/td

@@ -21,6 +21,10 @@ Dir["#{gem_dir}/**/lib"].each do |libdir|
   $:.unshift libdir
 end
 
+# inject any code in ~/.td/updated/vendor/gems over top
+require 'td/updater'
+TreasureData::Updater.inject_libpath
+
 # start up the CLI
 require 'td/command/runner'
 TreasureData::Command::Runner.new.run ARGV

data/dist/resources/pkg/Distribution.erb

@@ -8,6 +8,8 @@
   </choices-outline>
   <choice id="td-client" title="TreasureData Client">
     <pkg-ref id="com.td.client"/>
+    <pkg-ref id="org.ruby-lang.installer"/>
   </choice>
   <pkg-ref id="com.td.client" installKBytes="<%= kbytes %>" version="<%= version %>" auth="Root">#td-client.pkg</pkg-ref>
+  <pkg-ref id="org.ruby-lang.installer" installKBytes="12868" auth="Root">#ruby.pkg</pkg-ref>
 </installer-script>

data/dist/resources/pkg/td

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/local/td/ruby/bin/ruby
 # -*- coding: utf-8 -*-
 
 # avoid conflicts with rvm
@@ -21,6 +21,10 @@ Dir["#{gem_dir}/**/lib"].each do |libdir|
   $:.unshift libdir
 end
 
+# inject any code in ~/.td/updated/vendor/gems over top
+require 'td/updater'
+TreasureData::Updater.inject_libpath
+
 # start up the CLI
 require "td/command/runner"
 TreasureData::Command::Runner.new.run ARGV

data/lib/td/command/list.rb

@@ -324,6 +324,8 @@ module List
   add_list 'help:all', %w[], 'Show usage of all commands'
   add_list 'help', %w[command], 'Show usage of a command'
 
+  add_list 'update', %w[], 'Update td and related libraries for TreasureData toolbelt'
+
   # aliases
   add_alias 'db', 'db:show'
   add_alias 'dbs', 'db:list'

data/lib/td/command/query.rb

@@ -60,7 +60,7 @@ module Command
     op.on('-q', '--query PATH', 'use file instead of inline query') {|s|
       query = File.open(s) { |f| f.read.strip }
     }
-    op.on('-t', '--type TYPE', 'set query type (hive or pig)') {|s|
+    op.on('-T', '--type TYPE', 'set query type (hive or pig)') {|s|
       type = s.to_sym
     }
     op.on('--sampling DENOMINATOR', 'enable random sampling to reduce records 1/DENOMINATOR', Integer) {|i|

data/lib/td/command/sched.rb

@@ -34,6 +34,7 @@ module Command
     result_ask_password = false
     priority = nil
     retry_limit = nil
+    type = nil
 
     op.on('-g', '--org ORGANIZATION', "create the schedule under this organization") {|s|
       org = s
@@ -65,6 +66,9 @@ module Command
     op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
       retry_limit = i
     }
+    op.on('-T', '--type TYPE', 'set query type (hive or pig)') {|s|
+      type = s
+    }
 
     name, cron, sql = op.cmd_parse
 
@@ -84,7 +88,7 @@ module Command
     get_database(client, db_name)
 
     begin
-      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority, :retry_limit=>retry_limit, :organization=>org)
+      first_time = client.create_schedule(name, :cron=>cron, :query=>sql, :database=>db_name, :result=>result_url, :timezone=>timezone, :delay=>delay, :priority=>priority, :retry_limit=>retry_limit, :organization=>org, :type=>type)
     rescue AlreadyExistsError
       cmd_debug_error $!
       $stderr.puts "Schedule '#{name}' already exists."
@@ -120,6 +124,7 @@ module Command
     delay = nil
     priority = nil
     retry_limit = nil
+    type = nil
 
     op.on('-s', '--schedule CRON', 'change the schedule') {|s|
       cron = s
@@ -148,6 +153,9 @@ module Command
     op.on('-R', '--retry COUNT', 'automatic retrying count', Integer) {|i|
       retry_limit = i
     }
+    op.on('-T', '--type TYPE', 'set query type (hive or pig)') {|s|
+      type = s
+    }
 
 
     name = op.cmd_parse
@@ -161,6 +169,7 @@ module Command
     params['delay'] = delay.to_s if delay
     params['priority'] = priority.to_s if priority
     params['retry_limit'] = retry_limit.to_s if retry_limit
+    params['type'] = type.to_s if type
 
     if params.empty?
       $stderr.puts op.to_s

data/lib/td/command/update.rb

@@ -0,0 +1,26 @@
+require "td/updater"
+
+module TreasureData
+module Command
+
+  def update(op)
+    if TreasureData::Updater.disable?
+      $stderr.puts TreasureData::Updater.disable_message
+      exit
+    end
+
+    $stderr.puts <<EOS
+Updating started at #{Time.now}
+from #{TreasureData::VERSION}
+EOS
+    if new_version = TreasureData::Updater.update
+      $stderr.puts "updated to #{new_version}"
+    else
+      $stderr.puts "nothing to update"
+    end
+    $stderr.puts "ended at #{Time.now}"
+  end
+
+end
+end
+

data/lib/td/helpers.rb

@@ -11,8 +11,16 @@ module TreasureData
       end
     end
 
+    def home_directory
+      on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
+    end
+
     def on_windows?
       RUBY_PLATFORM =~ /mswin32|mingw32/
     end
+
+    def on_mac?
+      RUBY_PLATFORM =~ /-darwin\d/
+    end
   end
 end

data/lib/td/updater.rb

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+require "fileutils"
+require "shellwords"
+require "zip/zip"
+
+module TreasureData
+  # This architecture is based on Heroku command
+  module Updater
+    def self.raise_error(message)
+      # TODO: Replace better Exception class
+      raise RuntimeError.new(message)
+    end
+
+    # copied from TreasureData::Helpers to avoid load issue.
+    def self.home_directory
+      on_windows? ? ENV['USERPROFILE'].gsub("\\","/") : ENV['HOME']
+    end
+
+    def self.on_windows?
+      RUBY_PLATFORM =~ /mswin32|mingw32/
+    end
+
+    def self.on_mac?
+      RUBY_PLATFORM =~ /-darwin\d/
+    end
+
+    def self.updating_lock_path
+      File.join(home_directory, ".td", "updating")
+    end
+
+    def self.installed_client_path
+      File.expand_path("../../..", __FILE__)
+    end
+
+    def self.updated_client_path
+      File.join(home_directory, ".td", "updated")
+    end
+
+    def self.latest_local_version
+      installed_version = client_version_from_path(installed_client_path)
+      updated_version = client_version_from_path(updated_client_path)
+      if compare_versions(updated_version, installed_version) > 0
+        updated_version
+      else
+        installed_version
+      end
+    end
+
+    def self.get_client_version_file(path)
+      td_gems = Dir[File.join(path, "vendor/gems/td-*")]
+      td_gems.each { |td_gem|
+        if td_gem =~ /#{"#{path}\/vendor\/gems\/td-\\d*.\\d*.\\d*"}/
+          return File.join(td_gem, "/lib/td/version.rb")
+        end
+      }
+
+      nil
+    end
+
+    def self.client_version_from_path(path)
+      if version_file = get_client_version_file(path)
+        File.read(version_file).match(/VERSION = '([^']+)'/)[1]
+      else
+        '0.0.0'
+      end
+    end
+
+    def self.disable(message)
+      @disable = message
+    end
+
+    def self.disable?
+      !@disable.nil?
+    end
+
+    def self.disable_message
+      @disable
+    end
+
+    def self.wait_for_lock(path, wait_for = 5, check_every = 0.5)
+      start = Time.now.to_i
+      while File.exists?(path)
+        sleep check_every
+        if (Time.now.to_i - start) > wait_for
+          raise_error "Unable to acquire update lock"
+        end
+      end
+      begin
+        FileUtils.touch(path)
+        ret = yield
+      ensure
+        FileUtils.rm_f(path)
+      end
+      ret
+    end
+
+    def self.package_category
+      case 
+      when on_windows?
+        'exe'
+      when on_mac?
+        'pkg'
+      else
+        raise_error "Non supported environment"
+      end
+    end
+
+    def self.fetch(uri)
+      require 'net/http'
+      require 'openssl'
+
+      # open-uri can't treat 'http -> https' redirection and
+      # Net::HTTP.get_response can't get response from HTTPS endpoint.
+      # So we use following code to avoid above issues.
+      u = URI(uri)
+      response = if u.scheme == 'https'
+                   http = Net::HTTP.new(u.host, u.port)        
+                   http.use_ssl = true
+                   http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+                   http.request(Net::HTTP::Get.new(u.path))
+                 else
+                   Net::HTTP.get_response(u)
+                 end
+
+      case response
+      when Net::HTTPSuccess then response.body
+      when Net::HTTPRedirection then fetch(response['Location'])
+      else
+        response.error!
+      end
+    end
+
+    def self.version_endpoint
+      "http://toolbelt.treasure-data.com/version.#{package_category}"
+    end
+
+    def self.update_package_endpoint
+      "http://toolbelt.treasure-data.com/td-update-#{package_category}.zip"
+    end
+
+    def self.update(autoupdate = false)
+      wait_for_lock(updating_lock_path, 5) do
+        require "td"
+        require 'open-uri'
+        require "tmpdir"
+        require "zip/zip"
+
+        latest_version = fetch(version_endpoint)
+
+        if compare_versions(latest_version, latest_local_version) > 0
+          Dir.mktmpdir do |download_dir|
+            File.open("#{download_dir}/td-update.zip", "wb") do |file|
+              file.print fetch(update_package_endpoint)
+            end
+
+            Zip::ZipFile.open("#{download_dir}/td-update.zip") do |zip|
+              zip.each do |entry|
+                target = File.join(download_dir, entry.to_s)
+                FileUtils.mkdir_p(File.dirname(target))
+                zip.extract(entry, target) { true }
+              end
+            end
+
+            FileUtils.rm "#{download_dir}/td-update.zip"
+
+            old_version = latest_local_version
+            new_version = client_version_from_path(download_dir)
+
+            if compare_versions(new_version, old_version) < 0 && !autoupdate
+              raise_error "Installed version (#{old_version}) is newer than the latest available update (#{new_version})"
+            end
+
+            FileUtils.rm_rf updated_client_path
+            FileUtils.mkdir_p File.dirname(updated_client_path)
+            FileUtils.cp_r(download_dir, updated_client_path)
+
+            new_version
+          end
+        else
+          false # already up to date
+        end
+      end
+    ensure
+      FileUtils.rm_f(updating_lock_path)
+    end
+
+    def self.compare_versions(first_version, second_version)
+      first_version.split('.').map { |part| Integer(part) rescue part } <=> second_version.split('.').map { |part| Integer(part) rescue part }
+    end
+
+    def self.inject_libpath
+      old_version = client_version_from_path(installed_client_path)
+      new_version = client_version_from_path(updated_client_path)
+
+      if compare_versions(new_version, old_version) > 0
+        vendored_gems = Dir[File.join(updated_client_path, "vendor", "gems", "*")]
+        vendored_gems.each do |vendored_gem|
+          $:.unshift File.join(vendored_gem, "lib")
+        end
+        load('td/updater.rb') # reload updated updater
+      end
+
+      background_update!
+    end
+
+    def self.last_autoupdate_path
+      File.join(home_directory, ".td", "autoupdate.last")
+    end
+
+    def self.background_update!
+      if File.exists?(last_autoupdate_path)
+        return if (Time.now.to_i - File.mtime(last_autoupdate_path).to_i) < 60 * 60 * 1 # every 1 hours
+      end
+      log_path = File.join(home_directory, '.td', 'autoupdate.log')
+      FileUtils.mkdir_p File.dirname(log_path)
+      td_binary = File.expand_path($0)
+      pid = if defined?(RUBY_VERSION) and RUBY_VERSION =~ /^1\.8\.\d+/
+        fork do
+          exec("#{Shellwords.escape(td_binary)} update &> #{Shellwords.escape(log_path)} 2>&1")
+        end
+      else
+        log_file = File.open(log_path, "w")
+        spawn(td_binary, 'update', :err => log_file, :out => log_file)
+      end
+      Process.detach(pid)
+      FileUtils.mkdir_p File.dirname(last_autoupdate_path)
+      FileUtils.touch last_autoupdate_path
+    end
+  end
+end

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.75'
+VERSION = '0.10.76'
 
 end

data/td.gemspec

@@ -20,8 +20,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1.0"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.5.19"
-  gem.add_dependency "td-client", "~> 0.8.46"
+  gem.add_dependency "td-client", "~> 0.8.48"
   gem.add_dependency "td-logger", "~> 0.3.16"
+  gem.add_dependency "rubyzip", "~> 0.9.9"
   gem.add_development_dependency "rake", "~> 0.9"
   gem.add_development_dependency "rspec", "~> 2.11.0"
   gem.add_development_dependency "simplecov", "~> 0.5.4"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.75
+  version: 0.10.76
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-09 00:00:00.000000000 Z
+date: 2013-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.46
+        version: 0.8.48
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.46
+        version: 0.8.48
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +107,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.3.16
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.9.9
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -182,6 +198,7 @@ files:
 - dist/resources/pkg/Distribution.erb
 - dist/resources/pkg/PackageInfo.erb
 - dist/resources/pkg/postinstall
+- dist/resources/pkg/ruby-2.0.0-p0.pkg
 - dist/resources/pkg/td
 - java/td-bulk-import-0.1.2-SNAPSHOT.jar
 - java/td-bulk-import-java.version
@@ -211,6 +228,7 @@ files:
 - lib/td/command/server.rb
 - lib/td/command/status.rb
 - lib/td/command/table.rb
+- lib/td/command/update.rb
 - lib/td/command/user.rb
 - lib/td/compat_core.rb
 - lib/td/compat_gzip_reader.rb
@@ -218,6 +236,7 @@ files:
 - lib/td/distribution.rb
 - lib/td/file_reader.rb
 - lib/td/helpers.rb
+- lib/td/updater.rb
 - lib/td/version.rb
 - spec/file_reader/filter_spec.rb
 - spec/file_reader/io_filter_spec.rb
@@ -243,7 +262,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1038070565617120851
+      hash: -1565388258148285832
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -252,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1038070565617120851
+      hash: -1565388258148285832
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23