td 0.10.73 → 0.10.74

data/ChangeLog

@@ -1,3 +1,15 @@
+== 2013-03-29 version 0.10.74
+
+* Include zsh and bash completion
+* Use SSL by default
+* Add --insecure option to disable SSL
+* Add IP limitation features:
+  * ip_limit:set <org> <ip_range...>
+  * ip_limit:delete <org>
+  * ip_limit:list
+  * ip_limit:show <org>
+* td-client v0.8.46
+
 == 2013-03-15 version 0.10.73
 
 * Add --sampling option to query subcommand

data/contrib/completion/_td

@@ -0,0 +1,274 @@
+#compdef td
+
+# (( $+functions[_td] )) ||
+function _td()
+{
+typeset -A opt_args
+local context state line args
+
+local -a subcommands
+subcommands=(
+    "account":"Setup a Treasure Data account"
+    "account\:usage":"Show resource usage information"
+
+    "apikey\:show":"Show Treasure Data API key"
+    "apikey\:set":"Set Treasure Data API key"
+
+    "bulk_import\:list":"List bulk import sessions"
+    "bulk_import\:show":"Show list of uploaded parts"
+    "bulk_import\:create":"Create a new bulk import session to the the table"
+    "bulk_import\:prepare_parts":"Convert files into part file format"
+    #"bulk_import\:prepare_parts2":"Convert files into part file format"
+    "bulk_import\:upload_part":"Upload or re-upload a file into a bulk import session"
+    "bulk_import\:upload_parts":"Upload or re-upload files into a bulk import session"
+    #"bulk_import\:upload_parts2":"Upload or re-upload files into a bulk import session"
+    "bulk_import\:delete_part":"Delete a uploaded file from a bulk import session"
+    "bulk_import\:delete_parts":"Delete uploaded files from a bulk import session"
+    "bulk_import\:perform":"Start to validate and convert uploaded files"
+    "bulk_import\:error_records":"Show records which did not pass validations"
+    "bulk_import\:commit":"Start to commit a performed bulk import session"
+    "bulk_import\:delete":"Delete a bulk import session"
+    "bulk_import\:freeze":"Reject succeeding uploadings to a bulk import session"
+    "bulk_import\:unfreeze":"Unfreeze a frozen bulk import session"
+
+    "db\:list":"Show list of tables in a database"
+    "db\:show":"Describe information of a database"
+    "db\:create":"Create a database"
+    "db\:delete":"Delete a database"
+
+    "help\:all":"Show usage of all commands"
+    "help":"Show usage of a command"
+
+    "job\:show":"Show status and result of a job"
+    "job\:status":"Show status progress of a job"
+    "job\:list":"Show list of jobs"
+    "job\:kill":"Kill or cancel a job"
+
+    "password\:change":"Change password"
+
+    "query":"issue a query"
+
+    "result\:list":"Show list of result URLs"
+    "result\:show":"Describe information of a result URL"
+    "result\:create":"Create a result URL"
+    "result\:delete":"Delete a result URL"
+
+    "sample\:apache":"Create a sample log file"
+
+    "sched\:list":"Show list of schedules"
+    "sched\:create":"Create a schedule"
+    "sched\:delete":"Delete a schedule"
+    "sched\:update":"Modify a schedule"
+    "sched\:history":"Show history of scheduled queries"
+    "sched\:run":"Run scheduled queries for the specified time"
+
+    "server\:status":"Show status of the Treasure Data server"
+
+    "schema\:show":"Show schema of a table"
+    "schema\:set":"Set new schema on a table"
+    "schema\:add":"Add new columns to a table"
+    "schema\:remove":"Remove columns from a table"
+
+    "status":"Show scheds, jobs, tables and results"
+
+    "table\:list":"Show list of tables"
+    "table\:show":"Describe information of a table"
+    "table\:create":"Create a table"
+    "table\:delete":"Delete a table"
+    "table\:import":"Parse and import files to a table"
+    "table\:export":"Dump logs in a table to the specified storage"
+    "table\:swap":"Swap names of two tables"
+    "table\:tail":"Get recently imported logs"
+    "table\:partial_delete":"Delete logs from the table within the specified time range"
+
+    # TODO: Add ACL related commands
+)
+
+_arguments -C -S \
+    '(--config -c)'{--config,-c}'[path to config file ('~/.td/td.conf')]:PATH:' \
+    '(--apikey -k)'{--apikey,-k}'[use this API key instead of reading the config file]:APIKEY:' \
+    '(--verbose -v)'{--verbose,-v}'[verbose mode]' \
+    '(--help -h)'{--help,-h}'[show help]' \
+    '--version[show td client version]' \
+    '*:: :->subcommand' && ret=0
+
+if (( CURRENT == 1 )); then
+    _describe -t subcommands "td subcommands" subcommands
+    return
+fi
+
+local -a _subcommand_args
+case "$words[1]" in
+    account)
+        _subcommand_args=(
+            '(-f|--force)'{-f,--force}'[overwrite current account setting]' \
+                )
+        ;;
+    apikey:set)
+        _subcommand_args=(
+            '(-f|--force)'{-f,--force}'[overwrite current account setting]' \
+                )
+        ;;
+    bulk_import:create)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[create the bukl import session under this organization]' \
+                )
+        ;;
+    bulk_import:upload_parts)
+        _subcommand_args=(
+            '(-P|--prefix)'{-P,--prefix}'[add prefix to parts name]' \
+            '(-s|--use-suffix)'{-s,--use-suffix}'[use COUNT number of . (dots) in the source file name to the parts name]' \
+            '--auto-perform[perform bulk import job automatically]' \
+            '--parallel[perform uploading in parallel (default: 2; max 8)]' \
+                )
+        ;;
+    bulk_import:perform)
+        _subcommand_args=(
+            '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \
+            '(-f|--force)'{-f,--force}'[force start performing]' \
+                )
+        ;;
+    bulk_import:prepare_parts)
+        _subcommand_args=(
+            '(-s|--split-size)'{-s,--split-size}'[size of each parts]' \
+            '(-o|--output)'{-o,--output}'[output directory]' \
+                )
+        ;;
+    db:create)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[create the database under this organization]' \
+                )
+        ;;
+    db:delete)
+        _subcommand_args=(
+            '(-f|--force)'{-f,--force}'[clear tables and delete the database]' \
+                )
+        ;;
+    job:list)
+        _subcommand_args=(
+            '(-p|--page)'{-p,--page}'[skip N pages]' \
+            '(-s|--skip)'{-s,--skip}'[skip N jobs]' \
+            '(-R|--running)'{-R,--running}'[show only running jobs]' \
+            '(-S|--success)'{-S,--success}'[show only succeeded jobs]' \
+            '(-E|--error)'{-E,--error}'[show only failed jobs]' \
+            '--show[show slow queries (default threshold: 3600 seconds)]' \
+                )
+        ;;
+    job:show)
+        _subcommand_args=(
+            '(-v|--verbose)'{-v,--verbose}'[show logs]' \
+            '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \
+            '(-G|--vertical)'{-G,--vertical}'[use vertical table to show results]' \
+            '(-o|--output)'{-o,--output}'[write result to the file]' \
+            '(-f|--format)'{-f,--format}'[format of the result to write to the file (tsv, csv, json or msgpack)]'
+                )
+        ;;
+    query)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[issue the query under this organization]' \
+            '(-d|--database)'{-d,--database}'[use the database (required)]' \
+            '(-w|--wait)'{-w,--wait}'[wait for finishing the job]' \
+            '(-G|--vertical)'{-G,--vertical}'[use vertical table to show results]' \
+            '(-o|--output)'{-o,--output}'[write result to the file]' \
+            '(-f|--format)'{-f,--format}'[format of the result to write to the file (tsv, csv, json or msgpack)]'
+            '(-r|--result)'{-r,--result}'[write result to the URL (see also result:create subcommand)]' \
+            '(-u|--user)'{-u,--user}'[set user name for the result URL]' \
+            '(-p|--password)'{-p,--password}'[ask password for the result URL]' \
+            '(-P|--priority)'{-P,--priority}'[set priority]' \
+            '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \
+            '(-q|--query)'{-q,--query}'[use file instead of inline query]' \
+            '--sampling[enable random sampling to reduce records 1/DENOMINATOR]' \
+                )
+        ;;
+    result:create)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[create the result under this organization]' \
+            '(-u|--user)'{-u,--user}'[set user name for authentication]' \
+            '(-p|--password)'{-p,--password}'[ask password for authentication]' \
+                )
+        ;;
+    sched:create)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[create the schedule under this organization]' \
+            '(-d|--database)'{-d,--database}'[use the database (required)]' \
+            '(-t|--timezone)'{-t,--timezone}'[name of the timezone (like Asia/Tokyo)]' \
+            '(-D|--delay)'{-D,--delay}'[delay time of the schedule]' \
+            '(-o|--output)'{-o,--output}'[write result to the file]' \
+            '(-r|--result)'{-r,--result}'[write result to the URL (see also result:create subcommand)]' \
+            '(-u|--user)'{-u,--user}'[set user name for the result URL]' \
+            '(-p|--password)'{-p,--password}'[ask password for the result URL]' \
+            '(-P|--priority)'{-P,--priority}'[set priority]' \
+            '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \
+                )
+        ;;
+    sched:update)
+        _subcommand_args=(
+            '(-s|--schedule)'{-s,--schedule}'[change the schedule]' \
+            '(-q|--query)'{-q,--query}'[change the query]' \
+            '(-d|--database)'{-d,--database}'[change the database]' \
+            '(-r|--result)'{-r,--result}'[change the result table]' \
+            '(-t|--timezone)'{-t,--timezone}'[change the name of the timezone]' \
+            '(-D|--delay)'{-D,--delay}'[change the delay time of the schedule]' \
+            '(-P|--priority)'{-P,--priority}'[set priority]' \
+            '(-R|--retry)'{-R,--retry}'[automatic retrying count]' \
+                )
+        ;;
+    sched:histroy)
+        _subcommand_args=(
+            '(-p|--page)'{-p,--page}'[skip N pages]' \
+            '(-s|--skip)'{-s,--skip}'[skip N jobs]' \
+                )
+        ;;
+    sched:run)
+        _subcommand_args=(
+            '(-n|--num)'{-n,--num}'[number of jobs to run]' \
+                )
+        ;;
+    table:delete)
+        _subcommand_args=(
+            '(-f|--force)'{-f,--force}'[never prompt]' \
+                )
+        ;;
+    table:list)
+        _subcommand_args=(
+            '(-n|--num_threads)'{-num,--num_threads}'[number of threads to get list in parallel]' \
+            '--show-bytes[show estimated table in bytes]' \
+                )
+        ;;
+    table:tail)
+        _subcommand_args=(
+            '(-t|--to)'{-t,--to}'[end time of logs to get]' \
+            '(-f|--from)'{-f,--from}'[start time of logs to get]' \
+            '(-c|--count)'{-c,--count}'[number of logs to get]' \
+            '(-P|--pretty)'{-P,--pretty}'[pretty print]' \
+                )
+        ;;
+    table:import)
+        _subcommand_args=(
+            '--format[file format (default: apache)]' \
+            '--apache[same as --format apache; apache common log format]' \
+            '--syslog[same as --format syslog; syslog]' \
+            '--msgpack[same as --format msgpack; msgpack stream format]' \
+            '--json[same as --format json; LF-separated json format]' \
+            '(-t|--time-key)'{-t,--time-key}'[time key name for json and msgpack format (e.g. created_at)]' \
+            "--auto-create-table[create table and database if doesn't exist]" \
+                )
+        ;;
+    table:export)
+        _subcommand_args=(
+            '(-g|--org)'{-g,--org}'[export the data under this organization]' \
+            '(-t|--to)'{-t,--to}'[export data which is older than the TIME]' \
+            '(-f|--from)'{-f,--from}'[export data which is newer than or same with the TIME]' \
+            '(-b|--bucket)'{-b,--bucket}'[name of the destination S3 bucket (required)]' \
+            '(-k|--aws-key-id)'{-k,--aws-key-id}'[AWS access key id to export data (required)]' \
+            '(-s|--aws-secret-key)'{-s,--aws-secret-key}'[AWS secret key to export data (required)]' \
+                )
+        ;;
+esac
+
+_arguments \
+  $_subcommand_args \
+  && return 0
+}
+
+_td

data/contrib/completion/td-comletion.bash

@@ -0,0 +1,28 @@
+# bash completion for td commands
+
+_td()
+{
+    COMP_WORDBREAKS=${COMP_WORDBREAKS//:}
+    local cur="${COMP_WORDS[COMP_CWORD]}"
+    local prev="${COMP_WORDS[COMP_CWORD - 1]}"
+    local list="`td help:all | awk '{print $1}' | grep '^[a-z]' | xargs`"
+
+    # echo "cur=$cur, prev=$prev"
+
+    if [[ "$prev" == "td" ]]; then
+        if [[ "$cur" == "" ]]; then
+            COMPREPLY=($list)
+        else
+            COMPREPLY=($(compgen -W "$list" -- "$cur"))
+        fi
+    fi
+}
+complete -F _td td
+
+# Local variables:
+# # mode: shell-script
+# # sh-basic-offset: 4
+# # sh-indent-comment: t
+# # indent-tabs-mode: nil
+# # End:
+# # ex: ts=4 sw=4 et filetype=sh

data/lib/td/command/bulk_import.rb

@@ -27,7 +27,7 @@ module Command
   def bulk_import_create(op)
     org = nil
 
-    op.on('-g', '--org ORGANIZATION', "create the database under this organization") {|s|
+    op.on('-g', '--org ORGANIZATION', "create the bulk import session under this organization") {|s|
       org = s
     }
 

data/lib/td/command/common.rb

@@ -16,6 +16,9 @@ module Command
   end
 
   def get_client(opts={})
+    unless opts.has_key?(:ssl)
+      opts[:ssl] = Config.secure
+    end
     apikey = Config.apikey
     unless apikey
       raise ConfigError, "Account is not configured."

data/lib/td/command/ip_limit.rb

@@ -0,0 +1,53 @@
+
+module TreasureData
+module Command
+  def ip_limit_list(op)
+    op.cmd_parse
+
+    client = get_client
+
+    ip_limits = client.ip_limits
+    rows = ip_limits.map { |ip_limit|
+      {:Organization => ip_limit.org, 'IP Range' => ip_limit.ip_range}
+    }
+
+    puts cmd_render_table(rows, :fields => [:Organization, 'IP Range'])
+
+    if rows.empty?
+      $stderr.puts "There are no IP range limitations."
+      $stderr.puts "Use '#{$prog} ip_limit:set <organization> <ip_range>' to create IP range limitation."
+    end
+  end
+
+  def ip_limit_show(op)
+    organization = op.cmd_parse
+
+    client = get_client
+
+    ip_limits = client.ip_limits
+    rows = ip_limits.select { |ip_limit|
+      ip_limit.org == organization
+    }.map { |ip_limit| {'IP Range' => ip_limit.ip_range} }
+
+    puts cmd_render_table(rows, :fields => ['IP Range'])
+  end
+
+  def ip_limit_set(op)
+    organization, *ip_ranges = op.cmd_parse
+
+    client = get_client
+    client.set_ip_limit(organization, ip_ranges)
+
+    $stderr.puts "IP range limitations [#{ip_ranges.join(' ')}] are set to #{organization}"
+  end
+
+  def ip_limit_delete(op)
+    organization = op.cmd_parse
+
+    client = get_client
+    client.delete_ip_limit(organization)
+
+    $stderr.puts "All IP range limitations are deleted from #{organization}"
+  end
+end
+end

data/lib/td/command/list.rb

@@ -303,6 +303,11 @@ module List
   add_list 'acl:revoke', %w[subject action scope], 'Revoke an access control'
   # TODO acl:test
 
+  add_list 'ip_limit:list', %w[], 'Show list of all IP range limitations'
+  add_list 'ip_limit:show', %w[org], "Show list of org's IP range limitations"
+  add_list 'ip_limit:set', %w[org ip_range_], 'Set an IP range limitation'
+  add_list 'ip_limit:delete', %w[org], 'Delete an IP range limitation'
+
   add_list 'aggr:list', %w[], 'Show list of aggregation schemas'
   add_list 'aggr:show', %w[name], 'Describe a aggregation schema'
   add_list 'aggr:create', %w[name relation_key], 'Create a aggregation schema'
@@ -372,6 +377,8 @@ module List
   add_alias 'acl', 'acl:list'
   add_alias 'acls', 'acl:list'
 
+  add_alias 'ip_limits', 'ip_limit:list'
+
   add_alias 'aggr', 'aggr:show'
   add_alias 'aggrs', 'aggr:list'
 

data/lib/td/command/result.rb

@@ -51,7 +51,7 @@ module Command
     result_user = nil
     result_ask_password = false
 
-    op.on('-g', '--org ORGANIZATION', "create the database under this organization") {|s|
+    op.on('-g', '--org ORGANIZATION', "create the result under this organization") {|s|
       org = s
     }
     op.on('-u', '--user NAME', 'set user name for authentication') {|s|

data/lib/td/command/runner.rb

@@ -8,9 +8,10 @@ class Runner
     @config_path = nil
     @apikey = nil
     @prog_name = nil
+    @secure = true
   end
 
-  attr_accessor :apikey, :config_path, :prog_name
+  attr_accessor :apikey, :config_path, :prog_name, :secure
 
   def run(argv=ARGV)
     require 'td/version'
@@ -67,6 +68,7 @@ EOF
 
     config_path = @config_path
     apikey = @apikey
+    insecure = nil
     $verbose = false
     #$debug = false
 
@@ -78,6 +80,10 @@ EOF
       apikey = s
     }
 
+    op.on('--insecure', "Insecure access: disable SSL") { |b|
+      insecure = true
+    }
+
     op.on('-v', '--verbose', "verbose mode", TrueClass) {|b|
       $verbose = b
     }
@@ -102,6 +108,9 @@ EOF
       if apikey
         TreasureData::Config.apikey = apikey
       end
+      if insecure
+        TreasureData::Config.secure = false
+      end
     rescue
       usage $!.to_s
     end

data/lib/td/config.rb

@@ -91,6 +91,7 @@ class Config
   @@path = ENV['TREASURE_DATA_CONFIG_PATH'] || ENV['TD_CONFIG_PATH'] || File.join(ENV['HOME'], '.td', 'td.conf')
   @@apikey = ENV['TREASURE_DATA_API_KEY'] || ENV['TD_API_KEY']
   @@apikey = nil if @@apikey == ""
+  @@secure = true
 
   def self.path
     @@path
@@ -107,6 +108,14 @@ class Config
   def self.apikey=(apikey)
     @@apikey = apikey
   end
+
+  def self.secure
+    @@secure
+  end
+
+  def self.secure=(secure)
+    @@secure = secure
+  end
 end
 
 

data/lib/td/version.rb

@@ -1,5 +1,5 @@
 module TreasureData
 
-VERSION = '0.10.73'
+VERSION = '0.10.74'
 
 end

data/td.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "yajl-ruby", "~> 1.1.0"
   gem.add_dependency "hirb", ">= 0.4.5"
   gem.add_dependency "parallel", "~> 0.5.19"
-  gem.add_dependency "td-client", "~> 0.8.45"
+  gem.add_dependency "td-client", "~> 0.8.46"
   gem.add_dependency "td-logger", "~> 0.3.16"
   gem.add_development_dependency "rake", "~> 0.9"
   gem.add_development_dependency "rspec", "~> 2.10.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: td
 version: !ruby/object:Gem::Version
-  version: 0.10.73
+  version: 0.10.74
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-15 00:00:00.000000000 Z
+date: 2013-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -82,7 +82,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.45
+        version: 0.8.46
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.8.45
+        version: 0.8.46
 - !ruby/object:Gem::Dependency
   name: td-logger
   requirement: !ruby/object:Gem::Requirement
@@ -169,6 +169,8 @@ files:
 - Rakefile
 - bin/td
 - build/update-td-bulk-import-java.sh
+- contrib/completion/_td
+- contrib/completion/td-comletion.bash
 - data/sample_apache.json
 - data/sample_apache_gen.rb
 - dist/exe.rake
@@ -194,6 +196,7 @@ files:
 - lib/td/command/export.rb
 - lib/td/command/help.rb
 - lib/td/command/import.rb
+- lib/td/command/ip_limit.rb
 - lib/td/command/job.rb
 - lib/td/command/list.rb
 - lib/td/command/org.rb
@@ -239,7 +242,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4066201097881457752
+      hash: 1161270452810499022
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -248,7 +251,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4066201097881457752
+      hash: 1161270452810499022
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23