iij-dag 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +18 -0
- data/Gemfile +16 -0
- data/LICENSE.txt +174 -0
- data/Rakefile +27 -0
- data/bin/dag +7 -0
- data/iij-dag.gemspec +34 -0
- data/lib/dag/cli.rb +17 -0
- data/lib/dag/cli/base.rb +128 -0
- data/lib/dag/cli/command.rb +83 -0
- data/lib/dag/cli/command/import.rb +30 -0
- data/lib/dag/cli/command/query.rb +14 -0
- data/lib/dag/cli/sub_command.rb +18 -0
- data/lib/dag/cli/sub_command/bucket.rb +90 -0
- data/lib/dag/cli/sub_command/cluster.rb +187 -0
- data/lib/dag/cli/sub_command/db.rb +38 -0
- data/lib/dag/cli/sub_command/job.rb +341 -0
- data/lib/dag/cli/sub_command/table.rb +168 -0
- data/lib/dag/cli/utils.rb +7 -0
- data/lib/dag/cli/utils/number_to_human.rb +23 -0
- data/lib/dag/cli/utils/terminal_table.rb +10 -0
- data/lib/dag/cli/utils/time_format.rb +8 -0
- data/lib/dag/cli/version.rb +5 -0
- metadata +221 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
module Dag
|
|
3
|
+
module CLI
|
|
4
|
+
class Table < SubCommand
|
|
5
|
+
include Dag::CLI::Utils::NumberToHuman
|
|
6
|
+
|
|
7
|
+
desc 'list ([DB])', 'Tables list name/location/comment/created_at'
|
|
8
|
+
def list(database = nil)
|
|
9
|
+
headers = %w(database table location comment created_at)
|
|
10
|
+
rows = []
|
|
11
|
+
|
|
12
|
+
handle_api_failure {
|
|
13
|
+
if database
|
|
14
|
+
open_client.database(database).tables.each do |table|
|
|
15
|
+
rows << [
|
|
16
|
+
table.db_name,
|
|
17
|
+
table.name,
|
|
18
|
+
table.location,
|
|
19
|
+
table.comment,
|
|
20
|
+
time_format(table.created_at)
|
|
21
|
+
]
|
|
22
|
+
end
|
|
23
|
+
else
|
|
24
|
+
open_client.databases.each do |db|
|
|
25
|
+
db.tables.each do |table|
|
|
26
|
+
rows << [
|
|
27
|
+
table.db_name,
|
|
28
|
+
table.name,
|
|
29
|
+
table.location,
|
|
30
|
+
table.comment,
|
|
31
|
+
time_format(table.created_at)
|
|
32
|
+
]
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
terminal_table(rows, headers: headers)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
desc 'info [DB] [TABLE]', 'Table info database/table/schema/location/comment/created_at'
|
|
42
|
+
def info(database, table)
|
|
43
|
+
headers = %w(item info)
|
|
44
|
+
rows = []
|
|
45
|
+
|
|
46
|
+
tbl = handle_api_failure do
|
|
47
|
+
open_client.database(database).table(table)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
rows << ["name", tbl.name]
|
|
51
|
+
rows << ["format", tbl.format]
|
|
52
|
+
rows << ["schema", tbl.schema]
|
|
53
|
+
rows << ["location", tbl.location]
|
|
54
|
+
rows << ["comment", tbl.comment]
|
|
55
|
+
rows << ["created_at", time_format(tbl.created_at)]
|
|
56
|
+
rows << ["modified_at", time_format(tbl.modified_at)]
|
|
57
|
+
|
|
58
|
+
terminal_table(rows, headers: headers, max_width: Float::INFINITY)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
desc 'create [DB] [TABLE]', 'Create table'
|
|
62
|
+
option :format, type: :string, aliases: '-f', desc: "format [csv/tsv/json/json_agent]"
|
|
63
|
+
option :comment, type: :string, desc: "table comment"
|
|
64
|
+
option :schema, type: :string, aliases: '-s', desc: "table schema"
|
|
65
|
+
def create(database, table)
|
|
66
|
+
params = {}
|
|
67
|
+
params.merge!({ format: options[:format] }) if options[:format]
|
|
68
|
+
params.merge!({ comment: options[:comment] }) if options[:comment]
|
|
69
|
+
params.merge!({ schema: options[:schema] }) if options[:schema]
|
|
70
|
+
|
|
71
|
+
tbl = handle_api_failure do
|
|
72
|
+
open_client.create_table(database, table, params)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
say_status "Create table", tbl.name
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
desc 'update [DB] [TABLE]', 'Update table'
|
|
79
|
+
option :format, type: :string, aliases: '-f', desc: "format [csv/tsv/json/json_agent]"
|
|
80
|
+
option :comment, type: :string, desc: "table comment"
|
|
81
|
+
def update(database, table)
|
|
82
|
+
params = {}
|
|
83
|
+
params.merge!({ format: options[:format] }) if options[:format]
|
|
84
|
+
params.merge!({ comment: options[:comment] }) if options[:comment]
|
|
85
|
+
|
|
86
|
+
tbl = handle_api_failure do
|
|
87
|
+
open_client.database(database).table(table).update(params)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
say_status "Update table", tbl.name
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
desc 'delete [DB] [TABLE]', 'Delete Table'
|
|
94
|
+
def delete(database, table)
|
|
95
|
+
handle_api_failure do
|
|
96
|
+
open_client.database(database).table(table).delete
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
say_status "Delete table", table
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
desc 'split [DB] [TABLE]', 'Split objects into table'
|
|
103
|
+
option 'input-objects', type: :array, aliases: '-i', desc: "input objects", required: true
|
|
104
|
+
option :format, type: :string, aliases: '-f', desc: "format [csv/tsv/json]", required: true
|
|
105
|
+
option :comment, type: :string, desc: "table comment"
|
|
106
|
+
def split(database, table)
|
|
107
|
+
params = {
|
|
108
|
+
input_format: options[:format],
|
|
109
|
+
input_object_keys: options['input-objects']
|
|
110
|
+
}
|
|
111
|
+
params.merge!({ comment: options[:comment] }) if options[:comment]
|
|
112
|
+
|
|
113
|
+
job = handle_api_failure do
|
|
114
|
+
db = open_client.database(database)
|
|
115
|
+
db.split(table, params)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
say_status "Accepted split", "job_id: #{job.id}"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
desc 'size [DB] [TABLE]', 'data size of table'
|
|
122
|
+
option :sample, type: :numeric, aliases: '-s', desc: "Guess Compress Ratio by sample N objects", default: 0
|
|
123
|
+
def size(database, table)
|
|
124
|
+
res=[{:Title=>"Total", :Objects=>0, :Size=>0},
|
|
125
|
+
{:Title=>"Gzip-ed", :Objects=>0, :Size=>0},
|
|
126
|
+
{:Title=>"Skipped", :Objects=>0, :Size=>0}]
|
|
127
|
+
bucket = client.buckets[database]
|
|
128
|
+
rows = handle_api_failure{
|
|
129
|
+
bucket.objects.where(prefix: table, delimiter: nil)
|
|
130
|
+
}
|
|
131
|
+
ckeys=[]
|
|
132
|
+
rows.each{|r|
|
|
133
|
+
if [".", "_"].include?(File.basename(r.name)[0]) ||
|
|
134
|
+
r.name.end_with?("_$folder$")
|
|
135
|
+
res[2][:Objects]+=1
|
|
136
|
+
res[2][:Size]+=r.size
|
|
137
|
+
next
|
|
138
|
+
end
|
|
139
|
+
res[0][:Objects]+=1
|
|
140
|
+
res[0][:Size]+=r.size
|
|
141
|
+
if r.name.end_with? ".gz"
|
|
142
|
+
ckeys << [r.size, r.name]
|
|
143
|
+
res[1][:Objects]+=1
|
|
144
|
+
res[1][:Size]+=r.size
|
|
145
|
+
end
|
|
146
|
+
}
|
|
147
|
+
ucsz = 0
|
|
148
|
+
csz = 0
|
|
149
|
+
ckeys.sample(options[:sample]).each{|k|
|
|
150
|
+
d = bucket.objects[k[1]].read(-4..-1)
|
|
151
|
+
csz += k[0]
|
|
152
|
+
ucsz += d.unpack("V")[0]
|
|
153
|
+
}
|
|
154
|
+
fields=[:Title, :Objects, :Size]
|
|
155
|
+
if csz!=0
|
|
156
|
+
res[1]["Ratio%"]="%.2f" %[csz*100.0/ucsz]
|
|
157
|
+
unc=(res[1][:Size]/(csz*1.0/ucsz)).to_i
|
|
158
|
+
res[1][:Uncompressed]=number_to_human(unc)
|
|
159
|
+
res[0][:Uncompressed]=number_to_human(res[0][:Size]-res[1][:Size]+unc)
|
|
160
|
+
fields << "Ratio%"
|
|
161
|
+
fields << :Uncompressed
|
|
162
|
+
end
|
|
163
|
+
res.each{|m| m[:Size]=number_to_human(m[:Size]) }
|
|
164
|
+
terminal_table res, fields: fields
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module Dag::CLI::Utils
|
|
2
|
+
module NumberToHuman
|
|
3
|
+
require 'active_support/number_helper/number_converter'
|
|
4
|
+
require 'active_support/number_helper/number_to_delimited_converter'
|
|
5
|
+
require 'active_support/number_helper/number_to_rounded_converter'
|
|
6
|
+
require 'active_support/number_helper/number_to_human_size_converter'
|
|
7
|
+
|
|
8
|
+
include ActiveSupport::NumberHelper
|
|
9
|
+
I18n.enforce_available_locales = false
|
|
10
|
+
|
|
11
|
+
def number_to_human(number, options = {})
|
|
12
|
+
NumberToHumanSizeConverter.new(number, options).execute
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
#{
|
|
16
|
+
# 'B' => 1024,
|
|
17
|
+
# 'KB' => 1024 * 1024,
|
|
18
|
+
# 'MB' => 1024 * 1024 * 1024,
|
|
19
|
+
# 'GB' => 1024 * 1024 * 1024 * 1024,
|
|
20
|
+
# 'TB' => 1024 * 1024 * 1024 * 1024 * 1024
|
|
21
|
+
#}.each_pair { |e, s| return "#{(num.to_f / (s / 1024)).round(2)} #{e}" if num < s }
|
|
22
|
+
end
|
|
23
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: iij-dag
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 1.0.1
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- iij
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2015-06-04 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: thor
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 0.19.1
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: 0.19.1
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: ruby-progressbar
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: 1.1.0
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: 1.1.0
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: iij-dag-client
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 1.0.0
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 1.0.0
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: hirb
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 0.7.3
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 0.7.3
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: hirb-unicode
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: 0.0.5
|
|
76
|
+
type: :runtime
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 0.0.5
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: anbt-sql-formatter
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: 0.0.3
|
|
90
|
+
type: :runtime
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - "~>"
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: 0.0.3
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: oj
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: 2.11.2
|
|
104
|
+
type: :runtime
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: 2.11.2
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: activesupport
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: 4.2.0
|
|
118
|
+
type: :runtime
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: 4.2.0
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: bundler
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - "~>"
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '1.3'
|
|
132
|
+
type: :development
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - "~>"
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '1.3'
|
|
139
|
+
- !ruby/object:Gem::Dependency
|
|
140
|
+
name: rake
|
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - ">="
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: '0'
|
|
146
|
+
type: :development
|
|
147
|
+
prerelease: false
|
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - ">="
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: '0'
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: rspec
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - "~>"
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: 3.1.0
|
|
160
|
+
type: :development
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - "~>"
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: 3.1.0
|
|
167
|
+
description: iij gio dag command line tool
|
|
168
|
+
email:
|
|
169
|
+
- dag-info@iij.ad.jp
|
|
170
|
+
executables:
|
|
171
|
+
- dag
|
|
172
|
+
extensions: []
|
|
173
|
+
extra_rdoc_files: []
|
|
174
|
+
files:
|
|
175
|
+
- ".gitignore"
|
|
176
|
+
- Gemfile
|
|
177
|
+
- LICENSE.txt
|
|
178
|
+
- Rakefile
|
|
179
|
+
- bin/dag
|
|
180
|
+
- iij-dag.gemspec
|
|
181
|
+
- lib/dag/cli.rb
|
|
182
|
+
- lib/dag/cli/base.rb
|
|
183
|
+
- lib/dag/cli/command.rb
|
|
184
|
+
- lib/dag/cli/command/import.rb
|
|
185
|
+
- lib/dag/cli/command/query.rb
|
|
186
|
+
- lib/dag/cli/sub_command.rb
|
|
187
|
+
- lib/dag/cli/sub_command/bucket.rb
|
|
188
|
+
- lib/dag/cli/sub_command/cluster.rb
|
|
189
|
+
- lib/dag/cli/sub_command/db.rb
|
|
190
|
+
- lib/dag/cli/sub_command/job.rb
|
|
191
|
+
- lib/dag/cli/sub_command/table.rb
|
|
192
|
+
- lib/dag/cli/utils.rb
|
|
193
|
+
- lib/dag/cli/utils/number_to_human.rb
|
|
194
|
+
- lib/dag/cli/utils/terminal_table.rb
|
|
195
|
+
- lib/dag/cli/utils/time_format.rb
|
|
196
|
+
- lib/dag/cli/version.rb
|
|
197
|
+
homepage: http://www.iij.ad.jp/biz/storage/
|
|
198
|
+
licenses:
|
|
199
|
+
- Apache License 2.0
|
|
200
|
+
metadata: {}
|
|
201
|
+
post_install_message:
|
|
202
|
+
rdoc_options: []
|
|
203
|
+
require_paths:
|
|
204
|
+
- lib
|
|
205
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
206
|
+
requirements:
|
|
207
|
+
- - ">="
|
|
208
|
+
- !ruby/object:Gem::Version
|
|
209
|
+
version: '0'
|
|
210
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
211
|
+
requirements:
|
|
212
|
+
- - ">="
|
|
213
|
+
- !ruby/object:Gem::Version
|
|
214
|
+
version: '0'
|
|
215
|
+
requirements: []
|
|
216
|
+
rubyforge_project:
|
|
217
|
+
rubygems_version: 2.2.3
|
|
218
|
+
signing_key:
|
|
219
|
+
specification_version: 4
|
|
220
|
+
summary: iij gio dag command line tool
|
|
221
|
+
test_files: []
|