td 0.7.3 → 0.7.4
Sign up to get free protection for your applications and to get access to all the features.
- data/ChangeLog +6 -0
- data/lib/td/api.rb +54 -3
- data/lib/td/client.rb +63 -28
- data/lib/td/command/common.rb +11 -1
- data/lib/td/command/list.rb +9 -3
- data/lib/td/command/table.rb +68 -7
- data/lib/td/command/td.rb +2 -0
- data/lib/td/version.rb +1 -1
- metadata +4 -4
data/ChangeLog
CHANGED
data/lib/td/api.rb
CHANGED
@@ -36,12 +36,43 @@ class API
|
|
36
36
|
unless name =~ /^([a-z0-9_]+)$/
|
37
37
|
raise "Name must consist only of alphabets, numbers, '_'."
|
38
38
|
end
|
39
|
+
name
|
39
40
|
end
|
40
41
|
|
41
42
|
def self.validate_table_name(name)
|
42
43
|
validate_database_name(name)
|
43
44
|
end
|
44
45
|
|
46
|
+
def self.validate_column_name(name)
|
47
|
+
name = name.to_s
|
48
|
+
if name.empty?
|
49
|
+
raise "Empty column name is not allowed"
|
50
|
+
end
|
51
|
+
if 32 < name.length
|
52
|
+
raise "Column name must be to 32 characters, got #{name.length} characters."
|
53
|
+
end
|
54
|
+
unless name =~ /^([a-z0-9_]+)$/
|
55
|
+
raise "Column name must consist only of alphabets, numbers, '_'."
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.normalize_type_name(name)
|
60
|
+
case name
|
61
|
+
when /int/i, /integer/i
|
62
|
+
"int"
|
63
|
+
when /long/i, /bigint/i
|
64
|
+
"long"
|
65
|
+
when /string/i
|
66
|
+
"string"
|
67
|
+
when /float/i
|
68
|
+
"float"
|
69
|
+
when /double/i
|
70
|
+
"double"
|
71
|
+
else
|
72
|
+
raise "Type name must eather of int, long, string float or double"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
45
76
|
####
|
46
77
|
## Database API
|
47
78
|
##
|
@@ -94,19 +125,20 @@ class API
|
|
94
125
|
name = m['name']
|
95
126
|
type = (m['type'] || '?').to_sym
|
96
127
|
count = (m['count'] || 0).to_i # TODO?
|
97
|
-
|
128
|
+
schema = JSON.parse(m['schema']) # FIXME?
|
129
|
+
result[name] = [type, schema, count]
|
98
130
|
}
|
99
131
|
return result
|
100
132
|
end
|
101
133
|
|
102
|
-
|
103
|
-
def create_table(db, table, type)
|
134
|
+
def create_log_or_item_table(db, table, type)
|
104
135
|
code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
|
105
136
|
if code != "200"
|
106
137
|
raise_error("Create #{type} table failed", res)
|
107
138
|
end
|
108
139
|
return true
|
109
140
|
end
|
141
|
+
private :create_log_or_item_table
|
110
142
|
|
111
143
|
# => true
|
112
144
|
def create_log_table(db, table)
|
@@ -118,6 +150,25 @@ class API
|
|
118
150
|
create_table(db, table, :item)
|
119
151
|
end
|
120
152
|
|
153
|
+
def create_table(db, table, type)
|
154
|
+
schema = schema.to_s
|
155
|
+
code, body, res = post("/v3/table/create/#{e db}/#{e table}/#{type}")
|
156
|
+
if code != "200"
|
157
|
+
raise_error("Create #{type} table failed", res)
|
158
|
+
end
|
159
|
+
return true
|
160
|
+
end
|
161
|
+
private :create_table
|
162
|
+
|
163
|
+
# => true
|
164
|
+
def update_schema(db, table, schema_json)
|
165
|
+
code, body, res = post("/v3/table/update-schema/#{e db}/#{e table}", {'schema'=>schema_json})
|
166
|
+
if code != "200"
|
167
|
+
raise_error("Create schema table failed", res)
|
168
|
+
end
|
169
|
+
return true
|
170
|
+
end
|
171
|
+
|
121
172
|
# => type:Symbol
|
122
173
|
def delete_table(db, table)
|
123
174
|
code, body, res = post("/v3/table/delete/#{e db}/#{e table}")
|
data/lib/td/client.rb
CHANGED
@@ -59,18 +59,18 @@ class Client
|
|
59
59
|
end
|
60
60
|
|
61
61
|
# => true
|
62
|
-
def
|
63
|
-
@api.
|
62
|
+
def create_log_table(db_name, table_name)
|
63
|
+
@api.create_log_table(db_name, table_name)
|
64
64
|
end
|
65
65
|
|
66
66
|
# => true
|
67
|
-
def
|
68
|
-
|
67
|
+
def create_item_table(db_name, table_name)
|
68
|
+
@api.create_item_table(db_name, table_name)
|
69
69
|
end
|
70
70
|
|
71
71
|
# => true
|
72
|
-
def
|
73
|
-
|
72
|
+
def update_schema(db_name, table_name, schema)
|
73
|
+
@api.update_schema(db_name, table_name, schema.to_json)
|
74
74
|
end
|
75
75
|
|
76
76
|
# => type:Symbol
|
@@ -81,17 +81,17 @@ class Client
|
|
81
81
|
# => [Table]
|
82
82
|
def tables(db_name)
|
83
83
|
m = @api.list_tables(db_name)
|
84
|
-
m.map {|table_name,(type,count)|
|
85
|
-
|
84
|
+
m.map {|table_name,(type,schema,count)|
|
85
|
+
schema = Schema.new.from_json(schema)
|
86
|
+
Table.new(self, db_name, table_name, type, schema, count)
|
86
87
|
}
|
87
88
|
end
|
88
89
|
|
89
90
|
# => Table
|
90
91
|
def table(db_name, table_name)
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
return Table.new(self, db_name, name, type, count)
|
92
|
+
tables(db_name).each {|t|
|
93
|
+
if t.name == table_name
|
94
|
+
return t
|
95
95
|
end
|
96
96
|
}
|
97
97
|
raise NotFoundError, "Table '#{db_name}.#{table_name}' does not exist"
|
@@ -168,16 +168,12 @@ class Database < Model
|
|
168
168
|
@tables
|
169
169
|
end
|
170
170
|
|
171
|
-
def create_table(name, type)
|
172
|
-
@client.create_table(@db_name, name, type)
|
173
|
-
end
|
174
|
-
|
175
171
|
def create_log_table(name)
|
176
|
-
|
172
|
+
@client.create_log_table(@db_name, name)
|
177
173
|
end
|
178
174
|
|
179
175
|
def create_item_table(name)
|
180
|
-
|
176
|
+
@client.create_item_table(@db_name, name)
|
181
177
|
end
|
182
178
|
|
183
179
|
def table(table_name)
|
@@ -194,28 +190,24 @@ class Database < Model
|
|
194
190
|
end
|
195
191
|
|
196
192
|
class Table < Model
|
197
|
-
def initialize(client, db_name, table_name, type, count)
|
193
|
+
def initialize(client, db_name, table_name, type, schema, count)
|
198
194
|
super(client)
|
199
195
|
@db_name = db_name
|
200
196
|
@table_name = table_name
|
201
197
|
@type = type
|
198
|
+
@schema = schema
|
202
199
|
@count = count
|
203
200
|
end
|
204
201
|
|
205
|
-
attr_reader :type, :count
|
202
|
+
attr_reader :type, :db_name, :table_name, :schema, :count
|
206
203
|
|
207
|
-
|
208
|
-
|
209
|
-
end
|
204
|
+
alias database_name db_name
|
205
|
+
alias name table_name
|
210
206
|
|
211
207
|
def database
|
212
208
|
@client.database(@db_name)
|
213
209
|
end
|
214
210
|
|
215
|
-
def name
|
216
|
-
@table_name
|
217
|
-
end
|
218
|
-
|
219
211
|
def identifier
|
220
212
|
"#{@db_name}.#{@table_name}"
|
221
213
|
end
|
@@ -225,6 +217,50 @@ class Table < Model
|
|
225
217
|
end
|
226
218
|
end
|
227
219
|
|
220
|
+
class Schema
|
221
|
+
class Field
|
222
|
+
def initialize(name, type)
|
223
|
+
@name = name
|
224
|
+
@type = type
|
225
|
+
end
|
226
|
+
attr_reader :name
|
227
|
+
attr_reader :type
|
228
|
+
end
|
229
|
+
|
230
|
+
def self.parse(cols)
|
231
|
+
fields = cols.split(',').map {|col|
|
232
|
+
name, type, *_ = col.split(':')
|
233
|
+
Field.new(name, type)
|
234
|
+
}
|
235
|
+
Schema.new(fields)
|
236
|
+
end
|
237
|
+
|
238
|
+
def initialize(fields=[])
|
239
|
+
@fields = fields
|
240
|
+
end
|
241
|
+
|
242
|
+
attr_reader :fields
|
243
|
+
|
244
|
+
def add_field(name, type)
|
245
|
+
@fields << Field.new(name, type)
|
246
|
+
end
|
247
|
+
|
248
|
+
def to_s
|
249
|
+
@fields.map {|f| "#{f.name}:#{f.type}" }.join(',')
|
250
|
+
end
|
251
|
+
|
252
|
+
def to_json(*args)
|
253
|
+
@fields.map {|f| [f.name, f.type] }.to_json(*args)
|
254
|
+
end
|
255
|
+
|
256
|
+
def from_json(obj)
|
257
|
+
@fields = obj.map {|f|
|
258
|
+
Field.new(f[0], f[1])
|
259
|
+
}
|
260
|
+
self
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
228
264
|
class Job < Model
|
229
265
|
def initialize(client, job_id, type, query, status=nil, url=nil, debug=nil, start_at=nil, end_at=nil, result=nil)
|
230
266
|
super(client)
|
@@ -332,6 +368,5 @@ class Job < Model
|
|
332
368
|
end
|
333
369
|
end
|
334
370
|
|
335
|
-
|
336
371
|
end
|
337
372
|
|
data/lib/td/command/common.rb
CHANGED
@@ -1,11 +1,22 @@
|
|
1
1
|
|
2
2
|
module TreasureData
|
3
|
+
|
4
|
+
autoload :API, 'td/api'
|
5
|
+
autoload :Client, 'td/client'
|
6
|
+
autoload :Database, 'td/client'
|
7
|
+
autoload :Table, 'td/client'
|
8
|
+
autoload :Schema, 'td/client'
|
9
|
+
autoload :Job, 'td/client'
|
10
|
+
|
3
11
|
module Command
|
4
12
|
private
|
5
13
|
def cmd_opt(name, *args)
|
6
14
|
if args.last.to_s =~ /_$/
|
7
15
|
multi = true
|
8
16
|
args.push args.pop.to_s[0..-2]+'...'
|
17
|
+
elsif args.last.to_s =~ /_\?$/
|
18
|
+
multi = true
|
19
|
+
args.push args.pop.to_s[0..-3]+'...?'
|
9
20
|
end
|
10
21
|
|
11
22
|
req_args, opt_args = args.partition {|a| a.to_s !~ /\?$/ }
|
@@ -61,7 +72,6 @@ EOF
|
|
61
72
|
unless apikey
|
62
73
|
raise ConfigError, "Account is not configured."
|
63
74
|
end
|
64
|
-
require 'td/client'
|
65
75
|
Client.new(apikey)
|
66
76
|
end
|
67
77
|
|
data/lib/td/command/list.rb
CHANGED
@@ -33,25 +33,31 @@ module List
|
|
33
33
|
add_list 'server', 'server-status', 'Show status of the Treasure Data server'
|
34
34
|
add_list 'database', 'show-databases', 'Show list of databases'
|
35
35
|
add_list 'table', 'show-tables', 'Show list of tables'
|
36
|
-
add_list 'query', 'show-jobs', 'Show list of jobs'
|
37
36
|
add_list 'database', 'create-database', 'Create a database'
|
38
37
|
add_list 'table', 'create-log-table', 'Create a log table'
|
39
38
|
#add_list 'table', 'create-item-table', 'Create a item table'
|
39
|
+
add_list 'table', 'set-schema', 'Set a schema on a table'
|
40
|
+
add_list 'table', 'describe-table', 'Describe information of a table'
|
40
41
|
add_list 'database', 'drop-database', 'Delete a database'
|
41
42
|
add_list 'table', 'drop-table', 'Delete a table'
|
42
43
|
add_list 'query', 'query', 'Start a query'
|
43
44
|
add_list 'query', 'job', 'Show status and result of a job'
|
45
|
+
add_list 'query', 'show-jobs', 'Show list of jobs'
|
44
46
|
add_list 'import', 'import', 'Import files to a table'
|
45
47
|
add_list 'list', 'version', 'Show version'
|
46
48
|
|
47
49
|
add_alias 'show-dbs', 'show-databases'
|
48
|
-
add_alias '
|
50
|
+
add_alias 'databases', 'show-databases'
|
51
|
+
add_alias 'dbs', 'show-databases'
|
49
52
|
add_alias 'create-db', 'create-databases'
|
50
53
|
add_alias 'drop-db', 'create-databases'
|
51
|
-
add_alias '
|
54
|
+
add_alias 'tables', 'show-tables'
|
55
|
+
add_alias 'table', 'describe-table'
|
56
|
+
add_alias 'show-table', 'describe-table'
|
52
57
|
add_alias 'delete-database', 'drop-database'
|
53
58
|
add_alias 'delete-table', 'drop-table'
|
54
59
|
add_alias 'jobs', 'show-jobs'
|
60
|
+
add_alias 'update-schema', 'set-schema'
|
55
61
|
|
56
62
|
add_guess 'create-table', 'create-log-table'
|
57
63
|
add_guess 'drop-log-table', 'drop-table'
|
data/lib/td/command/table.rb
CHANGED
@@ -2,13 +2,17 @@
|
|
2
2
|
module TreasureData
|
3
3
|
module Command
|
4
4
|
|
5
|
-
def
|
5
|
+
def create_log_or_item_table(mode_log, db_name, table_name)
|
6
6
|
client = get_client
|
7
7
|
|
8
8
|
API.validate_table_name(table_name)
|
9
9
|
|
10
10
|
begin
|
11
|
-
|
11
|
+
if mode_log
|
12
|
+
client.create_log_table(db_name, table_name)
|
13
|
+
else
|
14
|
+
client.create_item_table(db_name, table_name)
|
15
|
+
end
|
12
16
|
rescue NotFoundError
|
13
17
|
cmd_debug_error $!
|
14
18
|
$stderr.puts "Database '#{db_name}' does not exist."
|
@@ -22,20 +26,20 @@ module Command
|
|
22
26
|
|
23
27
|
$stderr.puts "Table '#{db_name}.#{table_name}' is created."
|
24
28
|
end
|
25
|
-
private :
|
29
|
+
private :create_log_or_item_table
|
26
30
|
|
27
31
|
def create_log_table
|
28
32
|
op = cmd_opt 'create-log-table', :db_name, :table_name
|
29
33
|
db_name, table_name = op.cmd_parse
|
30
34
|
|
31
|
-
|
35
|
+
create_log_or_item_table(true, db_name, table_name)
|
32
36
|
end
|
33
37
|
|
34
38
|
def create_item_table
|
35
39
|
op = cmd_opt 'create-item-table', :db_name, :table_name
|
36
40
|
db_name, table_name = op.cmd_parse
|
37
41
|
|
38
|
-
|
42
|
+
create_log_or_item_table(false, db_name, table_name)
|
39
43
|
end
|
40
44
|
|
41
45
|
def drop_table
|
@@ -56,6 +60,42 @@ module Command
|
|
56
60
|
$stderr.puts "Table '#{db_name}.#{table_name}' is deleted."
|
57
61
|
end
|
58
62
|
|
63
|
+
def set_schema
|
64
|
+
op = cmd_opt 'set-schema', :db_name, :table_name, :columns_?
|
65
|
+
|
66
|
+
db_name, table_name, *columns = op.cmd_parse
|
67
|
+
|
68
|
+
schema = Schema.new
|
69
|
+
columns.each {|column|
|
70
|
+
name, type = column.split(':',2)
|
71
|
+
name = name.to_s
|
72
|
+
type = type.to_s
|
73
|
+
|
74
|
+
API.validate_column_name(name)
|
75
|
+
type = API.normalize_type_name(type)
|
76
|
+
|
77
|
+
if schema.fields.find {|f| f.name == name }
|
78
|
+
$stderr.puts "Column name '#{name}' is duplicated."
|
79
|
+
exit 1
|
80
|
+
end
|
81
|
+
schema.add_field(name, type)
|
82
|
+
|
83
|
+
if name == 'v' || name == 'time'
|
84
|
+
$stderr.puts "Column name '#{name}' is reserved."
|
85
|
+
exit 1
|
86
|
+
end
|
87
|
+
}
|
88
|
+
|
89
|
+
client = get_client
|
90
|
+
|
91
|
+
find_table(client, db_name, table_name)
|
92
|
+
|
93
|
+
client.update_schema(db_name, table_name, schema)
|
94
|
+
|
95
|
+
$stderr.puts "Schema is updated on #{db_name}.#{table_name} table."
|
96
|
+
$stderr.puts "Use '#{$prog} describe-table #{db_name} #{table_name}' to confirm the schema."
|
97
|
+
end
|
98
|
+
|
59
99
|
def show_tables
|
60
100
|
op = cmd_opt 'show-tables', :db_name?
|
61
101
|
db_name = op.cmd_parse
|
@@ -72,14 +112,17 @@ module Command
|
|
72
112
|
rows = []
|
73
113
|
dbs.each {|db|
|
74
114
|
db.tables.each {|table|
|
75
|
-
|
115
|
+
pschema = table.schema.fields.map {|f|
|
116
|
+
"#{f.name}:#{f.type}"
|
117
|
+
}.join(', ')
|
118
|
+
rows << {:Database => db.name, :Table => table.name, :Type => table.type.to_s, :Count => table.count.to_s, :Schema=>pschema.to_s}
|
76
119
|
}
|
77
120
|
}
|
78
121
|
rows = rows.sort_by {|map|
|
79
122
|
[map[:Database], map[:Type].size, map[:Table]]
|
80
123
|
}
|
81
124
|
|
82
|
-
puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count])
|
125
|
+
puts cmd_render_table(rows, :fields => [:Database, :Table, :Type, :Count, :Schema])
|
83
126
|
|
84
127
|
if rows.empty?
|
85
128
|
if db_name
|
@@ -95,6 +138,24 @@ module Command
|
|
95
138
|
end
|
96
139
|
end
|
97
140
|
|
141
|
+
def describe_table
|
142
|
+
op = cmd_opt 'describe-table', :db_name, :table_name
|
143
|
+
|
144
|
+
db_name, table_name = op.cmd_parse
|
145
|
+
|
146
|
+
client = get_client
|
147
|
+
|
148
|
+
table = find_table(client, db_name, table_name)
|
149
|
+
|
150
|
+
puts "Name : #{table.db_name}.#{table.name}"
|
151
|
+
puts "Type : #{table.type}"
|
152
|
+
puts "Count : #{table.count}"
|
153
|
+
puts "Schema : ("
|
154
|
+
table.schema.fields.each {|f|
|
155
|
+
puts " #{f.name}:#{f.type}"
|
156
|
+
}
|
157
|
+
puts ")"
|
158
|
+
end
|
98
159
|
end
|
99
160
|
end
|
100
161
|
|
data/lib/td/command/td.rb
CHANGED
data/lib/td/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: td
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 11
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 7
|
9
|
-
-
|
10
|
-
version: 0.7.
|
9
|
+
- 4
|
10
|
+
version: 0.7.4
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Sadayuki Furuhashi
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2011-08-
|
18
|
+
date: 2011-08-18 00:00:00 +09:00
|
19
19
|
default_executable: td
|
20
20
|
dependencies:
|
21
21
|
- !ruby/object:Gem::Dependency
|