zillabyte-cli 0.0.13 → 0.0.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/#api.rb# +170 -0
- data/lib/zillabyte/api/flows.rb +5 -7
- data/lib/zillabyte/api/logs.rb +3 -7
- data/lib/zillabyte/api/queries.rb +13 -0
- data/lib/zillabyte/cli/flows.rb +39 -17
- data/lib/zillabyte/cli/query.rb +28 -23
- data/lib/zillabyte/cli/relations.rb +60 -57
- metadata +3 -3
- data/lib/zillabyte/cli/templates/ruby/Gemfile.lock +0 -37
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MjU1MzRkMmY2YWJmNGIxNjA0ODJmODhiNmQ1ODA5YTg5MGM4YTQyNA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
OTU2MDA5ZGM4NGFmYTc3NTAxMmUwMDY3MmM3NzljNjZlYjUwNjAzMQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MWExOTZiOWI2N2E5NTgyNTc0NmRkNjE0NDAyMDk3NDkwMzdlNjIwMjc0YjY5
|
10
|
+
NjRmNTcyZjgyODRmMzA5NmYwOTgxYjBkOTRiZjFjNjZmYTQxYWQxNTFlYjNi
|
11
|
+
YWFkMDMyNDNhYjc4ODc2ODIxNGU4OTE1ZWU1YWQ5NWVhZGM5ZGY=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NjlhYjA0YzY1OWZhMDE0NWMxNmUyYmU1ZDQyNGQ1MjNlMjQ2Mjc2MjQ4Nzlj
|
14
|
+
NTNlOWY4NDAzYTMwN2RkOWVmZGNjN2IzZjBiNmJlMDg3ODkwMzMzN2YzODE5
|
15
|
+
ZTQ2MGM5Njk3NGI2NzkwYzFkYWE3NjJkZmU0NDUzMThlZGI1MTY=
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# load('zillabyte/helpers.rb') # reload helpers after possible inject_loadpath
|
2
|
+
# load('zillabyte/updater.rb') # reload updater after possible inject_loadpath
|
3
|
+
|
4
|
+
require "base64"
|
5
|
+
require "excon"
|
6
|
+
# require "securerandom"
|
7
|
+
require "uri"
|
8
|
+
require "zlib"
|
9
|
+
require "active_support/core_ext/object/to_query.rb"
|
10
|
+
require 'json'
|
11
|
+
|
12
|
+
|
13
|
+
# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
|
14
|
+
if RUBY_VERSION =~ /^1.8.6/
|
15
|
+
# require('zillabyte-api')
|
16
|
+
require('rest_client')
|
17
|
+
end
|
18
|
+
|
19
|
+
class Zillabyte::API
|
20
|
+
|
21
|
+
HEADERS = {
|
22
|
+
'Accept' => 'application/json',
|
23
|
+
'Content-Type' => 'application/json',
|
24
|
+
# 'Accept-Encoding' => 'gzip',
|
25
|
+
'User-Agent' => "zillabyte/#{Zillabyte::CLI::VERSION}",
|
26
|
+
'X-Ruby-Version' => RUBY_VERSION,
|
27
|
+
'X-Ruby-Platform' => RUBY_PLATFORM
|
28
|
+
}
|
29
|
+
|
30
|
+
OPTIONS = {
|
31
|
+
:headers => {},
|
32
|
+
:host => ENV['ZILLABYTE_API_HOST'] || 'api.zillabyte.com',
|
33
|
+
:port => (ENV['ZILLABYTE_API_PORT'] || '80').to_i,
|
34
|
+
:nonblock => false,
|
35
|
+
:scheme => 'http'
|
36
|
+
}
|
37
|
+
|
38
|
+
|
39
|
+
|
40
|
+
def initialize(options={})
|
41
|
+
options = OPTIONS.merge(options)
|
42
|
+
|
43
|
+
@progress = options.delete(:progress)
|
44
|
+
@api_key = options.delete(:api_key) || ENV['ZILLABYTE_API_KEY']
|
45
|
+
|
46
|
+
options[:headers] = HEADERS.merge({
|
47
|
+
'Authorization' => "auth_token #{@api_key}",
|
48
|
+
}).merge(options[:headers])
|
49
|
+
|
50
|
+
@options = options.clone
|
51
|
+
@options[:bubble_exceptions] = ENV['BUBBLE_EXCEPTIONS'] || false
|
52
|
+
|
53
|
+
@connection = Excon.new("#{options[:scheme]}://#{options[:host]}", options)
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
def host
|
59
|
+
@options[:host]
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
def request(params, &block)
|
64
|
+
begin
|
65
|
+
# The @connection object handles connections to api.zillabyte.com via Excon, this sends a command to the Rails api code
|
66
|
+
# with parameters in params (one of which contains the path to the method to be run in the form of a url, e.g. if the
|
67
|
+
# :path parameter is set to /functions/#{id}?tar=1, then the show method in functions_controller.rb will be automatically
|
68
|
+
# called...it will have it's own set of parameters which correspond to the items in the query string).
|
69
|
+
response = @connection.request(params, &block)
|
70
|
+
rescue Excon::Errors::HTTPStatusError => error
|
71
|
+
@progress.error "internal server error" if @progress
|
72
|
+
raise error
|
73
|
+
end
|
74
|
+
|
75
|
+
if response.body && !response.body.empty?
|
76
|
+
# decompress_response!(response)
|
77
|
+
begin
|
78
|
+
response.body = JSON.parse(response.body)
|
79
|
+
rescue Exception => e
|
80
|
+
raise e if @options[:bubble_exceptions]
|
81
|
+
@progress.error "unknown server response: \n #{response.body}" if @progress
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Errors?
|
86
|
+
if response.body.is_a?(Hash)
|
87
|
+
if response.body["status"] == "error"
|
88
|
+
if @progress
|
89
|
+
if response.body["error"] == "authentication_error"
|
90
|
+
@progress.error "authentication error. run 'zillabyte login'"
|
91
|
+
else
|
92
|
+
@progress.error response.body["error_message"]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
# reset (non-persistent) connection
|
100
|
+
@connection.reset
|
101
|
+
|
102
|
+
response
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
def data
|
107
|
+
@_data ||= ::Zillabyte::API::Data.new(self)
|
108
|
+
end
|
109
|
+
|
110
|
+
def logs
|
111
|
+
@_logs ||= ::Zillabyte::API::Logs.new(self)
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
def queries
|
116
|
+
@_queries ||= ::Zillabyte::API::Queries.new(self)
|
117
|
+
end
|
118
|
+
alias_method :query, :queries
|
119
|
+
|
120
|
+
def flows
|
121
|
+
@_flows ||= ::Zillabyte::API::Flows.new(self)
|
122
|
+
end
|
123
|
+
alias_method :flow, :flows
|
124
|
+
|
125
|
+
def metrics
|
126
|
+
@_metrics ||= ::Zillabyte::API::Metrics.new(self)
|
127
|
+
end
|
128
|
+
alias_method :metric, :metrics
|
129
|
+
|
130
|
+
def zillalogs
|
131
|
+
@_rules ||= ::Zillabyte::API::Zillalogs.new(self)
|
132
|
+
end
|
133
|
+
alias_method :zillalog, :zillalogs
|
134
|
+
|
135
|
+
def sources
|
136
|
+
@_sources ||= ::Zillabyte::API::Sources.new(self)
|
137
|
+
end
|
138
|
+
alias_method :source, :sources
|
139
|
+
|
140
|
+
|
141
|
+
def collectors
|
142
|
+
@_collectors ||= ::Zillabyte::API::Collectors.new(self)
|
143
|
+
end
|
144
|
+
alias_method :collector, :collectors
|
145
|
+
|
146
|
+
def semantic_tags
|
147
|
+
@_rules ||= ::Zillabyte::API::SemanticTags.new(self)
|
148
|
+
end
|
149
|
+
alias_method :zillalog, :zillalogs
|
150
|
+
|
151
|
+
|
152
|
+
|
153
|
+
|
154
|
+
|
155
|
+
def self.load
|
156
|
+
Dir[File.join(File.dirname(__FILE__), "api", "*.rb")].sort.each do |file|
|
157
|
+
require file
|
158
|
+
end
|
159
|
+
|
160
|
+
Dir[File.join(File.dirname(__FILE__), "api", "helpers", "*.rb")].sort.each do |file|
|
161
|
+
require file
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
# Load all helpers...
|
170
|
+
Zillabyte::API.load()
|
data/lib/zillabyte/api/flows.rb
CHANGED
@@ -235,22 +235,20 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
|
|
235
235
|
|
236
236
|
info_file = SecureRandom.uuid
|
237
237
|
arg = "--info --file #{info_file}"
|
238
|
-
stderr = ""
|
239
|
-
stderr = " 2> /dev/null" unless options[:test]
|
240
238
|
|
241
239
|
case hash["language"]
|
242
240
|
when "ruby"
|
243
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}
|
244
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}
|
241
|
+
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
242
|
+
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
245
243
|
when "python"
|
246
244
|
if(File.directory?("#{dir}/vEnv"))
|
247
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}
|
245
|
+
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}"
|
248
246
|
else
|
249
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}
|
247
|
+
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}"
|
250
248
|
end
|
251
249
|
when "js"
|
252
250
|
# command = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" --info"
|
253
|
-
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}
|
251
|
+
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}"
|
254
252
|
else
|
255
253
|
progress.error "unsupported language: #{hash["language"]}" if progress
|
256
254
|
return nil
|
data/lib/zillabyte/api/logs.rb
CHANGED
@@ -9,16 +9,12 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
|
|
9
9
|
|
10
10
|
options = {
|
11
11
|
}.merge(options)
|
12
|
-
|
13
|
-
since = ""
|
14
|
-
if options[:since]
|
15
|
-
since = "?since=#{URI::encode(options[:since])}"
|
16
|
-
end
|
17
|
-
|
12
|
+
|
18
13
|
res = @api.request(
|
19
14
|
:expects => 200,
|
20
15
|
:method => :get,
|
21
|
-
:path => "/flows/#{flow_id}/logs/#{operation_id}
|
16
|
+
:path => "/flows/#{flow_id}/logs/#{operation_id}",
|
17
|
+
:body => options.to_json
|
22
18
|
)
|
23
19
|
|
24
20
|
res.body
|
@@ -19,6 +19,19 @@ class Zillabyte::API::Queries < Zillabyte::API::Base
|
|
19
19
|
|
20
20
|
end
|
21
21
|
|
22
|
+
def pull_to_s3(query, options)
|
23
|
+
options = {
|
24
|
+
'query' => query
|
25
|
+
}.merge(options)
|
26
|
+
|
27
|
+
res = @api.request(
|
28
|
+
:expects => 200,
|
29
|
+
:method => :post,
|
30
|
+
:path => "/query_pull_to_s3",
|
31
|
+
:body => options.to_json
|
32
|
+
)
|
33
|
+
res.body
|
34
|
+
end
|
22
35
|
|
23
36
|
|
24
37
|
# GET /sql
|
data/lib/zillabyte/cli/flows.rb
CHANGED
@@ -65,7 +65,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
65
65
|
display "Starting up your flow...please wait..."
|
66
66
|
previous_logs = {}
|
67
67
|
while true
|
68
|
-
all_logs = api.logs.get(res['id'], "_ALL_", {:
|
68
|
+
all_logs = api.logs.get(res['id'], "_ALL_", {:push => true})
|
69
69
|
all_logs.each_pair do |op, logs|
|
70
70
|
next if logs.length == 0
|
71
71
|
difference = previous_logs[op] ? logs - previous_logs[op] : logs
|
@@ -131,6 +131,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
131
131
|
# -f, --force # don't ask for confirmation
|
132
132
|
def delete
|
133
133
|
id = options[:id] || shift_argument
|
134
|
+
if id.nil?
|
135
|
+
id = read_name_from_conf(options)
|
136
|
+
options[:flow_name] = true
|
137
|
+
end
|
138
|
+
|
134
139
|
forced = options[:force]
|
135
140
|
if not forced
|
136
141
|
while true
|
@@ -145,7 +150,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
145
150
|
confirmed = forced || confirm == "yes"
|
146
151
|
|
147
152
|
if confirmed
|
148
|
-
response = api.flows.delete(id)
|
153
|
+
response = api.flows.delete(id, options)
|
149
154
|
display response["body"]
|
150
155
|
|
151
156
|
end
|
@@ -204,7 +209,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
204
209
|
#
|
205
210
|
def init
|
206
211
|
|
207
|
-
|
208
212
|
lang = options[:lang] || shift_argument || "ruby"
|
209
213
|
dir = options[:dir] || shift_argument || Dir.pwd
|
210
214
|
languages = ["ruby","python", "js"]
|
@@ -219,7 +223,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
219
223
|
|
220
224
|
|
221
225
|
|
222
|
-
#
|
226
|
+
# flows:logs FLOW_ID [OPERATION_NAME]
|
223
227
|
#
|
224
228
|
# streams logs from the distributed workers
|
225
229
|
#
|
@@ -236,9 +240,13 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
236
240
|
:category => category
|
237
241
|
}
|
238
242
|
api_options = {}
|
239
|
-
|
240
|
-
|
241
|
-
|
243
|
+
|
244
|
+
if flow_id.nil?
|
245
|
+
flow_id = read_name_from_conf(options)
|
246
|
+
api_options["flow_name"] = true
|
247
|
+
end
|
248
|
+
|
249
|
+
display "Retrieving logs for flow ##{flow_id}...please wait..."
|
242
250
|
begin
|
243
251
|
|
244
252
|
res = self.api.logs.get(flow_id, operation_id, api_options)
|
@@ -255,7 +263,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
255
263
|
alias_command "logs", "flows:logs"
|
256
264
|
|
257
265
|
|
258
|
-
#
|
266
|
+
# flows:cycles ID [OPTIONS]
|
259
267
|
#
|
260
268
|
# operations on the flow's cycles (batches).
|
261
269
|
# with no options, the command lists the flows cycles
|
@@ -265,25 +273,28 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
265
273
|
flow_id = options[:id] || shift_argument
|
266
274
|
trigger_next = options[:next] || false
|
267
275
|
trigger_forever = options[:forever] || false
|
268
|
-
|
276
|
+
|
277
|
+
if flow_id.nil?
|
278
|
+
flow_id = read_name_from_conf(options)
|
279
|
+
options[:flow_name] = true
|
280
|
+
end
|
269
281
|
|
270
282
|
if trigger_next
|
271
283
|
# Trigger the next flow
|
272
|
-
response = api.flows.create_cycle(flow_id)
|
284
|
+
response = api.flows.create_cycle(flow_id, options)
|
273
285
|
elsif trigger_forever
|
274
|
-
response = api.flows.run_forever(flow_id)
|
286
|
+
response = api.flows.run_forever(flow_id, options)
|
275
287
|
else
|
276
288
|
# List the flows
|
277
|
-
response = api.flows.list_cycles(flow_id)
|
289
|
+
response = api.flows.list_cycles(flow_id, options)
|
278
290
|
# TODO List the sequence number for this flow.
|
279
291
|
display "Most recent cyles of the flow:"
|
280
|
-
headings = ["
|
292
|
+
headings = ["Cycle_id", "State", "Start", "End"]
|
281
293
|
rows = response["cycles"]
|
282
294
|
rows = rows.map do |row|
|
283
295
|
start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
|
284
296
|
end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
|
285
|
-
|
286
|
-
[row["state"], start_time, end_time , duration ] #TODO Pretty print time
|
297
|
+
[row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
|
287
298
|
end
|
288
299
|
display Terminal::Table.new(:headings => headings, :rows => rows).to_s
|
289
300
|
display "Total number of cycles executed: #{response['total']}"
|
@@ -677,8 +688,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
677
688
|
def kill
|
678
689
|
|
679
690
|
id = options[:id] || shift_argument
|
680
|
-
|
681
|
-
|
691
|
+
if id.nil?
|
692
|
+
id = read_name_from_conf(options)
|
693
|
+
options[:flow_name] = true
|
694
|
+
end
|
695
|
+
|
696
|
+
display "Killing flow ##{id}...please wait..."
|
697
|
+
api.flows.kill(id, options)
|
698
|
+
display "Flow ##{id} killed"
|
682
699
|
|
683
700
|
end
|
684
701
|
|
@@ -888,5 +905,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
888
905
|
|
889
906
|
end
|
890
907
|
|
908
|
+
def read_name_from_conf(options = {})
|
909
|
+
hash = Zillabyte::API::Flows.get_rich_meta_info_from_script Dir.pwd, options
|
910
|
+
error "No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow." if hash["error"]
|
911
|
+
hash["name"]
|
912
|
+
end
|
891
913
|
|
892
914
|
end
|
data/lib/zillabyte/cli/query.rb
CHANGED
@@ -38,20 +38,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
38
38
|
|
39
39
|
filtered = []
|
40
40
|
headings = []
|
41
|
-
if res['
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
res['column_aliases'].each do |al|
|
46
|
-
if(al["concrete_name"] == ch)
|
47
|
-
headings << al["alias"]
|
48
|
-
has_alias = true
|
49
|
-
end
|
50
|
-
end
|
51
|
-
headings << ch if !has_alias
|
52
|
-
end
|
53
|
-
rows = res['rows'].each do |obj|
|
54
|
-
new_row = concrete_headings.map do |heading|
|
41
|
+
if res['results'].first
|
42
|
+
headings = res['results'].first.keys
|
43
|
+
rows = res['results'].each do |obj|
|
44
|
+
new_row = headings.map do |heading|
|
55
45
|
obj[heading]
|
56
46
|
end
|
57
47
|
unless seen[new_row.to_s]
|
@@ -173,17 +163,32 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
173
163
|
alias_command "sql", "query:sql"
|
174
164
|
|
175
165
|
|
176
|
-
|
177
|
-
|
178
|
-
# rules:init
|
166
|
+
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET S3_FILE_PATH
|
179
167
|
#
|
180
|
-
#
|
168
|
+
# pulls query data to S3_BUCKET/FILE_PATH/part***.gz
|
181
169
|
#
|
182
|
-
def
|
183
|
-
display "TODO: init new rule"
|
184
|
-
end
|
185
|
-
|
186
|
-
|
170
|
+
def pull_to_s3
|
187
171
|
|
172
|
+
query = options[:query] || shift_argument
|
173
|
+
error "query cannot be empty" if query.nil?
|
174
|
+
|
175
|
+
user_s3_access_key = options[:s3_access_key] || shift_argument
|
176
|
+
user_s3_secret = options[:s3_secret] || shift_argument
|
177
|
+
user_s3_bucket = options[:s3_bucket] || shift_argument
|
178
|
+
user_s3_file_path = options[:s3_file_path] || shift_argument
|
179
|
+
error "no s3 access key provided" if user_s3_access_key.nil?
|
180
|
+
error "no s3 secret provided" if user_s3_secret.nil?
|
181
|
+
error "no s3 bucket provided" if user_s3_bucket.nil?
|
182
|
+
error "no s3 file path specified" if user_s3_file_path.nil?
|
183
|
+
|
184
|
+
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
185
|
+
:s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
|
186
|
+
|
187
|
+
res = self.api.queries.pull_to_s3(query, s3_params)
|
188
|
+
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
189
|
+
display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
|
190
|
+
|
191
|
+
end
|
192
|
+
alias_command "query:pull:s3", "query:pull_to_s3"
|
188
193
|
|
189
194
|
end
|
@@ -74,52 +74,6 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
74
74
|
end
|
75
75
|
alias_command "rl:delete", "relations:delete"
|
76
76
|
|
77
|
-
# relations:status ID
|
78
|
-
#
|
79
|
-
# shows status of a relation
|
80
|
-
#
|
81
|
-
def status
|
82
|
-
|
83
|
-
name = options[:relation_name] || shift_argument
|
84
|
-
error "no name given" if name.nil?
|
85
|
-
|
86
|
-
res = api.request(
|
87
|
-
:path => "/metrics/#{name}",
|
88
|
-
:method => :get,
|
89
|
-
:expects => 200
|
90
|
-
)
|
91
|
-
|
92
|
-
if res['error']
|
93
|
-
error res['error_message']
|
94
|
-
end
|
95
|
-
|
96
|
-
data = res.body.first
|
97
|
-
|
98
|
-
if data.nil?
|
99
|
-
error "metric not found"
|
100
|
-
end
|
101
|
-
|
102
|
-
data = data["datapoints"]
|
103
|
-
|
104
|
-
last_date = data.last(30).first[1]
|
105
|
-
formatted = []
|
106
|
-
i = 0
|
107
|
-
data.each do |tup|
|
108
|
-
value = tup[0]
|
109
|
-
date = tup[1] - last_date
|
110
|
-
if date > 0
|
111
|
-
formatted << [i, value || 0]
|
112
|
-
i += 1
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
display "#{name}:"
|
117
|
-
display AsciiCharts::Cartesian.new(formatted, :bar => true, :hide_zero => true).draw
|
118
|
-
|
119
|
-
|
120
|
-
end
|
121
|
-
alias_command "rl:status", "relations:status"
|
122
|
-
|
123
77
|
|
124
78
|
# relations:create NAME
|
125
79
|
#
|
@@ -195,7 +149,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
195
149
|
#
|
196
150
|
# pulls relation data into OUTPUT.gz
|
197
151
|
#
|
198
|
-
#
|
152
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
199
153
|
#
|
200
154
|
def pull
|
201
155
|
|
@@ -204,15 +158,15 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
204
158
|
error "no id given" if id.nil?
|
205
159
|
error "no output file given" if file.nil?
|
206
160
|
file = file+".gz"
|
207
|
-
type = options[:type]
|
208
161
|
|
209
|
-
res = self.api.data.pull(id)
|
162
|
+
res = self.api.data.pull(id, options)
|
210
163
|
if(res["uri"])
|
164
|
+
display "Waiting for download."
|
211
165
|
File.open(file, "w") do |f|
|
212
166
|
f.write open(res["uri"]).read
|
213
167
|
end
|
214
168
|
elsif(res["s3_credentials"])
|
215
|
-
display "
|
169
|
+
display "Request sent. Depending on the size of your file, this may take a while."
|
216
170
|
s3 = AWS::S3.new(res["s3_credentials"])
|
217
171
|
bucket = s3.buckets[res["s3_bucket"]]
|
218
172
|
obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
|
@@ -243,7 +197,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
243
197
|
#
|
244
198
|
# pulls relation data to S3_BUCKET/FILE_PATH/part***.gz
|
245
199
|
#
|
246
|
-
#
|
200
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
247
201
|
#
|
248
202
|
def pull_to_s3
|
249
203
|
|
@@ -261,6 +215,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
261
215
|
|
262
216
|
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
263
217
|
:s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
|
218
|
+
s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
|
264
219
|
|
265
220
|
res = self.api.data.pull_to_s3(id, s3_params)
|
266
221
|
display "downloading relation data to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
@@ -275,14 +230,15 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
275
230
|
#
|
276
231
|
# shows raw data in a relation. run 'queries' for more elaborate functionality
|
277
232
|
#
|
278
|
-
# --
|
233
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
234
|
+
# --no_truncation # don't truncate long strings
|
279
235
|
#
|
280
236
|
def show
|
281
237
|
|
282
238
|
name = options[:name] || shift_argument
|
283
239
|
error "no id given" if name.nil?
|
284
240
|
|
285
|
-
res = self.api.data.get(name)
|
241
|
+
res = self.api.data.get(name, options)
|
286
242
|
|
287
243
|
if res["rows"].size > 0
|
288
244
|
headings = []
|
@@ -359,8 +315,8 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
359
315
|
description = ask.strip
|
360
316
|
end
|
361
317
|
|
362
|
-
valid_types=["
|
363
|
-
meta_names=["confidence", "since", "source"]
|
318
|
+
valid_types=["string", "integer", "float"]
|
319
|
+
meta_names=["id", "confidence", "since", "source"]
|
364
320
|
if schema.nil?
|
365
321
|
schema = ""
|
366
322
|
|
@@ -374,7 +330,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
374
330
|
cname = ask.strip
|
375
331
|
break if cname == "" || cname.nil?
|
376
332
|
if(cname =~ /^v[0-9]+$/i or meta_names.member?(cname.downcase))
|
377
|
-
warn "\"v[number]\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your column name something else."
|
333
|
+
warn "\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your column name something else."
|
378
334
|
next
|
379
335
|
end
|
380
336
|
if(already_aliased[cname])
|
@@ -383,7 +339,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
383
339
|
end
|
384
340
|
|
385
341
|
display "Type for column #{index} [#{valid_types.join(', ')}]: ", false
|
386
|
-
ctype = ask.
|
342
|
+
ctype = ask.downcase
|
387
343
|
unless valid_types.member?(ctype)
|
388
344
|
warn "invalid type: #{ctype}"
|
389
345
|
next
|
@@ -455,4 +411,51 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
455
411
|
end
|
456
412
|
rows
|
457
413
|
end
|
414
|
+
|
415
|
+
# # relations:status ID
|
416
|
+
# #
|
417
|
+
# # shows status of a relation
|
418
|
+
# #
|
419
|
+
# def status
|
420
|
+
|
421
|
+
# name = options[:relation_name] || shift_argument
|
422
|
+
# error "no name given" if name.nil?
|
423
|
+
|
424
|
+
# res = api.request(
|
425
|
+
# :path => "/metrics/#{name}",
|
426
|
+
# :method => :get,
|
427
|
+
# :expects => 200
|
428
|
+
# )
|
429
|
+
|
430
|
+
# if res['error']
|
431
|
+
# error res['error_message']
|
432
|
+
# end
|
433
|
+
|
434
|
+
# data = res.body.first
|
435
|
+
|
436
|
+
# if data.nil?
|
437
|
+
# error "metric not found"
|
438
|
+
# end
|
439
|
+
|
440
|
+
# data = data["datapoints"]
|
441
|
+
|
442
|
+
# last_date = data.last(30).first[1]
|
443
|
+
# formatted = []
|
444
|
+
# i = 0
|
445
|
+
# data.each do |tup|
|
446
|
+
# value = tup[0]
|
447
|
+
# date = tup[1] - last_date
|
448
|
+
# if date > 0
|
449
|
+
# formatted << [i, value || 0]
|
450
|
+
# i += 1
|
451
|
+
# end
|
452
|
+
# end
|
453
|
+
|
454
|
+
# display "#{name}:"
|
455
|
+
# display AsciiCharts::Cartesian.new(formatted, :bar => true, :hide_zero => true).draw
|
456
|
+
|
457
|
+
|
458
|
+
# end
|
459
|
+
# alias_command "rl:status", "relations:status"
|
460
|
+
|
458
461
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -201,6 +201,7 @@ extensions: []
|
|
201
201
|
extra_rdoc_files: []
|
202
202
|
files:
|
203
203
|
- bin/zillabyte
|
204
|
+
- lib/zillabyte/#api.rb#
|
204
205
|
- lib/zillabyte/api/base.rb
|
205
206
|
- lib/zillabyte/api/data.rb
|
206
207
|
- lib/zillabyte/api/flows.rb
|
@@ -233,7 +234,6 @@ files:
|
|
233
234
|
- lib/zillabyte/cli/templates/python/simple_function.py
|
234
235
|
- lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
|
235
236
|
- lib/zillabyte/cli/templates/ruby/Gemfile
|
236
|
-
- lib/zillabyte/cli/templates/ruby/Gemfile.lock
|
237
237
|
- lib/zillabyte/cli/templates/ruby/simple_function.rb
|
238
238
|
- lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml
|
239
239
|
- lib/zillabyte/cli/version.rb
|
@@ -1,37 +0,0 @@
|
|
1
|
-
GEM
|
2
|
-
remote: https://rubygems.org/
|
3
|
-
specs:
|
4
|
-
activesupport (3.2.17)
|
5
|
-
i18n (~> 0.6, >= 0.6.4)
|
6
|
-
multi_json (~> 1.0)
|
7
|
-
ascii_charts (0.9.1)
|
8
|
-
chronic (0.10.2)
|
9
|
-
colorize (0.6.0)
|
10
|
-
excon (0.31.0)
|
11
|
-
i18n (0.6.9)
|
12
|
-
indentation (0.1.1)
|
13
|
-
mime-types (2.1)
|
14
|
-
multi_json (1.8.4)
|
15
|
-
netrc (0.7.7)
|
16
|
-
rest-client (1.6.7)
|
17
|
-
mime-types (>= 1.16)
|
18
|
-
terminal-table (1.4.5)
|
19
|
-
zillabyte (0.0.9)
|
20
|
-
zillabyte-cli
|
21
|
-
zillabyte-cli (0.0.10)
|
22
|
-
activesupport (~> 3.2.11)
|
23
|
-
ascii_charts (~> 0.9.1)
|
24
|
-
bundler (~> 1.3)
|
25
|
-
chronic (~> 0.10)
|
26
|
-
colorize (~> 0.6)
|
27
|
-
excon (~> 0.31)
|
28
|
-
indentation (~> 0.1)
|
29
|
-
netrc (~> 0.7.7)
|
30
|
-
rest-client (~> 1.6.1)
|
31
|
-
terminal-table (~> 1.4)
|
32
|
-
|
33
|
-
PLATFORMS
|
34
|
-
ruby
|
35
|
-
|
36
|
-
DEPENDENCIES
|
37
|
-
zillabyte
|