zillabyte-cli 0.0.13 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/#api.rb# +170 -0
- data/lib/zillabyte/api/flows.rb +5 -7
- data/lib/zillabyte/api/logs.rb +3 -7
- data/lib/zillabyte/api/queries.rb +13 -0
- data/lib/zillabyte/cli/flows.rb +39 -17
- data/lib/zillabyte/cli/query.rb +28 -23
- data/lib/zillabyte/cli/relations.rb +60 -57
- metadata +3 -3
- data/lib/zillabyte/cli/templates/ruby/Gemfile.lock +0 -37
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MjU1MzRkMmY2YWJmNGIxNjA0ODJmODhiNmQ1ODA5YTg5MGM4YTQyNA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
OTU2MDA5ZGM4NGFmYTc3NTAxMmUwMDY3MmM3NzljNjZlYjUwNjAzMQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MWExOTZiOWI2N2E5NTgyNTc0NmRkNjE0NDAyMDk3NDkwMzdlNjIwMjc0YjY5
|
10
|
+
NjRmNTcyZjgyODRmMzA5NmYwOTgxYjBkOTRiZjFjNjZmYTQxYWQxNTFlYjNi
|
11
|
+
YWFkMDMyNDNhYjc4ODc2ODIxNGU4OTE1ZWU1YWQ5NWVhZGM5ZGY=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NjlhYjA0YzY1OWZhMDE0NWMxNmUyYmU1ZDQyNGQ1MjNlMjQ2Mjc2MjQ4Nzlj
|
14
|
+
NTNlOWY4NDAzYTMwN2RkOWVmZGNjN2IzZjBiNmJlMDg3ODkwMzMzN2YzODE5
|
15
|
+
ZTQ2MGM5Njk3NGI2NzkwYzFkYWE3NjJkZmU0NDUzMThlZGI1MTY=
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# load('zillabyte/helpers.rb') # reload helpers after possible inject_loadpath
|
2
|
+
# load('zillabyte/updater.rb') # reload updater after possible inject_loadpath
|
3
|
+
|
4
|
+
require "base64"
|
5
|
+
require "excon"
|
6
|
+
# require "securerandom"
|
7
|
+
require "uri"
|
8
|
+
require "zlib"
|
9
|
+
require "active_support/core_ext/object/to_query.rb"
|
10
|
+
require 'json'
|
11
|
+
|
12
|
+
|
13
|
+
# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
|
14
|
+
if RUBY_VERSION =~ /^1.8.6/
|
15
|
+
# require('zillabyte-api')
|
16
|
+
require('rest_client')
|
17
|
+
end
|
18
|
+
|
19
|
+
class Zillabyte::API
|
20
|
+
|
21
|
+
HEADERS = {
|
22
|
+
'Accept' => 'application/json',
|
23
|
+
'Content-Type' => 'application/json',
|
24
|
+
# 'Accept-Encoding' => 'gzip',
|
25
|
+
'User-Agent' => "zillabyte/#{Zillabyte::CLI::VERSION}",
|
26
|
+
'X-Ruby-Version' => RUBY_VERSION,
|
27
|
+
'X-Ruby-Platform' => RUBY_PLATFORM
|
28
|
+
}
|
29
|
+
|
30
|
+
OPTIONS = {
|
31
|
+
:headers => {},
|
32
|
+
:host => ENV['ZILLABYTE_API_HOST'] || 'api.zillabyte.com',
|
33
|
+
:port => (ENV['ZILLABYTE_API_PORT'] || '80').to_i,
|
34
|
+
:nonblock => false,
|
35
|
+
:scheme => 'http'
|
36
|
+
}
|
37
|
+
|
38
|
+
|
39
|
+
|
40
|
+
def initialize(options={})
|
41
|
+
options = OPTIONS.merge(options)
|
42
|
+
|
43
|
+
@progress = options.delete(:progress)
|
44
|
+
@api_key = options.delete(:api_key) || ENV['ZILLABYTE_API_KEY']
|
45
|
+
|
46
|
+
options[:headers] = HEADERS.merge({
|
47
|
+
'Authorization' => "auth_token #{@api_key}",
|
48
|
+
}).merge(options[:headers])
|
49
|
+
|
50
|
+
@options = options.clone
|
51
|
+
@options[:bubble_exceptions] = ENV['BUBBLE_EXCEPTIONS'] || false
|
52
|
+
|
53
|
+
@connection = Excon.new("#{options[:scheme]}://#{options[:host]}", options)
|
54
|
+
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
def host
|
59
|
+
@options[:host]
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
def request(params, &block)
|
64
|
+
begin
|
65
|
+
# The @connection object handles connections to api.zillabyte.com via Excon, this sends a command to the Rails api code
|
66
|
+
# with parameters in params (one of which contains the path to the method to be run in the form of a url, e.g. if the
|
67
|
+
# :path parameter is set to /functions/#{id}?tar=1, then the show method in functions_controller.rb will be automatically
|
68
|
+
# called...it will have it's own set of parameters which correspond to the items in the query string).
|
69
|
+
response = @connection.request(params, &block)
|
70
|
+
rescue Excon::Errors::HTTPStatusError => error
|
71
|
+
@progress.error "internal server error" if @progress
|
72
|
+
raise error
|
73
|
+
end
|
74
|
+
|
75
|
+
if response.body && !response.body.empty?
|
76
|
+
# decompress_response!(response)
|
77
|
+
begin
|
78
|
+
response.body = JSON.parse(response.body)
|
79
|
+
rescue Exception => e
|
80
|
+
raise e if @options[:bubble_exceptions]
|
81
|
+
@progress.error "unknown server response: \n #{response.body}" if @progress
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Errors?
|
86
|
+
if response.body.is_a?(Hash)
|
87
|
+
if response.body["status"] == "error"
|
88
|
+
if @progress
|
89
|
+
if response.body["error"] == "authentication_error"
|
90
|
+
@progress.error "authentication error. run 'zillabyte login'"
|
91
|
+
else
|
92
|
+
@progress.error response.body["error_message"]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
# reset (non-persistent) connection
|
100
|
+
@connection.reset
|
101
|
+
|
102
|
+
response
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
def data
|
107
|
+
@_data ||= ::Zillabyte::API::Data.new(self)
|
108
|
+
end
|
109
|
+
|
110
|
+
def logs
|
111
|
+
@_logs ||= ::Zillabyte::API::Logs.new(self)
|
112
|
+
end
|
113
|
+
|
114
|
+
|
115
|
+
def queries
|
116
|
+
@_queries ||= ::Zillabyte::API::Queries.new(self)
|
117
|
+
end
|
118
|
+
alias_method :query, :queries
|
119
|
+
|
120
|
+
def flows
|
121
|
+
@_flows ||= ::Zillabyte::API::Flows.new(self)
|
122
|
+
end
|
123
|
+
alias_method :flow, :flows
|
124
|
+
|
125
|
+
def metrics
|
126
|
+
@_metrics ||= ::Zillabyte::API::Metrics.new(self)
|
127
|
+
end
|
128
|
+
alias_method :metric, :metrics
|
129
|
+
|
130
|
+
def zillalogs
|
131
|
+
@_rules ||= ::Zillabyte::API::Zillalogs.new(self)
|
132
|
+
end
|
133
|
+
alias_method :zillalog, :zillalogs
|
134
|
+
|
135
|
+
def sources
|
136
|
+
@_sources ||= ::Zillabyte::API::Sources.new(self)
|
137
|
+
end
|
138
|
+
alias_method :source, :sources
|
139
|
+
|
140
|
+
|
141
|
+
def collectors
|
142
|
+
@_collectors ||= ::Zillabyte::API::Collectors.new(self)
|
143
|
+
end
|
144
|
+
alias_method :collector, :collectors
|
145
|
+
|
146
|
+
def semantic_tags
|
147
|
+
@_rules ||= ::Zillabyte::API::SemanticTags.new(self)
|
148
|
+
end
|
149
|
+
alias_method :zillalog, :zillalogs
|
150
|
+
|
151
|
+
|
152
|
+
|
153
|
+
|
154
|
+
|
155
|
+
def self.load
|
156
|
+
Dir[File.join(File.dirname(__FILE__), "api", "*.rb")].sort.each do |file|
|
157
|
+
require file
|
158
|
+
end
|
159
|
+
|
160
|
+
Dir[File.join(File.dirname(__FILE__), "api", "helpers", "*.rb")].sort.each do |file|
|
161
|
+
require file
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
|
166
|
+
end
|
167
|
+
|
168
|
+
|
169
|
+
# Load all helpers...
|
170
|
+
Zillabyte::API.load()
|
data/lib/zillabyte/api/flows.rb
CHANGED
@@ -235,22 +235,20 @@ class Zillabyte::API::Flows < Zillabyte::API::Base
|
|
235
235
|
|
236
236
|
info_file = SecureRandom.uuid
|
237
237
|
arg = "--info --file #{info_file}"
|
238
|
-
stderr = ""
|
239
|
-
stderr = " 2> /dev/null" unless options[:test]
|
240
238
|
|
241
239
|
case hash["language"]
|
242
240
|
when "ruby"
|
243
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}
|
244
|
-
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}
|
241
|
+
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
242
|
+
command = "cd \"#{dir}\"; unset BUNDLE_GEMFILE; ZILLABYTE_HARNESS=1 bundle exec ruby \"#{full_script}\" #{arg}"
|
245
243
|
when "python"
|
246
244
|
if(File.directory?("#{dir}/vEnv"))
|
247
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}
|
245
|
+
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte #{dir}/vEnv/bin/python \"#{full_script}\" #{arg}"
|
248
246
|
else
|
249
|
-
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}
|
247
|
+
command = "cd \"#{dir}\"; PYTHONPATH=~/zb1/multilang/python/Zillabyte python \"#{full_script}\" #{arg}"
|
250
248
|
end
|
251
249
|
when "js"
|
252
250
|
# command = "#{Zillabyte::API::CASPERJS_BIN} #{Zillabyte::API::API_CLIENT_JS} \"#{full_script}\" --info"
|
253
|
-
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}
|
251
|
+
command = "cd \"#{dir}\"; NODE_PATH=~/zb1/multilang/js/src/lib #{Zillabyte::API::NODEJS_BIN} \"#{full_script}\" #{arg}"
|
254
252
|
else
|
255
253
|
progress.error "unsupported language: #{hash["language"]}" if progress
|
256
254
|
return nil
|
data/lib/zillabyte/api/logs.rb
CHANGED
@@ -9,16 +9,12 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
|
|
9
9
|
|
10
10
|
options = {
|
11
11
|
}.merge(options)
|
12
|
-
|
13
|
-
since = ""
|
14
|
-
if options[:since]
|
15
|
-
since = "?since=#{URI::encode(options[:since])}"
|
16
|
-
end
|
17
|
-
|
12
|
+
|
18
13
|
res = @api.request(
|
19
14
|
:expects => 200,
|
20
15
|
:method => :get,
|
21
|
-
:path => "/flows/#{flow_id}/logs/#{operation_id}
|
16
|
+
:path => "/flows/#{flow_id}/logs/#{operation_id}",
|
17
|
+
:body => options.to_json
|
22
18
|
)
|
23
19
|
|
24
20
|
res.body
|
@@ -19,6 +19,19 @@ class Zillabyte::API::Queries < Zillabyte::API::Base
|
|
19
19
|
|
20
20
|
end
|
21
21
|
|
22
|
+
def pull_to_s3(query, options)
|
23
|
+
options = {
|
24
|
+
'query' => query
|
25
|
+
}.merge(options)
|
26
|
+
|
27
|
+
res = @api.request(
|
28
|
+
:expects => 200,
|
29
|
+
:method => :post,
|
30
|
+
:path => "/query_pull_to_s3",
|
31
|
+
:body => options.to_json
|
32
|
+
)
|
33
|
+
res.body
|
34
|
+
end
|
22
35
|
|
23
36
|
|
24
37
|
# GET /sql
|
data/lib/zillabyte/cli/flows.rb
CHANGED
@@ -65,7 +65,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
65
65
|
display "Starting up your flow...please wait..."
|
66
66
|
previous_logs = {}
|
67
67
|
while true
|
68
|
-
all_logs = api.logs.get(res['id'], "_ALL_", {:
|
68
|
+
all_logs = api.logs.get(res['id'], "_ALL_", {:push => true})
|
69
69
|
all_logs.each_pair do |op, logs|
|
70
70
|
next if logs.length == 0
|
71
71
|
difference = previous_logs[op] ? logs - previous_logs[op] : logs
|
@@ -131,6 +131,11 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
131
131
|
# -f, --force # don't ask for confirmation
|
132
132
|
def delete
|
133
133
|
id = options[:id] || shift_argument
|
134
|
+
if id.nil?
|
135
|
+
id = read_name_from_conf(options)
|
136
|
+
options[:flow_name] = true
|
137
|
+
end
|
138
|
+
|
134
139
|
forced = options[:force]
|
135
140
|
if not forced
|
136
141
|
while true
|
@@ -145,7 +150,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
145
150
|
confirmed = forced || confirm == "yes"
|
146
151
|
|
147
152
|
if confirmed
|
148
|
-
response = api.flows.delete(id)
|
153
|
+
response = api.flows.delete(id, options)
|
149
154
|
display response["body"]
|
150
155
|
|
151
156
|
end
|
@@ -204,7 +209,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
204
209
|
#
|
205
210
|
def init
|
206
211
|
|
207
|
-
|
208
212
|
lang = options[:lang] || shift_argument || "ruby"
|
209
213
|
dir = options[:dir] || shift_argument || Dir.pwd
|
210
214
|
languages = ["ruby","python", "js"]
|
@@ -219,7 +223,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
219
223
|
|
220
224
|
|
221
225
|
|
222
|
-
#
|
226
|
+
# flows:logs FLOW_ID [OPERATION_NAME]
|
223
227
|
#
|
224
228
|
# streams logs from the distributed workers
|
225
229
|
#
|
@@ -236,9 +240,13 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
236
240
|
:category => category
|
237
241
|
}
|
238
242
|
api_options = {}
|
239
|
-
|
240
|
-
|
241
|
-
|
243
|
+
|
244
|
+
if flow_id.nil?
|
245
|
+
flow_id = read_name_from_conf(options)
|
246
|
+
api_options["flow_name"] = true
|
247
|
+
end
|
248
|
+
|
249
|
+
display "Retrieving logs for flow ##{flow_id}...please wait..."
|
242
250
|
begin
|
243
251
|
|
244
252
|
res = self.api.logs.get(flow_id, operation_id, api_options)
|
@@ -255,7 +263,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
255
263
|
alias_command "logs", "flows:logs"
|
256
264
|
|
257
265
|
|
258
|
-
#
|
266
|
+
# flows:cycles ID [OPTIONS]
|
259
267
|
#
|
260
268
|
# operations on the flow's cycles (batches).
|
261
269
|
# with no options, the command lists the flows cycles
|
@@ -265,25 +273,28 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
265
273
|
flow_id = options[:id] || shift_argument
|
266
274
|
trigger_next = options[:next] || false
|
267
275
|
trigger_forever = options[:forever] || false
|
268
|
-
|
276
|
+
|
277
|
+
if flow_id.nil?
|
278
|
+
flow_id = read_name_from_conf(options)
|
279
|
+
options[:flow_name] = true
|
280
|
+
end
|
269
281
|
|
270
282
|
if trigger_next
|
271
283
|
# Trigger the next flow
|
272
|
-
response = api.flows.create_cycle(flow_id)
|
284
|
+
response = api.flows.create_cycle(flow_id, options)
|
273
285
|
elsif trigger_forever
|
274
|
-
response = api.flows.run_forever(flow_id)
|
286
|
+
response = api.flows.run_forever(flow_id, options)
|
275
287
|
else
|
276
288
|
# List the flows
|
277
|
-
response = api.flows.list_cycles(flow_id)
|
289
|
+
response = api.flows.list_cycles(flow_id, options)
|
278
290
|
# TODO List the sequence number for this flow.
|
279
291
|
display "Most recent cyles of the flow:"
|
280
|
-
headings = ["
|
292
|
+
headings = ["Cycle_id", "State", "Start", "End"]
|
281
293
|
rows = response["cycles"]
|
282
294
|
rows = rows.map do |row|
|
283
295
|
start_time = DateTime.parse(row["start"]).strftime("%m/%d/%Y %I:%M%p")
|
284
296
|
end_time = row["end"].nil? ? "---" : DateTime.parse(row["end"]).strftime("%m/%d/%Y %I:%M%p")
|
285
|
-
|
286
|
-
[row["state"], start_time, end_time , duration ] #TODO Pretty print time
|
297
|
+
[row["cycle_id"], row["state"], start_time, end_time ] #TODO Pretty print time
|
287
298
|
end
|
288
299
|
display Terminal::Table.new(:headings => headings, :rows => rows).to_s
|
289
300
|
display "Total number of cycles executed: #{response['total']}"
|
@@ -677,8 +688,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
677
688
|
def kill
|
678
689
|
|
679
690
|
id = options[:id] || shift_argument
|
680
|
-
|
681
|
-
|
691
|
+
if id.nil?
|
692
|
+
id = read_name_from_conf(options)
|
693
|
+
options[:flow_name] = true
|
694
|
+
end
|
695
|
+
|
696
|
+
display "Killing flow ##{id}...please wait..."
|
697
|
+
api.flows.kill(id, options)
|
698
|
+
display "Flow ##{id} killed"
|
682
699
|
|
683
700
|
end
|
684
701
|
|
@@ -888,5 +905,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
888
905
|
|
889
906
|
end
|
890
907
|
|
908
|
+
def read_name_from_conf(options = {})
|
909
|
+
hash = Zillabyte::API::Flows.get_rich_meta_info_from_script Dir.pwd, options
|
910
|
+
error "No id given and current directory does not contain a valid Zillabyte configuration file. Please specify a flow id or run command from the directory containing the flow." if hash["error"]
|
911
|
+
hash["name"]
|
912
|
+
end
|
891
913
|
|
892
914
|
end
|
data/lib/zillabyte/cli/query.rb
CHANGED
@@ -38,20 +38,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
38
38
|
|
39
39
|
filtered = []
|
40
40
|
headings = []
|
41
|
-
if res['
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
res['column_aliases'].each do |al|
|
46
|
-
if(al["concrete_name"] == ch)
|
47
|
-
headings << al["alias"]
|
48
|
-
has_alias = true
|
49
|
-
end
|
50
|
-
end
|
51
|
-
headings << ch if !has_alias
|
52
|
-
end
|
53
|
-
rows = res['rows'].each do |obj|
|
54
|
-
new_row = concrete_headings.map do |heading|
|
41
|
+
if res['results'].first
|
42
|
+
headings = res['results'].first.keys
|
43
|
+
rows = res['results'].each do |obj|
|
44
|
+
new_row = headings.map do |heading|
|
55
45
|
obj[heading]
|
56
46
|
end
|
57
47
|
unless seen[new_row.to_s]
|
@@ -173,17 +163,32 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
173
163
|
alias_command "sql", "query:sql"
|
174
164
|
|
175
165
|
|
176
|
-
|
177
|
-
|
178
|
-
# rules:init
|
166
|
+
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET S3_FILE_PATH
|
179
167
|
#
|
180
|
-
#
|
168
|
+
# pulls query data to S3_BUCKET/FILE_PATH/part***.gz
|
181
169
|
#
|
182
|
-
def
|
183
|
-
display "TODO: init new rule"
|
184
|
-
end
|
185
|
-
|
186
|
-
|
170
|
+
def pull_to_s3
|
187
171
|
|
172
|
+
query = options[:query] || shift_argument
|
173
|
+
error "query cannot be empty" if query.nil?
|
174
|
+
|
175
|
+
user_s3_access_key = options[:s3_access_key] || shift_argument
|
176
|
+
user_s3_secret = options[:s3_secret] || shift_argument
|
177
|
+
user_s3_bucket = options[:s3_bucket] || shift_argument
|
178
|
+
user_s3_file_path = options[:s3_file_path] || shift_argument
|
179
|
+
error "no s3 access key provided" if user_s3_access_key.nil?
|
180
|
+
error "no s3 secret provided" if user_s3_secret.nil?
|
181
|
+
error "no s3 bucket provided" if user_s3_bucket.nil?
|
182
|
+
error "no s3 file path specified" if user_s3_file_path.nil?
|
183
|
+
|
184
|
+
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
185
|
+
:s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
|
186
|
+
|
187
|
+
res = self.api.queries.pull_to_s3(query, s3_params)
|
188
|
+
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
189
|
+
display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
|
190
|
+
|
191
|
+
end
|
192
|
+
alias_command "query:pull:s3", "query:pull_to_s3"
|
188
193
|
|
189
194
|
end
|
@@ -74,52 +74,6 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
74
74
|
end
|
75
75
|
alias_command "rl:delete", "relations:delete"
|
76
76
|
|
77
|
-
# relations:status ID
|
78
|
-
#
|
79
|
-
# shows status of a relation
|
80
|
-
#
|
81
|
-
def status
|
82
|
-
|
83
|
-
name = options[:relation_name] || shift_argument
|
84
|
-
error "no name given" if name.nil?
|
85
|
-
|
86
|
-
res = api.request(
|
87
|
-
:path => "/metrics/#{name}",
|
88
|
-
:method => :get,
|
89
|
-
:expects => 200
|
90
|
-
)
|
91
|
-
|
92
|
-
if res['error']
|
93
|
-
error res['error_message']
|
94
|
-
end
|
95
|
-
|
96
|
-
data = res.body.first
|
97
|
-
|
98
|
-
if data.nil?
|
99
|
-
error "metric not found"
|
100
|
-
end
|
101
|
-
|
102
|
-
data = data["datapoints"]
|
103
|
-
|
104
|
-
last_date = data.last(30).first[1]
|
105
|
-
formatted = []
|
106
|
-
i = 0
|
107
|
-
data.each do |tup|
|
108
|
-
value = tup[0]
|
109
|
-
date = tup[1] - last_date
|
110
|
-
if date > 0
|
111
|
-
formatted << [i, value || 0]
|
112
|
-
i += 1
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
display "#{name}:"
|
117
|
-
display AsciiCharts::Cartesian.new(formatted, :bar => true, :hide_zero => true).draw
|
118
|
-
|
119
|
-
|
120
|
-
end
|
121
|
-
alias_command "rl:status", "relations:status"
|
122
|
-
|
123
77
|
|
124
78
|
# relations:create NAME
|
125
79
|
#
|
@@ -195,7 +149,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
195
149
|
#
|
196
150
|
# pulls relation data into OUTPUT.gz
|
197
151
|
#
|
198
|
-
#
|
152
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
199
153
|
#
|
200
154
|
def pull
|
201
155
|
|
@@ -204,15 +158,15 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
204
158
|
error "no id given" if id.nil?
|
205
159
|
error "no output file given" if file.nil?
|
206
160
|
file = file+".gz"
|
207
|
-
type = options[:type]
|
208
161
|
|
209
|
-
res = self.api.data.pull(id)
|
162
|
+
res = self.api.data.pull(id, options)
|
210
163
|
if(res["uri"])
|
164
|
+
display "Waiting for download."
|
211
165
|
File.open(file, "w") do |f|
|
212
166
|
f.write open(res["uri"]).read
|
213
167
|
end
|
214
168
|
elsif(res["s3_credentials"])
|
215
|
-
display "
|
169
|
+
display "Request sent. Depending on the size of your file, this may take a while."
|
216
170
|
s3 = AWS::S3.new(res["s3_credentials"])
|
217
171
|
bucket = s3.buckets[res["s3_bucket"]]
|
218
172
|
obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
|
@@ -243,7 +197,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
243
197
|
#
|
244
198
|
# pulls relation data to S3_BUCKET/FILE_PATH/part***.gz
|
245
199
|
#
|
246
|
-
#
|
200
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
247
201
|
#
|
248
202
|
def pull_to_s3
|
249
203
|
|
@@ -261,6 +215,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
261
215
|
|
262
216
|
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
263
217
|
:s3_bucket => user_s3_bucket, :s3_object_key => user_s3_file_path}
|
218
|
+
s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
|
264
219
|
|
265
220
|
res = self.api.data.pull_to_s3(id, s3_params)
|
266
221
|
display "downloading relation data to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
@@ -275,14 +230,15 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
275
230
|
#
|
276
231
|
# shows raw data in a relation. run 'queries' for more elaborate functionality
|
277
232
|
#
|
278
|
-
# --
|
233
|
+
# --cycle_id [cycle_id] # if the relation is associated with a flow, cycle_id indicates the flow cycle to retrieve data for (defaults to the last cycle)
|
234
|
+
# --no_truncation # don't truncate long strings
|
279
235
|
#
|
280
236
|
def show
|
281
237
|
|
282
238
|
name = options[:name] || shift_argument
|
283
239
|
error "no id given" if name.nil?
|
284
240
|
|
285
|
-
res = self.api.data.get(name)
|
241
|
+
res = self.api.data.get(name, options)
|
286
242
|
|
287
243
|
if res["rows"].size > 0
|
288
244
|
headings = []
|
@@ -359,8 +315,8 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
359
315
|
description = ask.strip
|
360
316
|
end
|
361
317
|
|
362
|
-
valid_types=["
|
363
|
-
meta_names=["confidence", "since", "source"]
|
318
|
+
valid_types=["string", "integer", "float"]
|
319
|
+
meta_names=["id", "confidence", "since", "source"]
|
364
320
|
if schema.nil?
|
365
321
|
schema = ""
|
366
322
|
|
@@ -374,7 +330,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
374
330
|
cname = ask.strip
|
375
331
|
break if cname == "" || cname.nil?
|
376
332
|
if(cname =~ /^v[0-9]+$/i or meta_names.member?(cname.downcase))
|
377
|
-
warn "\"v[number]\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your column name something else."
|
333
|
+
warn "\"v[number]\", \"id\", \"confidence\", \"since\" and \"source\" are special names in Zillabyte. Please name your column name something else."
|
378
334
|
next
|
379
335
|
end
|
380
336
|
if(already_aliased[cname])
|
@@ -383,7 +339,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
383
339
|
end
|
384
340
|
|
385
341
|
display "Type for column #{index} [#{valid_types.join(', ')}]: ", false
|
386
|
-
ctype = ask.
|
342
|
+
ctype = ask.downcase
|
387
343
|
unless valid_types.member?(ctype)
|
388
344
|
warn "invalid type: #{ctype}"
|
389
345
|
next
|
@@ -455,4 +411,51 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
455
411
|
end
|
456
412
|
rows
|
457
413
|
end
|
414
|
+
|
415
|
+
# # relations:status ID
|
416
|
+
# #
|
417
|
+
# # shows status of a relation
|
418
|
+
# #
|
419
|
+
# def status
|
420
|
+
|
421
|
+
# name = options[:relation_name] || shift_argument
|
422
|
+
# error "no name given" if name.nil?
|
423
|
+
|
424
|
+
# res = api.request(
|
425
|
+
# :path => "/metrics/#{name}",
|
426
|
+
# :method => :get,
|
427
|
+
# :expects => 200
|
428
|
+
# )
|
429
|
+
|
430
|
+
# if res['error']
|
431
|
+
# error res['error_message']
|
432
|
+
# end
|
433
|
+
|
434
|
+
# data = res.body.first
|
435
|
+
|
436
|
+
# if data.nil?
|
437
|
+
# error "metric not found"
|
438
|
+
# end
|
439
|
+
|
440
|
+
# data = data["datapoints"]
|
441
|
+
|
442
|
+
# last_date = data.last(30).first[1]
|
443
|
+
# formatted = []
|
444
|
+
# i = 0
|
445
|
+
# data.each do |tup|
|
446
|
+
# value = tup[0]
|
447
|
+
# date = tup[1] - last_date
|
448
|
+
# if date > 0
|
449
|
+
# formatted << [i, value || 0]
|
450
|
+
# i += 1
|
451
|
+
# end
|
452
|
+
# end
|
453
|
+
|
454
|
+
# display "#{name}:"
|
455
|
+
# display AsciiCharts::Cartesian.new(formatted, :bar => true, :hide_zero => true).draw
|
456
|
+
|
457
|
+
|
458
|
+
# end
|
459
|
+
# alias_command "rl:status", "relations:status"
|
460
|
+
|
458
461
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-03-
|
11
|
+
date: 2014-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -201,6 +201,7 @@ extensions: []
|
|
201
201
|
extra_rdoc_files: []
|
202
202
|
files:
|
203
203
|
- bin/zillabyte
|
204
|
+
- lib/zillabyte/#api.rb#
|
204
205
|
- lib/zillabyte/api/base.rb
|
205
206
|
- lib/zillabyte/api/data.rb
|
206
207
|
- lib/zillabyte/api/flows.rb
|
@@ -233,7 +234,6 @@ files:
|
|
233
234
|
- lib/zillabyte/cli/templates/python/simple_function.py
|
234
235
|
- lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
|
235
236
|
- lib/zillabyte/cli/templates/ruby/Gemfile
|
236
|
-
- lib/zillabyte/cli/templates/ruby/Gemfile.lock
|
237
237
|
- lib/zillabyte/cli/templates/ruby/simple_function.rb
|
238
238
|
- lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml
|
239
239
|
- lib/zillabyte/cli/version.rb
|
@@ -1,37 +0,0 @@
|
|
1
|
-
GEM
|
2
|
-
remote: https://rubygems.org/
|
3
|
-
specs:
|
4
|
-
activesupport (3.2.17)
|
5
|
-
i18n (~> 0.6, >= 0.6.4)
|
6
|
-
multi_json (~> 1.0)
|
7
|
-
ascii_charts (0.9.1)
|
8
|
-
chronic (0.10.2)
|
9
|
-
colorize (0.6.0)
|
10
|
-
excon (0.31.0)
|
11
|
-
i18n (0.6.9)
|
12
|
-
indentation (0.1.1)
|
13
|
-
mime-types (2.1)
|
14
|
-
multi_json (1.8.4)
|
15
|
-
netrc (0.7.7)
|
16
|
-
rest-client (1.6.7)
|
17
|
-
mime-types (>= 1.16)
|
18
|
-
terminal-table (1.4.5)
|
19
|
-
zillabyte (0.0.9)
|
20
|
-
zillabyte-cli
|
21
|
-
zillabyte-cli (0.0.10)
|
22
|
-
activesupport (~> 3.2.11)
|
23
|
-
ascii_charts (~> 0.9.1)
|
24
|
-
bundler (~> 1.3)
|
25
|
-
chronic (~> 0.10)
|
26
|
-
colorize (~> 0.6)
|
27
|
-
excon (~> 0.31)
|
28
|
-
indentation (~> 0.1)
|
29
|
-
netrc (~> 0.7.7)
|
30
|
-
rest-client (~> 1.6.1)
|
31
|
-
terminal-table (~> 1.4)
|
32
|
-
|
33
|
-
PLATFORMS
|
34
|
-
ruby
|
35
|
-
|
36
|
-
DEPENDENCIES
|
37
|
-
zillabyte
|