zillabyte-cli 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +9 -9
- data/lib/#zillabyte-cli.rb# +5 -0
- data/lib/zillabyte/api/{flows.rb → apps.rb} +35 -23
- data/lib/zillabyte/api/data.rb +12 -1
- data/lib/zillabyte/api/logs.rb +4 -4
- data/lib/zillabyte/api/queries.rb +15 -6
- data/lib/zillabyte/api/zillalogs.rb +1 -1
- data/lib/zillabyte/api.rb +40 -38
- data/lib/zillabyte/auth.rb +19 -11
- data/lib/zillabyte/cli/#logs.rb# +12 -0
- data/lib/zillabyte/cli/{flows.rb → apps.rb} +407 -177
- data/lib/zillabyte/cli/auth.rb +1 -1
- data/lib/zillabyte/cli/base.rb +5 -4
- data/lib/zillabyte/cli/config.rb +3 -2
- data/lib/zillabyte/cli/counters.rb +1 -1
- data/lib/zillabyte/cli/help.rb +1 -1
- data/lib/zillabyte/cli/helpers/data_schema_builder.rb +1 -1
- data/lib/zillabyte/cli/helpers/table_output_builder.rb +25 -0
- data/lib/zillabyte/cli/log_formatter.rb +4 -3
- data/lib/zillabyte/cli/query.rb +107 -26
- data/lib/zillabyte/cli/relations.rb +226 -78
- data/lib/zillabyte/cli/sources.rb +1 -1
- data/lib/zillabyte/cli/templates/js/simple_function.js +5 -0
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +27 -0
- data/lib/zillabyte/cli/templates/python/simple_function.py +3 -0
- data/lib/zillabyte/cli/templates/ruby/{simple_function.rb → simple_app.rb} +6 -6
- data/lib/zillabyte/cli/templates/ruby/zillabyte.conf.yaml +1 -1
- data/lib/zillabyte/cli/version.rb +1 -1
- data/lib/zillabyte/cli/zillalogs.rb +1 -1
- data/lib/zillabyte/command.rb +10 -2
- data/lib/zillabyte/common/{progress.rb → session.rb} +1 -1
- data/lib/zillabyte/helpers.rb +9 -4
- data/lib/zillabyte-cli/version.rb +1 -1
- data/zillabyte-cli.gemspec +2 -0
- metadata +25 -7
data/lib/zillabyte/cli/auth.rb
CHANGED
data/lib/zillabyte/cli/base.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
1
|
require "fileutils"
|
2
2
|
require "zillabyte/api"
|
3
3
|
require "zillabyte/command"
|
4
|
+
require "action_view"
|
4
5
|
|
5
6
|
class Zillabyte::Command::Base
|
6
7
|
include Zillabyte::Helpers
|
8
|
+
include ActionView::Helpers::DateHelper
|
7
9
|
|
8
10
|
def self.namespace
|
9
11
|
self.to_s.split("::").last.downcase
|
@@ -56,7 +58,6 @@ protected
|
|
56
58
|
alias_command command.gsub(/_/, '-'), command if command =~ /_/
|
57
59
|
end
|
58
60
|
|
59
|
-
|
60
61
|
|
61
62
|
#
|
62
63
|
# Parse the caller format and identify the file and line number as identified
|
@@ -140,12 +141,12 @@ protected
|
|
140
141
|
end
|
141
142
|
|
142
143
|
def api
|
143
|
-
@__api ||= Zillabyte::API.new(:api_key => Zillabyte::Auth.api_key, :
|
144
|
+
@__api ||= Zillabyte::API.new(:api_key => Zillabyte::Auth.api_key, :session => self)
|
144
145
|
end
|
145
146
|
|
146
147
|
|
147
148
|
|
148
|
-
def
|
149
|
+
def session
|
149
150
|
self
|
150
151
|
end
|
151
152
|
|
@@ -158,4 +159,4 @@ module Zillabyte::Command
|
|
158
159
|
unless const_defined?(:BaseWithApp)
|
159
160
|
BaseWithApp = Base
|
160
161
|
end
|
161
|
-
end
|
162
|
+
end
|
data/lib/zillabyte/cli/config.rb
CHANGED
@@ -10,7 +10,7 @@ module Zillabyte
|
|
10
10
|
|
11
11
|
################################################################################
|
12
12
|
|
13
|
-
def self.get_config_info(dir,
|
13
|
+
def self.get_config_info(dir, session = nil, options = {})
|
14
14
|
|
15
15
|
conf_file = nil
|
16
16
|
if options[:config_file] and File.exists?(options[:config_file])
|
@@ -18,6 +18,7 @@ module Zillabyte
|
|
18
18
|
else
|
19
19
|
conf_file = File.join(dir, options[:config_file] || DEFAULT_CONFIG_FILE)
|
20
20
|
end
|
21
|
+
type = options[:type]
|
21
22
|
|
22
23
|
return nil unless File.exists?(conf_file)
|
23
24
|
hash = YAML.load_file(conf_file)
|
@@ -42,7 +43,7 @@ module Zillabyte
|
|
42
43
|
path = Pathname.new(File.expand_path(path_string))
|
43
44
|
path.relative_path_from(top_path).to_s()
|
44
45
|
rescue Exception => e
|
45
|
-
|
46
|
+
session.error(e.message() + "\n" + e.backtrace.join("\n"), type) if session
|
46
47
|
nil
|
47
48
|
end
|
48
49
|
}.compact()
|
data/lib/zillabyte/cli/help.rb
CHANGED
@@ -0,0 +1,25 @@
|
|
1
|
+
class Zillabyte::Helpers::TableOutputBuilder
|
2
|
+
|
3
|
+
|
4
|
+
def self.build_table(headings, rows, format = nil)
|
5
|
+
if format == "json"
|
6
|
+
self.build_json_table(headings, rows)
|
7
|
+
else
|
8
|
+
self.build_terminal_table(headings, rows)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.build_json_table(headings, rows)
|
13
|
+
out = []
|
14
|
+
rows.map { |row|
|
15
|
+
h = {}
|
16
|
+
row.each_with_index.map{|x,i| h[headings[i]] = x}
|
17
|
+
out << h
|
18
|
+
}
|
19
|
+
return out.to_json
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.build_terminal_table(headings, rows)
|
23
|
+
Terminal::Table.new(:headings => headings, :rows => rows).to_s
|
24
|
+
end
|
25
|
+
end
|
@@ -3,13 +3,14 @@ class LogFormatter
|
|
3
3
|
########################################################
|
4
4
|
## Base / Generic ######################################
|
5
5
|
########################################################
|
6
|
+
COLORS = [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
|
6
7
|
|
7
8
|
class Base
|
8
9
|
|
9
10
|
def initialize
|
10
|
-
@all_colors ||=
|
11
|
+
@all_colors ||= COLORS.clone
|
11
12
|
@color_map = {
|
12
|
-
'
|
13
|
+
'app' => :white
|
13
14
|
}
|
14
15
|
end
|
15
16
|
|
@@ -79,7 +80,7 @@ class LogFormatter
|
|
79
80
|
|
80
81
|
def handle_hidden(operation_name, operation_index, category, date, line)
|
81
82
|
case line
|
82
|
-
when 'REGISTER_FLOW_DEPLOYED', 'REGISTER_FLOW_TIMEOUT'
|
83
|
+
when 'REGISTER_FLOW_DEPLOYED', 'REGISTER_FLOW_TIMEOUT', 'STARTUP_ABORT'
|
83
84
|
# done reading the logs
|
84
85
|
exit(0)
|
85
86
|
end
|
data/lib/zillabyte/cli/query.rb
CHANGED
@@ -4,13 +4,17 @@ require "zillabyte/cli/base"
|
|
4
4
|
#
|
5
5
|
class Zillabyte::Command::Query < Zillabyte::Command::Base
|
6
6
|
|
7
|
+
MAX_POLL_SECONDS = 60 * 5
|
8
|
+
POLL_SLEEP = 0.5
|
9
|
+
|
7
10
|
# query:sxp EXPRESSION
|
8
11
|
#
|
9
12
|
# executes queries against the zillabyte corpus
|
10
13
|
#
|
11
14
|
# -o, --offset OFFSET # skips to the offset (default: 0)
|
12
15
|
# -l, --limit LIMIT # sets the result limit (default: 20)
|
13
|
-
#
|
16
|
+
# --type TYPE # the output format type i.e json
|
17
|
+
# -t, --tail TAIL # continuously watches for new results
|
14
18
|
#
|
15
19
|
#Examples:
|
16
20
|
#
|
@@ -21,12 +25,13 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
21
25
|
opts = {}
|
22
26
|
opts[:offset] = options[:offset] || 0
|
23
27
|
opts[:limit] = options[:limit] || 10
|
28
|
+
type = options[:type]
|
24
29
|
tail = options[:tail] || false
|
25
30
|
expression = options[:expression] || shift_argument
|
26
31
|
opts[:since] = options[:since]
|
27
32
|
|
28
33
|
if expression.nil?
|
29
|
-
error
|
34
|
+
error("no expression given", type)
|
30
35
|
end
|
31
36
|
|
32
37
|
seen = {}
|
@@ -34,8 +39,42 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
34
39
|
begin
|
35
40
|
|
36
41
|
validate_arguments!
|
37
|
-
res = api.query.sxp(expression, opts)
|
42
|
+
res = api.query.sxp(expression, :post, opts)
|
38
43
|
|
44
|
+
if res['job_id']
|
45
|
+
job_id = res['job_id']
|
46
|
+
opts[:job_id] = job_id
|
47
|
+
|
48
|
+
start = Time.now.utc
|
49
|
+
display "Fetching your data, please wait..." if type.nil?
|
50
|
+
|
51
|
+
while(Time.now.utc < start + MAX_POLL_SECONDS) do
|
52
|
+
|
53
|
+
res = api.query.sxp(expression, :get, opts)
|
54
|
+
case res['status']
|
55
|
+
when 'completed'
|
56
|
+
if(res['return'])
|
57
|
+
res = res['return']
|
58
|
+
else
|
59
|
+
error("something is wrong #{res}", type)
|
60
|
+
end
|
61
|
+
# success! continue below
|
62
|
+
break
|
63
|
+
when 'running'
|
64
|
+
sleep(POLL_SLEEP)
|
65
|
+
else
|
66
|
+
error("unknown status: #{res}", type)
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
else
|
71
|
+
if res['error']
|
72
|
+
error(res['error_message'] || res['error'], type)
|
73
|
+
else
|
74
|
+
error("remote server error(r256)", type)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
39
78
|
filtered = []
|
40
79
|
headings = []
|
41
80
|
if res['results'].first
|
@@ -52,12 +91,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
52
91
|
end
|
53
92
|
|
54
93
|
if filtered.size > 0
|
55
|
-
|
56
|
-
table = Terminal::Table.new :headings => headings, :rows => filtered
|
57
|
-
puts table.to_s
|
94
|
+
display TableOutputBuilder.build_table(headings, filtered)
|
58
95
|
else
|
59
96
|
unless tail
|
60
|
-
display "no results"
|
97
|
+
display "no results" if type.nil?
|
61
98
|
end
|
62
99
|
end
|
63
100
|
|
@@ -77,9 +114,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
77
114
|
#
|
78
115
|
# -o, --offset OFFSET # skips to the offset (default: 0)
|
79
116
|
# -l, --limit LIMIT # sets the result limit (default: 20)
|
80
|
-
# -t, --tail
|
117
|
+
# -t, --tail TAIL # continuously watches for new results
|
81
118
|
# -s, --since SINCE # newer records since
|
82
119
|
# --no_truncation # doesn't truncate long strings
|
120
|
+
# --type TYPE # The result display type
|
83
121
|
#
|
84
122
|
#Examples:
|
85
123
|
#
|
@@ -90,20 +128,56 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
90
128
|
opts = {}
|
91
129
|
opts[:offset] = options[:offset] || 0
|
92
130
|
opts[:limit] = options[:limit] || 10
|
131
|
+
|
93
132
|
tail = options[:tail] || false
|
94
133
|
expression = options[:expression] || shift_argument
|
95
134
|
opts[:since] = options[:since]
|
135
|
+
type = options[:type]
|
96
136
|
seen = {}
|
97
137
|
|
98
138
|
if expression.nil?
|
99
|
-
error
|
139
|
+
error("no expression given", type)
|
100
140
|
end
|
101
141
|
|
102
|
-
|
103
142
|
begin
|
104
143
|
|
105
144
|
validate_arguments!
|
106
|
-
res = api.query.sql(expression, opts)
|
145
|
+
res = api.query.sql(expression, :post, opts)
|
146
|
+
|
147
|
+
if res['job_id']
|
148
|
+
job_id = res['job_id']
|
149
|
+
opts[:job_id] = job_id
|
150
|
+
|
151
|
+
start = Time.now.utc
|
152
|
+
|
153
|
+
col_aliases = res['column_aliases']
|
154
|
+
while(Time.now.utc < start + MAX_POLL_SECONDS) do
|
155
|
+
|
156
|
+
res = api.query.sql(expression, :get, opts)
|
157
|
+
case res['status']
|
158
|
+
when 'completed'
|
159
|
+
if(res['return'])
|
160
|
+
res = res['return']
|
161
|
+
else
|
162
|
+
throw "something is wrong: #{res}"
|
163
|
+
end
|
164
|
+
# success! continue below
|
165
|
+
break
|
166
|
+
when 'running'
|
167
|
+
sleep(POLL_SLEEP)
|
168
|
+
# display ".", false
|
169
|
+
else
|
170
|
+
throw "unknown status: #{res}"
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
else
|
175
|
+
if res['error']
|
176
|
+
error(res['error_message'] || res['error'], type)
|
177
|
+
else
|
178
|
+
error("remote server error(r256)", type)
|
179
|
+
end
|
180
|
+
end
|
107
181
|
|
108
182
|
headings = []
|
109
183
|
filtered = []
|
@@ -111,7 +185,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
111
185
|
concrete_headings = res['rows'].first.keys
|
112
186
|
concrete_headings.each do |ch|
|
113
187
|
has_alias = false
|
114
|
-
|
188
|
+
col_aliases.each do |al|
|
115
189
|
if(al["concrete_name"] == ch)
|
116
190
|
headings << al["alias"]
|
117
191
|
has_alias = true
|
@@ -143,16 +217,17 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
143
217
|
end
|
144
218
|
end
|
145
219
|
|
146
|
-
|
147
220
|
if filtered.size > 0
|
148
|
-
|
149
|
-
table = Terminal::Table.new :headings => headings, :rows => filtered
|
150
|
-
puts table.to_s
|
151
221
|
opts[:since] = Time.now.utc
|
152
|
-
|
222
|
+
display TableOutputBuilder.build_table(headings, filtered, type)
|
223
|
+
|
153
224
|
else
|
154
225
|
unless tail
|
155
|
-
|
226
|
+
if type == "json"
|
227
|
+
display {}.to_json
|
228
|
+
else
|
229
|
+
display "no results"
|
230
|
+
end
|
156
231
|
end
|
157
232
|
end
|
158
233
|
|
@@ -166,28 +241,34 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
166
241
|
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
|
167
242
|
#
|
168
243
|
# pulls query data to S3_BUCKET/FILE_KEY/part***.gz
|
169
|
-
#
|
244
|
+
# --type TYPE # specify an output format type i.e. json
|
170
245
|
def pull_to_s3
|
171
246
|
|
172
247
|
query = options[:query] || shift_argument
|
173
|
-
error
|
248
|
+
error("query cannot be empty", type) if query.nil?
|
249
|
+
|
250
|
+
type = options [:type]
|
174
251
|
|
175
252
|
user_s3_access_key = options[:s3_access_key] || shift_argument
|
176
253
|
user_s3_secret = options[:s3_secret] || shift_argument
|
177
254
|
user_s3_bucket = options[:s3_bucket] || shift_argument
|
178
255
|
user_s3_file_key = options[:s3_file_key] || shift_argument
|
179
|
-
error
|
180
|
-
error
|
181
|
-
error
|
182
|
-
error
|
256
|
+
error("no s3 access key provided", type) if user_s3_access_key.nil?
|
257
|
+
error("no s3 secret provided", type) if user_s3_secret.nil?
|
258
|
+
error("no s3 bucket provided", type) if user_s3_bucket.nil?
|
259
|
+
error("no s3 file path specified", type) if user_s3_file_key.nil?
|
183
260
|
|
184
261
|
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
185
262
|
:s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
|
186
263
|
|
187
264
|
res = self.api.queries.pull_to_s3(query, s3_params)
|
188
|
-
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
189
|
-
display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
|
190
265
|
|
266
|
+
if type == "json"
|
267
|
+
display {}.to_json
|
268
|
+
else
|
269
|
+
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
270
|
+
display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
|
271
|
+
end
|
191
272
|
end
|
192
273
|
alias_command "query:pull:s3", "query:pull_to_s3"
|
193
274
|
|