zillabyte-cli 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/cli/apps.rb +1 -1
- data/lib/zillabyte/cli/data.rb +33 -13
- data/lib/zillabyte/runner/app_runner.rb +4 -4
- data/lib/zillabyte/runner/component_operation.rb +2 -3
- data/lib/zillabyte/runner/component_runner.rb +16 -6
- data/lib/zillabyte/runner/multilang_operation.rb +53 -7
- data/zillabyte-cli.gemspec +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MjFiZWM2YzdkZGQwZTI0OWQzOWVlMjViNTM5YmY0NmVlMDM4MjFjMg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
OGQ1NjRiYTQ2M2E0MWMyMzFhOTU3Yzg0MTMxZjJhODdiN2YyNzJkNA==
|
7
7
|
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZWRlMDM0NDc2NDhjODc2MDhiMDllZGQwOTc0YmE5YWRiYjYxMzg3YzZiNmEz
|
10
|
+
ZjMyYWFmNTM0NTE4NmU4ZmM2ODIwMWNlMWYyNjQ4YWE0OTkxMzMxYTczZTMw
|
11
|
+
MzJkMzE3MDc5MDAzMjc4YTU5MDcyYmJkNjY4NzExNjQ0NjIzOGY=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
OTY1NjVlYzYxZTQxMmY1MjQ5MjQxNTcyM2ZlNmI3MDFiYTM2YWNmYTA2NjYy
|
14
|
+
ZmYxYTNmOWYwMDZlNTgwMjI1NzBiM2FhNWVlNzk5ZmEyYTk4NTNiZTFjN2Vj
|
15
|
+
N2Q2ZjhlNjUyYzkxNmUxMDQ0ZjNkYWQzNTVhOGY4NTEwODA3Nzk=
|
data/lib/zillabyte/cli/apps.rb
CHANGED
@@ -364,7 +364,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
|
|
364
364
|
# List the apps
|
365
365
|
response = api.apps.list_cycles(app_id, options)
|
366
366
|
# TODO List the sequence number for this app.
|
367
|
-
display "Most recent
|
367
|
+
display "Most recent cycles of the app:" if type.nil?
|
368
368
|
headings = ["Cycle_id", "State", "Start", "End"]
|
369
369
|
rows = response["cycles"]
|
370
370
|
rows = rows.map do |row|
|
data/lib/zillabyte/cli/data.rb
CHANGED
@@ -252,11 +252,19 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
252
252
|
alias_command "dataset:download", "data:pull"
|
253
253
|
|
254
254
|
|
255
|
-
# data:pull:s3 ID
|
255
|
+
# data:pull:s3 ID S3_PATH
|
256
256
|
#
|
257
|
-
# Pulls dataset to
|
257
|
+
# Pulls dataset to s3_bucket/s3_key/part***.gz using the given s3_access and s3_secret credentials.
|
258
|
+
# S3_PATH may be given in the following forms:
|
259
|
+
# 1) s3://s3_access:s3_secret@s3_bucket/s3_key
|
260
|
+
# 2) s3://s3_bucket/s3_key: also supply --s3_access and --s3_secret OR set the environment variables S3_ACCESS and S3_SECRET
|
261
|
+
# 3) s3_key: also supply --s3_access, --s3_secret and --s3_bucket OR set the environment variables S3_ACCESS and S3_SECRET and supply --s3_bucket
|
258
262
|
#
|
259
263
|
# --cycle_id [cycle_id] # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
|
264
|
+
# --s3_access [s3_access_key] # S3 access key
|
265
|
+
# --s3_secret [s3_secret_key] # S3 secret key
|
266
|
+
# --s3_bucket [s3_bucket] # S3 bucket to store data at
|
267
|
+
# --s3_key [s3_file_key] # S3 key to store data at
|
260
268
|
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
261
269
|
#
|
262
270
|
def pull_to_s3
|
@@ -265,17 +273,29 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
|
|
265
273
|
type = options[:output_type]
|
266
274
|
error("no id given", type) if id.nil?
|
267
275
|
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
276
|
+
s3_path = options[:s3_path] || shift_argument
|
277
|
+
matches = s3_path.match(/^s3:\/\/([A-Z0-9]{20}):([$-\/:-?{-~!"^_`\[\]\w]{40})@([\w\.]*)\/([-\w\/]*)$/)
|
278
|
+
if !matches.nil?
|
279
|
+
s3_access, s3_secret, s3_bucket, s3_key = matches.captures
|
280
|
+
else
|
281
|
+
matches = s3_path.match(/^s3:\/\/([\w\.]*)\/([-\w\/]*)$/)
|
282
|
+
if !matches.nil?
|
283
|
+
s3_bucket, s3_key = matches.captures
|
284
|
+
else
|
285
|
+
s3_key = s3_path
|
286
|
+
end
|
287
|
+
end
|
288
|
+
s3_access = options[:s3_access] || ENV["S3_ACCESS"] if s3_access == "" or s3_access.nil?
|
289
|
+
s3_secret = options[:s3_secret] || ENV["S3_SECRET"] if s3_secret == "" or s3_secret.nil?
|
290
|
+
s3_bucket = options[:s3_bucket] if s3_bucket == "" or s3_bucket.nil?
|
291
|
+
|
292
|
+
error("No s3 access key or invalid access key provided. Please check that you have entered the access key correctly.", type) if s3_access.nil?
|
293
|
+
error("No s3 access secret key or invalid secret key provided. Please check that you have entered the secret key correctly.", type) if s3_secret.nil?
|
294
|
+
error("No s3 access bucket or invalid bucket provided. Please check that you have entered the bucket correctly.", type) if s3_bucket.nil?
|
295
|
+
error("No s3 file key provided. Please check that you have entered the file key correctly.", type) if s3_key.nil?
|
296
|
+
|
297
|
+
s3_params = {:s3_access_key => s3_access, :s3_secret => s3_secret,
|
298
|
+
:s3_bucket => s3_bucket, :s3_file_key => s3_key}
|
279
299
|
s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
|
280
300
|
|
281
301
|
res = self.api.data.pull_to_s3(id, s3_params)
|
@@ -170,8 +170,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
|
|
170
170
|
end
|
171
171
|
|
172
172
|
|
173
|
-
def cdisplay(name, message)
|
174
|
-
|
173
|
+
def cdisplay(name, message, useName=true)
|
174
|
+
|
175
175
|
color = @colors[name] || :default
|
176
176
|
if message == ""
|
177
177
|
display ""
|
@@ -182,8 +182,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
|
|
182
182
|
else
|
183
183
|
lines = message.split("\n")
|
184
184
|
end
|
185
|
-
|
186
|
-
prefix = "#{name} - "
|
185
|
+
|
186
|
+
prefix = useName ? "#{name} - " : ""
|
187
187
|
display "#{prefix}#{lines.first}".colorize(color)
|
188
188
|
lines[1..-1].each do |line|
|
189
189
|
display "#{' '*prefix.size}#{line}".colorize(color)
|
@@ -573,9 +573,8 @@ class Zillabyte::Runner::ComponentOperation
|
|
573
573
|
|
574
574
|
|
575
575
|
# Display a colored, formatted message
|
576
|
-
def self.cdisplay(msg)
|
577
|
-
|
578
|
-
@__tester.cdisplay(@__name, msg)
|
576
|
+
def self.cdisplay(msg, useName=true)
|
577
|
+
@__tester.cdisplay(@__name, msg, useName)
|
579
578
|
end
|
580
579
|
|
581
580
|
|
@@ -185,14 +185,24 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
|
185
185
|
end
|
186
186
|
|
187
187
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
188
|
+
def cdisplay(name, message, useName=true)
|
189
|
+
color = @colors[name] || :default
|
190
|
+
if message == ""
|
191
|
+
display ""
|
192
|
+
else
|
193
|
+
|
194
|
+
if message.is_a?(Array)
|
195
|
+
lines = message
|
193
196
|
else
|
194
|
-
|
197
|
+
lines = message.split("\n")
|
198
|
+
end
|
199
|
+
|
200
|
+
prefix = useName ? "#{name} - " : ""
|
201
|
+
display "#{prefix}#{lines.first}".colorize(color)
|
202
|
+
lines[1..-1].each do |line|
|
203
|
+
display "#{' '*prefix.size}#{line}".colorize(color)
|
195
204
|
end
|
205
|
+
end
|
196
206
|
end
|
197
207
|
|
198
208
|
|
@@ -846,8 +846,26 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
846
846
|
|
847
847
|
# Send a message to all consumers of the operation
|
848
848
|
def self.run_sink()
|
849
|
-
|
849
|
+
columns = @__node["columns"]
|
850
|
+
|
851
|
+
type_map = {
|
852
|
+
"string" => String,
|
853
|
+
"double" => Float,
|
854
|
+
"integer" => Integer
|
855
|
+
}
|
856
|
+
|
857
|
+
col_map = {}
|
858
|
+
columns.each do |hash|
|
859
|
+
key = hash.keys[0]
|
860
|
+
type = hash[key]
|
861
|
+
col_map[key] = type_map[type]
|
862
|
+
end
|
863
|
+
|
864
|
+
|
865
|
+
|
850
866
|
messages = []
|
867
|
+
|
868
|
+
output = @__options["output"]
|
851
869
|
loop do
|
852
870
|
# Read messages
|
853
871
|
msg = read_message(@__consumee[:rd_child])
|
@@ -855,8 +873,36 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
855
873
|
|
856
874
|
# Add row
|
857
875
|
if obj['tuple']
|
858
|
-
|
876
|
+
|
877
|
+
tuple = obj['tuple']
|
859
878
|
display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
|
879
|
+
|
880
|
+
if col_map.keys.length != tuple.keys.length
|
881
|
+
cdisplay "Error: invalid keys for sink tuple : Expected #{col_map.keys} , got: #{tuple.keys}"
|
882
|
+
cdisplay("\n \nPress Ctrl-C to exit", false)
|
883
|
+
return
|
884
|
+
end
|
885
|
+
|
886
|
+
# Check tuple columns for valid entry
|
887
|
+
columns_to_check = col_map.length
|
888
|
+
tuple.each_pair do |col,value|
|
889
|
+
types = type_map.each_value.map {|t| value.is_a? t}
|
890
|
+
if col_map.include? col
|
891
|
+
if value.is_a? col_map[col]
|
892
|
+
columns_to_check -= 1
|
893
|
+
else
|
894
|
+
break
|
895
|
+
end
|
896
|
+
end
|
897
|
+
end
|
898
|
+
|
899
|
+
if columns_to_check != 0
|
900
|
+
cdisplay "Error: invalid schema for sink tuple #{display_json}"
|
901
|
+
cdisplay("\n \nPress Ctrl-C to exit", false)
|
902
|
+
return
|
903
|
+
end
|
904
|
+
|
905
|
+
messages << msg
|
860
906
|
if @__options[:interactive]
|
861
907
|
cdisplay "received #{display_json}"
|
862
908
|
end
|
@@ -910,7 +956,7 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
910
956
|
|
911
957
|
# Output table
|
912
958
|
cdisplay("\n#{table.to_s}")
|
913
|
-
cdisplay ""
|
959
|
+
cdisplay "#{messages.length} rows"
|
914
960
|
|
915
961
|
# Write file
|
916
962
|
if output
|
@@ -920,7 +966,7 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
920
966
|
f.close()
|
921
967
|
cdisplay("output written to #{filename}")
|
922
968
|
end
|
923
|
-
cdisplay
|
969
|
+
cdisplay("\n \nPress Ctrl-C to exit", false)
|
924
970
|
end
|
925
971
|
|
926
972
|
|
@@ -1134,9 +1180,9 @@ module Zillabyte; module Runner; class MultilangOperation
|
|
1134
1180
|
|
1135
1181
|
|
1136
1182
|
# Display a colored, formatted message
|
1137
|
-
def self.cdisplay(msg)
|
1138
|
-
@__tester.cdisplay(@__name, msg)
|
1183
|
+
def self.cdisplay(msg, useName=true)
|
1184
|
+
@__tester.cdisplay(@__name, msg, useName)
|
1139
1185
|
end
|
1140
1186
|
|
1141
|
-
|
1142
1187
|
end; end; end
|
1188
|
+
|
data/zillabyte-cli.gemspec
CHANGED
@@ -41,7 +41,7 @@ Getting Started with Zillabyte
|
|
41
41
|
|
42
42
|
(1) Register for an auth token at http://zillabyte.com
|
43
43
|
(2) Log in by running 'zillabyte login' in the command line
|
44
|
-
(3) Build an empty app by running 'zillabyte
|
44
|
+
(3) Build an empty app by running 'zillabyte apps:init'
|
45
45
|
Or... check out our Quick Starts by visiting http://docs.zillabyte.com/
|
46
46
|
|
47
47
|
Questions, comments? Please visit us at http://docs.zillabyte.com
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-07-
|
11
|
+
date: 2014-07-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -257,7 +257,7 @@ licenses:
|
|
257
257
|
metadata: {}
|
258
258
|
post_install_message: ! "\nGetting Started with Zillabyte\n==============================\n\n(1)
|
259
259
|
Register for an auth token at http://zillabyte.com\n(2) Log in by running 'zillabyte
|
260
|
-
login' in the command line\n(3) Build an empty app by running 'zillabyte
|
260
|
+
login' in the command line\n(3) Build an empty app by running 'zillabyte apps:init'\n
|
261
261
|
\ Or... check out our Quick Starts by visiting http://docs.zillabyte.com/\n\nQuestions,
|
262
262
|
comments? Please visit us at http://docs.zillabyte.com\n"
|
263
263
|
rdoc_options: []
|