zillabyte-cli 0.1.6 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- Nzg4MjUyOWI3MDVkZGMwZjUyNWI2YmUzOTIyMjMyYjRhZWFmMTA1MA==
4
+ MjFiZWM2YzdkZGQwZTI0OWQzOWVlMjViNTM5YmY0NmVlMDM4MjFjMg==
5
5
  data.tar.gz: !binary |-
6
- MGY3ODY4NTU3MzIyZmIxZDcxMmIzODY1MjYyZDgxNmE5N2IxYjk3Yg==
6
+ OGQ1NjRiYTQ2M2E0MWMyMzFhOTU3Yzg0MTMxZjJhODdiN2YyNzJkNA==
7
7
  !binary "U0hBNTEy":
8
8
  metadata.gz: !binary |-
9
- NzQzZGRhNmYwOWFhMzVjM2RkODFhZjQ3NjNmYzNkNmIyY2JlZDg3Mjg5YTNh
10
- YTRjZTgwODY0MjZiNDgxNzhjMjUzMDg5OWRiNmExYmZiNzU0MTM3MGFiOWMw
11
- Nzc4YTU3NDJjZGY5ZDFkN2Q5MjBiNjM2ODY3YjE5OWUwNjc0NmU=
9
+ ZWRlMDM0NDc2NDhjODc2MDhiMDllZGQwOTc0YmE5YWRiYjYxMzg3YzZiNmEz
10
+ ZjMyYWFmNTM0NTE4NmU4ZmM2ODIwMWNlMWYyNjQ4YWE0OTkxMzMxYTczZTMw
11
+ MzJkMzE3MDc5MDAzMjc4YTU5MDcyYmJkNjY4NzExNjQ0NjIzOGY=
12
12
  data.tar.gz: !binary |-
13
- MzQyMWNjYTViOTYwMjgzNjc4ZmY2M2U3ZTM1ZjM0NDgyYTI5M2EyMjcxMzQ1
14
- MWVkNGZlMDM4ZmY2ZTVkZTJkOGVlNDM5ZDlmNzdhMDBjY2IzYjU2N2FjYmE0
15
- ZjgzNzcxYzY2MTQzMTY2NjIwYzVkOTU0YTVlOThhZTNkY2ZiOTA=
13
+ OTY1NjVlYzYxZTQxMmY1MjQ5MjQxNTcyM2ZlNmI3MDFiYTM2YWNmYTA2NjYy
14
+ ZmYxYTNmOWYwMDZlNTgwMjI1NzBiM2FhNWVlNzk5ZmEyYTk4NTNiZTFjN2Vj
15
+ N2Q2ZjhlNjUyYzkxNmUxMDQ0ZjNkYWQzNTVhOGY4NTEwODA3Nzk=
@@ -1,5 +1,5 @@
1
1
  module Zillabyte
2
2
  module CLI
3
- VERSION = "0.1.6"
3
+ VERSION = "0.1.7"
4
4
  end
5
5
  end
@@ -364,7 +364,7 @@ class Zillabyte::Command::Apps < Zillabyte::Command::Flows
364
364
  # List the apps
365
365
  response = api.apps.list_cycles(app_id, options)
366
366
  # TODO List the sequence number for this app.
367
- display "Most recent cyles of the app:" if type.nil?
367
+ display "Most recent cycles of the app:" if type.nil?
368
368
  headings = ["Cycle_id", "State", "Start", "End"]
369
369
  rows = response["cycles"]
370
370
  rows = rows.map do |row|
@@ -252,11 +252,19 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
252
252
  alias_command "dataset:download", "data:pull"
253
253
 
254
254
 
255
- # data:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
255
+ # data:pull:s3 ID S3_PATH
256
256
  #
257
- # Pulls dataset to S3_BUCKET/FILE_KEY/part***.gz.
257
+ # Pulls dataset to s3_bucket/s3_key/part***.gz using the given s3_access and s3_secret credentials.
258
+ # S3_PATH may be given in the following forms:
259
+ # 1) s3://s3_access:s3_secret@s3_bucket/s3_key
260
+ # 2) s3://s3_bucket/s3_key: also supply --s3_access and --s3_secret OR set the environment variables S3_ACCESS and S3_SECRET
261
+ # 3) s3_key: also supply --s3_access, --s3_secret and --s3_bucket OR set the environment variables S3_ACCESS and S3_SECRET and supply --s3_bucket
258
262
  #
259
263
  # --cycle_id [cycle_id] # Retrieve data generated during specified cycle if dataset is associated with an app [default: last cycle]
264
+ # --s3_access [s3_access_key] # S3 access key
265
+ # --s3_secret [s3_secret_key] # S3 secret key
266
+ # --s3_bucket [s3_bucket] # S3 bucket to store data at
267
+ # --s3_key [s3_file_key] # S3 key to store data at
260
268
  # --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
261
269
  #
262
270
  def pull_to_s3
@@ -265,17 +273,29 @@ class Zillabyte::Command::Data < Zillabyte::Command::Base
265
273
  type = options[:output_type]
266
274
  error("no id given", type) if id.nil?
267
275
 
268
- user_s3_access_key = options[:s3_access_key] || shift_argument
269
- user_s3_secret = options[:s3_secret] || shift_argument
270
- user_s3_bucket = options[:s3_bucket] || shift_argument
271
- user_s3_file_key = options[:s3_file_key] || shift_argument
272
- error("no s3 access key provided", type) if user_s3_access_key.nil?
273
- error("no s3 access secret provided", type) if user_s3_secret.nil?
274
- error("no s3 access bucket provided", type) if user_s3_bucket.nil?
275
- error("no s3 file path provided", type) if user_s3_file_key.nil?
276
-
277
- s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
278
- :s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
276
+ s3_path = options[:s3_path] || shift_argument
277
+ matches = s3_path.match(/^s3:\/\/([A-Z0-9]{20}):([$-\/:-?{-~!"^_`\[\]\w]{40})@([\w\.]*)\/([-\w\/]*)$/)
278
+ if !matches.nil?
279
+ s3_access, s3_secret, s3_bucket, s3_key = matches.captures
280
+ else
281
+ matches = s3_path.match(/^s3:\/\/([\w\.]*)\/([-\w\/]*)$/)
282
+ if !matches.nil?
283
+ s3_bucket, s3_key = matches.captures
284
+ else
285
+ s3_key = s3_path
286
+ end
287
+ end
288
+ s3_access = options[:s3_access] || ENV["S3_ACCESS"] if s3_access == "" or s3_access.nil?
289
+ s3_secret = options[:s3_secret] || ENV["S3_SECRET"] if s3_secret == "" or s3_secret.nil?
290
+ s3_bucket = options[:s3_bucket] if s3_bucket == "" or s3_bucket.nil?
291
+
292
+ error("No s3 access key or invalid access key provided. Please check that you have entered the access key correctly.", type) if s3_access.nil?
293
+ error("No s3 access secret key or invalid secret key provided. Please check that you have entered the secret key correctly.", type) if s3_secret.nil?
294
+ error("No s3 access bucket or invalid bucket provided. Please check that you have entered the bucket correctly.", type) if s3_bucket.nil?
295
+ error("No s3 file key provided. Please check that you have entered the file key correctly.", type) if s3_key.nil?
296
+
297
+ s3_params = {:s3_access_key => s3_access, :s3_secret => s3_secret,
298
+ :s3_bucket => s3_bucket, :s3_file_key => s3_key}
279
299
  s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
280
300
 
281
301
  res = self.api.data.pull_to_s3(id, s3_params)
@@ -170,8 +170,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
170
170
  end
171
171
 
172
172
 
173
- def cdisplay(name, message)
174
- require("colorize")
173
+ def cdisplay(name, message, useName=true)
174
+
175
175
  color = @colors[name] || :default
176
176
  if message == ""
177
177
  display ""
@@ -182,8 +182,8 @@ class Zillabyte::Runner::AppRunner < Zillabyte::Command::Base
182
182
  else
183
183
  lines = message.split("\n")
184
184
  end
185
-
186
- prefix = "#{name} - "
185
+
186
+ prefix = useName ? "#{name} - " : ""
187
187
  display "#{prefix}#{lines.first}".colorize(color)
188
188
  lines[1..-1].each do |line|
189
189
  display "#{' '*prefix.size}#{line}".colorize(color)
@@ -573,9 +573,8 @@ class Zillabyte::Runner::ComponentOperation
573
573
 
574
574
 
575
575
  # Display a colored, formatted message
576
- def self.cdisplay(msg)
577
-
578
- @__tester.cdisplay(@__name, msg)
576
+ def self.cdisplay(msg, useName=true)
577
+ @__tester.cdisplay(@__name, msg, useName)
579
578
  end
580
579
 
581
580
 
@@ -185,14 +185,24 @@ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
185
185
  end
186
186
 
187
187
 
188
- def cdisplay(name, message)
189
- require("colorize")
190
- color = @colors[name] || :default
191
- if message == ""
192
- display ""
188
+ def cdisplay(name, message, useName=true)
189
+ color = @colors[name] || :default
190
+ if message == ""
191
+ display ""
192
+ else
193
+
194
+ if message.is_a?(Array)
195
+ lines = message
193
196
  else
194
- display "#{name} - #{message}".colorize(color)
197
+ lines = message.split("\n")
198
+ end
199
+
200
+ prefix = useName ? "#{name} - " : ""
201
+ display "#{prefix}#{lines.first}".colorize(color)
202
+ lines[1..-1].each do |line|
203
+ display "#{' '*prefix.size}#{line}".colorize(color)
195
204
  end
205
+ end
196
206
  end
197
207
 
198
208
 
@@ -846,8 +846,26 @@ module Zillabyte; module Runner; class MultilangOperation
846
846
 
847
847
  # Send a message to all consumers of the operation
848
848
  def self.run_sink()
849
- output = @__options[:output]
849
+ columns = @__node["columns"]
850
+
851
+ type_map = {
852
+ "string" => String,
853
+ "double" => Float,
854
+ "integer" => Integer
855
+ }
856
+
857
+ col_map = {}
858
+ columns.each do |hash|
859
+ key = hash.keys[0]
860
+ type = hash[key]
861
+ col_map[key] = type_map[type]
862
+ end
863
+
864
+
865
+
850
866
  messages = []
867
+
868
+ output = @__options["output"]
851
869
  loop do
852
870
  # Read messages
853
871
  msg = read_message(@__consumee[:rd_child])
@@ -855,8 +873,36 @@ module Zillabyte; module Runner; class MultilangOperation
855
873
 
856
874
  # Add row
857
875
  if obj['tuple']
858
- messages << msg
876
+
877
+ tuple = obj['tuple']
859
878
  display_json = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}].to_json
879
+
880
+ if col_map.keys.length != tuple.keys.length
881
+ cdisplay "Error: invalid keys for sink tuple : Expected #{col_map.keys} , got: #{tuple.keys}"
882
+ cdisplay("\n \nPress Ctrl-C to exit", false)
883
+ return
884
+ end
885
+
886
+ # Check tuple columns for valid entry
887
+ columns_to_check = col_map.length
888
+ tuple.each_pair do |col,value|
889
+ types = type_map.each_value.map {|t| value.is_a? t}
890
+ if col_map.include? col
891
+ if value.is_a? col_map[col]
892
+ columns_to_check -= 1
893
+ else
894
+ break
895
+ end
896
+ end
897
+ end
898
+
899
+ if columns_to_check != 0
900
+ cdisplay "Error: invalid schema for sink tuple #{display_json}"
901
+ cdisplay("\n \nPress Ctrl-C to exit", false)
902
+ return
903
+ end
904
+
905
+ messages << msg
860
906
  if @__options[:interactive]
861
907
  cdisplay "received #{display_json}"
862
908
  end
@@ -910,7 +956,7 @@ module Zillabyte; module Runner; class MultilangOperation
910
956
 
911
957
  # Output table
912
958
  cdisplay("\n#{table.to_s}")
913
- cdisplay ""
959
+ cdisplay "#{messages.length} rows"
914
960
 
915
961
  # Write file
916
962
  if output
@@ -920,7 +966,7 @@ module Zillabyte; module Runner; class MultilangOperation
920
966
  f.close()
921
967
  cdisplay("output written to #{filename}")
922
968
  end
923
- cdisplay "\n \nPress Ctrl-C to exit"
969
+ cdisplay("\n \nPress Ctrl-C to exit", false)
924
970
  end
925
971
 
926
972
 
@@ -1134,9 +1180,9 @@ module Zillabyte; module Runner; class MultilangOperation
1134
1180
 
1135
1181
 
1136
1182
  # Display a colored, formatted message
1137
- def self.cdisplay(msg)
1138
- @__tester.cdisplay(@__name, msg)
1183
+ def self.cdisplay(msg, useName=true)
1184
+ @__tester.cdisplay(@__name, msg, useName)
1139
1185
  end
1140
1186
 
1141
-
1142
1187
  end; end; end
1188
+
@@ -41,7 +41,7 @@ Getting Started with Zillabyte
41
41
 
42
42
  (1) Register for an auth token at http://zillabyte.com
43
43
  (2) Log in by running 'zillabyte login' in the command line
44
- (3) Build an empty app by running 'zillabyte app:init'
44
+ (3) Build an empty app by running 'zillabyte apps:init'
45
45
  Or... check out our Quick Starts by visiting http://docs.zillabyte.com/
46
46
 
47
47
  Questions, comments? Please visit us at http://docs.zillabyte.com
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zillabyte-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.6
4
+ version: 0.1.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - zillabyte
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-07-18 00:00:00.000000000 Z
11
+ date: 2014-07-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -257,7 +257,7 @@ licenses:
257
257
  metadata: {}
258
258
  post_install_message: ! "\nGetting Started with Zillabyte\n==============================\n\n(1)
259
259
  Register for an auth token at http://zillabyte.com\n(2) Log in by running 'zillabyte
260
- login' in the command line\n(3) Build an empty app by running 'zillabyte app:init'\n
260
+ login' in the command line\n(3) Build an empty app by running 'zillabyte apps:init'\n
261
261
  \ Or... check out our Quick Starts by visiting http://docs.zillabyte.com/\n\nQuestions,
262
262
  comments? Please visit us at http://docs.zillabyte.com\n"
263
263
  rdoc_options: []