zillabyte-cli 0.1.1 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -15,11 +15,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
15
15
 
16
16
  # keys:add NAME [KEY]
17
17
  #
18
- # add a key for the current user
18
+ # Add a key for the current user.
19
+ # If no KEY is specified, will try to find ~/.ssh/id_rsa.pub.
19
20
  #
20
- # if no KEY is specified, will try to find ~/.ssh/id_rsa.pub
21
- #
22
- # --output_type OUTPUT_TYPE # The output format type
21
+ # --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
23
22
  #
24
23
  def add()
25
24
  type = options[:output_type] || nil
@@ -43,9 +42,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
43
42
 
44
43
  # keys:show NAME
45
44
  #
46
- # show a key for the current user
45
+ # Show a key for the current user.
47
46
  #
48
- # --output_type OUTPUT_TYPE # The output format type
47
+ # --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
49
48
  #
50
49
  def show()
51
50
  type = options[:output_type] || nil
@@ -64,11 +63,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
64
63
 
65
64
  # keys:remove [NAME]
66
65
  #
67
- # remove a key from the current user
66
+ # Remove a key from the current user.
68
67
  #
69
- # --output_type OUTPUT_TYPE # The output format type
70
- # -l, --long # display extended information for each key
71
- # -f, --force # Force -- remove key without prompting
68
+ # -l, --long # Display extended information for each key
69
+ # -f, --force # Force -- remove key without prompting
70
+ # --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
72
71
  #
73
72
  def remove()
74
73
  type = options[:output_type] || nil
@@ -127,10 +126,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
127
126
 
128
127
  # keys
129
128
  #
130
- # display keys for the current user
129
+ # Display keys for the current user.
130
+ #
131
+ # -l, --long # Display extended information for each key
132
+ # --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
131
133
  #
132
- # --output_type OUTPUT_TYPE # The output format type
133
- # -l, --long # display extended information for each key
134
134
  def index()
135
135
  type = options[:output_type] || nil
136
136
  long = options[:long]
@@ -144,9 +144,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
144
144
 
145
145
  # keys:clear
146
146
  #
147
- # remove all authentication keys from the current user
147
+ # Remove all authentication keys from the current user.
148
148
  #
149
- # --output_type OUTPUT_TYPE # The output format type
149
+ # --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
150
150
  #
151
151
  def clear()
152
152
  type = options[:output_type] || nil
@@ -159,7 +159,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
159
159
  end
160
160
  end
161
161
 
162
+
162
163
  private
164
+
165
+
163
166
  def format_keys(keys, long, type)
164
167
  outlong = long || type == "json"
165
168
  rows = keys.map() { |row|
@@ -7,14 +7,12 @@ require "zillabyte/common"
7
7
  #
8
8
  class Zillabyte::Command::Nuke < Zillabyte::Command::Base
9
9
 
10
-
11
-
12
10
 
13
11
  # apps:nuke
14
12
  #
15
- # destroys all state associated with the current user
13
+ # Destroys all state associated with the current user.
16
14
  #
17
- # --force # this option must always be given
15
+ # --force # This option must always be given
18
16
  #
19
17
  def index
20
18
 
@@ -34,9 +32,9 @@ class Zillabyte::Command::Nuke < Zillabyte::Command::Base
34
32
  if options[:output_type] == "json"
35
33
  display res.body.to_json
36
34
  else
37
- display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']} relations(s)"
35
+ display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']} dataset(s)"
38
36
  end
39
37
 
40
38
 
41
39
  end
42
- end
40
+ end
@@ -9,12 +9,13 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
9
9
 
10
10
  # query:sxp EXPRESSION
11
11
  #
12
- # executes queries against the zillabyte corpus
12
+ # Executes SXP queries against the zillabyte corpus.
13
13
  #
14
- # -o, --offset OFFSET # skips to the offset (default: 0)
15
- # -l, --limit LIMIT # sets the result limit (default: 20)
16
- # --output_type OUTPUT_TYPE # the output format type i.e json
17
- # -t, --tail TAIL # continuously watches for new results
14
+ # -o, --offset OFFSET # Skips to the offset [default: 0]
15
+ # -l, --limit LIMIT # Sets the result limit [default: 20]
16
+ # -t, --tail TAIL # Continuously watches for new results
17
+ # --meta # Show meta columns (since, confidence, source)
18
+ # --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
18
19
  #
19
20
  #Examples:
20
21
  #
@@ -26,6 +27,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
26
27
  opts[:offset] = options[:offset] || 0
27
28
  opts[:limit] = options[:limit] || 10
28
29
  type = options[:output_type]
30
+ show_meta = options[:meta] || false
29
31
  tail = options[:tail] || false
30
32
  expression = options[:expression] || shift_argument
31
33
  opts[:since] = options[:since]
@@ -79,6 +81,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
79
81
  headings = []
80
82
  if res['results'].first
81
83
  headings = res['results'].first.keys
84
+ META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
82
85
  rows = res['results'].each do |obj|
83
86
  new_row = headings.map do |heading|
84
87
  obj[heading]
@@ -110,14 +113,15 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
110
113
 
111
114
  # query:sql EXPRESSION
112
115
  #
113
- # executes queries against the zillabyte corpus
116
+ # Executes SQL queries against the zillabyte corpus.
114
117
  #
115
- # -o, --offset OFFSET # skips to the offset (default: 0)
116
- # -l, --limit LIMIT # sets the result limit (default: 20)
117
- # -t, --tail TAIL # continuously watches for new results
118
- # -s, --since SINCE # newer records since
119
- # --no_truncation # doesn't truncate long strings
120
- # --output_type OUTPUT_TYPE # The result display type
118
+ # -o, --offset OFFSET # Skips to the offset [default: 0]
119
+ # -l, --limit LIMIT # Sets the result limit [default: 20]
120
+ # -t, --tail TAIL # Continuously watches for new results
121
+ # -s, --since SINCE # Grab newer records since SINCE
122
+ # --no_truncation # Doesn't truncate long strings
123
+ # --meta # Show meta columns (since, confidence, source)
124
+ # --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
121
125
  #
122
126
  #Examples:
123
127
  #
@@ -132,6 +136,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
132
136
  tail = options[:tail] || false
133
137
  expression = options[:expression] || shift_argument
134
138
  opts[:since] = options[:since]
139
+ show_meta = options[:meta] || false
135
140
  type = options[:output_type]
136
141
  seen = {}
137
142
 
@@ -183,6 +188,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
183
188
  filtered = []
184
189
  if res['rows'].first
185
190
  concrete_headings = res['rows'].first.keys
191
+ META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
186
192
  concrete_headings.each do |ch|
187
193
  has_alias = false
188
194
  col_aliases.each do |al|
@@ -242,9 +248,8 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
242
248
 
243
249
  # query:pull QUERY FILE
244
250
  #
245
- # executes a queries and downloads the results to FILE
251
+ # Executes a query and downloads the results to FILE.
246
252
  #
247
- # pulls query data to OUTPUT.gz
248
253
  def pull
249
254
 
250
255
  query = options[:query] || shift_argument
@@ -264,43 +269,12 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
264
269
  )
265
270
  res = res.body
266
271
 
267
- if(res["uri"])
268
- display "Waiting for download." if type.nil?
269
- File.open(file, "w") do |f|
270
- f.write open(res["uri"]).read
271
- end
272
- elsif(res["s3_credentials"])
273
- display "Pulling query results." if type.nil?
274
- s3 = AWS::S3.new(res["s3_credentials"])
275
- bucket = s3.buckets[res["s3_bucket"]]
276
- obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
277
-
278
- while(true)
279
- keys = obj.collect(&:key)
280
- if keys.length > 0 and keys.include?("#{res["s3_file_key"]}/manifest")
281
- display "Starting to write to file..." if type.nil?
282
- File.open(file, "w") do |f|
283
- obj.each do |o|
284
- if o.key == "#{res["s3_file_key"]}/manifest"
285
- next
286
- end
287
- f.write(o.read)
288
- end
289
- end
290
- break
291
- else
292
- sleep(3)
293
- obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
294
- end
295
- end
296
- else
297
- throw "bad response: #{res}"
298
- end
272
+ handle_downloading_manifest(file, res, type)
299
273
 
300
274
  if type == "json"
301
275
  display "{}"
302
276
  else
303
- display "results downloaded to file #{file}"
277
+ display "finished downloading results to file"
304
278
  end
305
279
 
306
280
  end
@@ -310,8 +284,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
310
284
 
311
285
  # query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
312
286
  #
313
- # pulls query data to S3_BUCKET/FILE_KEY/part***.gz
314
- # --output_type OUTPUT_TYPE # specify an output format type i.e. json
287
+ # Executes a query and pulls data to S3_BUCKET/FILE_KEY/part***.gz.
288
+ #
289
+ # --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
290
+ #
315
291
  def pull_to_s3
316
292
 
317
293
  query = options[:query] || shift_argument
@@ -337,7 +313,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
337
313
  display {}.to_json
338
314
  else
339
315
  display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
340
- display "if the relation is large, this may take a while, please check your s3 account after a few minutes"
316
+ display "if the dataset is large, this may take a while, please check your s3 account after a few minutes"
341
317
  end
342
318
  end
343
319
  alias_command "query:pull:s3", "query:pull_to_s3"
@@ -7,9 +7,11 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
7
7
 
8
8
  # repl
9
9
  #
10
- # start a console session for zillabyte
11
- # --quiet # HIDDEN
12
- # --history HISTORY# HIDDEN hack to allow history for readline
10
+ # Start a console session for zillabyte.
11
+ #
12
+ # --quiet # HIDDEN
13
+ # --history HISTORY # HIDDEN hack to allow history for readline
14
+ #
13
15
  def index
14
16
  if !options[:quiet]
15
17
  Zillabyte::Command.run("version")
@@ -29,7 +31,7 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
29
31
  Readline::HISTORY << his
30
32
  end
31
33
  end
32
- # TODO: Add tab completion for basic commands, app/relation names etc.
34
+ # TODO: Add tab completion for basic commands, app/dataset names etc.
33
35
  buf = ""
34
36
  while cmd = Readline.readline(prompt, true)
35
37
  if cmd && cmd.length > 0
@@ -0,0 +1,123 @@
1
+ # Getting Started with Zillabyte Apps
2
+
3
+ Zillabyte Apps are chunks of code that can scalably analyze almost any type of data. You develop & test the app locally and then push to the Cloud when you are ready to scale.
4
+
5
+ Apps are fundamentally comprised of inputs (`sources`), outputs (`sinks`), and zero or more in-between `operations`. This style of programming is known as the [Pipe Programming Paradigm](http://blog.zillabyte.com/2014/05/14/the-pipe-programming-paradigm/) and it allows Zillabyte to scale your App in the cloud.
6
+
7
+ To learn more about Zillabyte Apps, please take a few moments and visit [docs.zillabyte.com](http://docs.zillabyte.com/)
8
+
9
+ ## Step 1: Understand your Data Options
10
+
11
+ `Sources` are how you get data into your App.
12
+
13
+ #### Source from Data Inside Zillabyte
14
+
15
+ You can store your datasets inside Zillabyte. In this regard, Zillabyte acts like a large database, ready to feed your App. Consider the following:
16
+
17
+ ```ruby
18
+ require "zillabyte"
19
+
20
+ # Create an app
21
+ app = Zillabyte.app("hello_world")
22
+
23
+ # Source from the 'web_pages' dataset
24
+ stream = app.source("web_pages")
25
+
26
+ # ...
27
+ ```
28
+
29
+ The above [example](http://docs.zillabyte.com/quickstart/hello_world/) will source data from the 'web_pages' dataset. This particular dataset happens to be an [Open Dataset](http://zillabyte.com/data) and is open to all users.
30
+
31
+ You are of course free to create private datasets that will not be accessible to the public. To do so, use the `zillabyte data` commands.
32
+
33
+ ```bash
34
+ $ zillabyte data --help
35
+ ```
36
+
37
+
38
+ #### Other Source Options
39
+
40
+ You are free to source from other datasets outside of Zillabyte. To do so, please see our documentation on [docs.zillabyte.com](http://docs.zillabyte.com).
41
+
42
+
43
+ ## Step 2: Build & Test your App
44
+
45
+ Once your app is built, you will want to test it locally. The `zillabyte test` is intended to help you identify bugs before submitting to the cloud. `zillabyte test` simulates the cloud environment on your local machine. In your App's directory, run the following:
46
+
47
+ ```bash
48
+ $ # in your app's directory
49
+ $ zillabyte test
50
+
51
+ inferring your app details...
52
+ app name: hello_world
53
+ app language: ruby
54
+ ==================== operation #0
55
+ name: source_1
56
+ type: source
57
+ #...
58
+ ```
59
+
60
+
61
+ ## Step 3: Submit your App to the Zillabyte Cloud
62
+
63
+ Once you are confident in your code, you can push to the cloud and run at scale. If you wish to push the entire directory, run the following:
64
+
65
+ ```bash
66
+ $ # in your app's directory
67
+ $ zillabyte push
68
+ packaging directory... done
69
+ uploading 31744 bytes...
70
+ #...
71
+ ```
72
+
73
+ The `zillabyte push` command may take a few minutes to complete. During this time, you will see log output coming from the cloud servers. You will notice your operations are being parallelized automatically. For example, if you had an operation named `each_1`, you will see it parallelized into `each_1.1`, `each_1.2`, `each_1.3`, and so forth.
74
+
75
+
76
+ ## Step 4: Examine the Your App's Logs
77
+
78
+ The `zillabyte push` will stop streaming logs once the push sequence is complete. If you wish to examine your app's logs further, run the `zillabyte logs` command
79
+
80
+ ```bash
81
+ $ zillabyte logs
82
+
83
+ Retrieving logs for flow #hello_world...please wait...
84
+ 2014-07-01T05:02:40.923+00:00 flow_1210[app] - [STARTUP] new app created.
85
+ 2014-07-01T05:02:42.563+00:00 flow_1210[app] - [STARTUP] registering app.
86
+ 2014-07-01T05:02:46.083+00:00 flow_1210[app] - [STARTUP] Beginning app deployment
87
+ 2014-07-01T05:02:48.915+00:00 flow_1210[flow] - [STARTUP] Pulling the code from our servers...
88
+ 2014-07-01T05:02:48.949+00:00 flow_1210[app] - [STARTUP] Initializing the app's environment...this might take a while.
89
+ 2014-07-01T05:03:06.775+00:00 flow_1210[flow] - [STARTUP] Performing the initial setup...
90
+ ```
91
+
92
+ The logs will stream indefinitely. To terminate, hit `ctrl+c`.
93
+
94
+ ## Step 5: Optionally Kill & Resubmit your App
95
+
96
+ If you notice a problem with your app, you can make the code change and `zillabyte push` right away. Only one instance of your app can be active at a time, so Zillabyte will automatically terminate your existing app before proceeding.
97
+
98
+ If you wish to kill your app without re-pushing, simple run the `kill` command.
99
+
100
+ ```bash
101
+ $ # in your app's directory
102
+ $ zillabyte apps:kill
103
+ ```
104
+
105
+ ## Step 6: Download Results
106
+
107
+ When your app is done processing, you will most likely have data to download from the cloud. This can be accomplished with the `zillabyte data:pull` command. For example:
108
+
109
+ ```bash
110
+ # list my datasets
111
+ $ zillabyte data
112
+ +------+---------------------+
113
+ | id | name |
114
+ +------+---------------------+
115
+ | 1516 | hello_worlds |
116
+ +------+---------------------+
117
+ ```
118
+
119
+ ```bash
120
+ # download the entire dataset to a local file
121
+ $ zillabyte data:pull hello_worlds my_local_file
122
+ ```
123
+
@@ -1,2 +1,5 @@
1
+ # This file is required by Zillabyte to understand your app. Most fields should
2
+ # be self-explanatory. For further information, please visit docs.zillabyte.com or
3
+ # shoot us a note at support@zillabyte.com.
1
4
  language: ruby
2
5
  script: app.rb
@@ -0,0 +1,81 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ actionpack (3.2.17)
5
+ activemodel (= 3.2.17)
6
+ activesupport (= 3.2.17)
7
+ builder (~> 3.0.0)
8
+ erubis (~> 2.7.0)
9
+ journey (~> 1.0.4)
10
+ rack (~> 1.4.5)
11
+ rack-cache (~> 1.2)
12
+ rack-test (~> 0.6.1)
13
+ sprockets (~> 2.2.1)
14
+ activemodel (3.2.17)
15
+ activesupport (= 3.2.17)
16
+ builder (~> 3.0.0)
17
+ activesupport (3.2.17)
18
+ i18n (~> 0.6, >= 0.6.4)
19
+ multi_json (~> 1.0)
20
+ ascii_charts (0.9.1)
21
+ aws-sdk (1.33.0)
22
+ json (~> 1.4)
23
+ nokogiri (>= 1.4.4)
24
+ uuidtools (~> 2.1)
25
+ builder (3.0.4)
26
+ chronic (0.10.2)
27
+ colorize (0.7.2)
28
+ erubis (2.7.0)
29
+ excon (0.33.0)
30
+ hike (1.2.3)
31
+ i18n (0.6.9)
32
+ indentation (0.1.1)
33
+ journey (1.0.4)
34
+ json (1.8.1)
35
+ mime-types (2.2)
36
+ mini_portile (0.5.3)
37
+ multi_json (1.9.2)
38
+ netrc (0.7.7)
39
+ nokogiri (1.6.1)
40
+ mini_portile (~> 0.5.0)
41
+ rack (1.4.5)
42
+ rack-cache (1.2)
43
+ rack (>= 0.4)
44
+ rack-test (0.6.2)
45
+ rack (>= 1.0)
46
+ rest-client (1.6.7)
47
+ mime-types (>= 1.16)
48
+ sprockets (2.2.2)
49
+ hike (~> 1.2)
50
+ multi_json (~> 1.0)
51
+ rack (~> 1.0)
52
+ tilt (~> 1.1, != 1.3.0)
53
+ terminal-table (1.4.5)
54
+ tilt (1.4.1)
55
+ time_difference (0.3.2)
56
+ activesupport
57
+ uuidtools (2.1.4)
58
+ zillabyte (0.0.20)
59
+ zillabyte-cli (~> 0.0.20)
60
+ zillabyte-cli (0.0.20)
61
+ actionpack (~> 3.2.13)
62
+ activesupport (~> 3.2.11)
63
+ ascii_charts (~> 0.9.1)
64
+ aws-sdk (~> 1.33.0)
65
+ bundler (~> 1.3)
66
+ chronic (~> 0.10)
67
+ colorize (~> 0.6)
68
+ excon (~> 0.31)
69
+ indentation (~> 0.1)
70
+ mini_portile (~> 0.5.0)
71
+ multi_json (~> 1.0)
72
+ netrc (~> 0.7.7)
73
+ rest-client (~> 1.6.1)
74
+ terminal-table (~> 1.4)
75
+ time_difference
76
+
77
+ PLATFORMS
78
+ ruby
79
+
80
+ DEPENDENCIES
81
+ zillabyte
@@ -153,6 +153,28 @@ module Zillabyte
153
153
  return nil
154
154
  end
155
155
 
156
+
157
+ def handle_downloading_manifest(file, res, type = nil)
158
+ if res["manifest"]
159
+ res["manifest"].each_with_index do |uri, index|
160
+
161
+ # Do we have multiple files? then suffix with numbers...
162
+ if res["manifest"].size <= 1
163
+ shard_file = file
164
+ else
165
+ shard_file = "#{File.basename(file, File.extname(file))}__#{'%04i' % index}#{File.extname(file)}"
166
+ end
167
+
168
+ display "downloading #{shard_file}" unless type == "json"
169
+
170
+ File.open(shard_file, "w") do |f|
171
+ f.write open(uri).read
172
+ end
173
+ end
174
+ else
175
+ error "expected remote server to return file manifest"
176
+ end
177
+ end
156
178
 
157
179
  end
158
180
  end
@@ -1,5 +1,5 @@
1
1
  module Zillabyte
2
2
  module CLI
3
- VERSION = "0.1.1"
3
+ VERSION = "0.1.2"
4
4
  end
5
5
  end
@@ -37,7 +37,6 @@ Gem::Specification.new do |spec|
37
37
  spec.add_dependency "chronic", "~> 0.10"
38
38
  spec.add_dependency "ascii_charts", "~> 0.9.1"
39
39
  spec.add_dependency "indentation", "~> 0.1"
40
- spec.add_dependency "aws-sdk", "~> 1.33.0"
41
40
  spec.add_dependency "time_difference"
42
41
  spec.add_dependency "mkfifo"
43
42