zillabyte-cli 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +14 -6
- data/lib/zillabyte/api/flows.rb +3 -1
- data/lib/zillabyte/cli/apps.rb +70 -71
- data/lib/zillabyte/cli/auth.rb +2 -4
- data/lib/zillabyte/cli/base.rb +2 -0
- data/lib/zillabyte/cli/components.rb +135 -47
- data/lib/zillabyte/cli/{relations.rb → data.rb} +70 -98
- data/lib/zillabyte/cli/flows.rb +50 -50
- data/lib/zillabyte/cli/git.rb +6 -7
- data/lib/zillabyte/cli/help.rb +3 -5
- data/lib/zillabyte/cli/keys.rb +18 -15
- data/lib/zillabyte/cli/nuke.rb +4 -6
- data/lib/zillabyte/cli/query.rb +26 -50
- data/lib/zillabyte/cli/repl.rb +6 -4
- data/lib/zillabyte/cli/templates/apps/ruby/README.md +123 -0
- data/lib/zillabyte/cli/templates/apps/ruby/zillabyte.conf.yaml +3 -0
- data/lib/zillabyte/cli/templates/ruby/Gemfile.lock +81 -0
- data/lib/zillabyte/helpers.rb +22 -0
- data/lib/zillabyte-cli/version.rb +1 -1
- data/zillabyte-cli.gemspec +0 -1
- metadata +50 -66
- data/lib/#zillabyte-cli.rb# +0 -5
- data/lib/zillabyte/cli/#logs.rb# +0 -12
- data/lib/zillabyte/cli/#repl.rb# +0 -43
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +0 -27
data/lib/zillabyte/cli/keys.rb
CHANGED
@@ -15,11 +15,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
15
15
|
|
16
16
|
# keys:add NAME [KEY]
|
17
17
|
#
|
18
|
-
#
|
18
|
+
# Add a key for the current user.
|
19
|
+
# If no KEY is specified, will try to find ~/.ssh/id_rsa.pub.
|
19
20
|
#
|
20
|
-
#
|
21
|
-
#
|
22
|
-
# --output_type OUTPUT_TYPE # The output format type
|
21
|
+
# --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
|
23
22
|
#
|
24
23
|
def add()
|
25
24
|
type = options[:output_type] || nil
|
@@ -43,9 +42,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
43
42
|
|
44
43
|
# keys:show NAME
|
45
44
|
#
|
46
|
-
#
|
45
|
+
# Show a key for the current user.
|
47
46
|
#
|
48
|
-
#
|
47
|
+
# --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
|
49
48
|
#
|
50
49
|
def show()
|
51
50
|
type = options[:output_type] || nil
|
@@ -64,11 +63,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
64
63
|
|
65
64
|
# keys:remove [NAME]
|
66
65
|
#
|
67
|
-
#
|
66
|
+
# Remove a key from the current user.
|
68
67
|
#
|
69
|
-
#
|
70
|
-
#
|
71
|
-
#
|
68
|
+
# -l, --long # Display extended information for each key
|
69
|
+
# -f, --force # Force -- remove key without prompting
|
70
|
+
# --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
|
72
71
|
#
|
73
72
|
def remove()
|
74
73
|
type = options[:output_type] || nil
|
@@ -127,10 +126,11 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
127
126
|
|
128
127
|
# keys
|
129
128
|
#
|
130
|
-
#
|
129
|
+
# Display keys for the current user.
|
130
|
+
#
|
131
|
+
# -l, --long # Display extended information for each key
|
132
|
+
# --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
|
131
133
|
#
|
132
|
-
# --output_type OUTPUT_TYPE # The output format type
|
133
|
-
# -l, --long # display extended information for each key
|
134
134
|
def index()
|
135
135
|
type = options[:output_type] || nil
|
136
136
|
long = options[:long]
|
@@ -144,9 +144,9 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
144
144
|
|
145
145
|
# keys:clear
|
146
146
|
#
|
147
|
-
#
|
147
|
+
# Remove all authentication keys from the current user.
|
148
148
|
#
|
149
|
-
#
|
149
|
+
# --output_type OUTPUT_TYPE # Specify an output format type i.e. json #HIDDEN
|
150
150
|
#
|
151
151
|
def clear()
|
152
152
|
type = options[:output_type] || nil
|
@@ -159,7 +159,10 @@ class Zillabyte::Command::Keys < Zillabyte::Command::Base
|
|
159
159
|
end
|
160
160
|
end
|
161
161
|
|
162
|
+
|
162
163
|
private
|
164
|
+
|
165
|
+
|
163
166
|
def format_keys(keys, long, type)
|
164
167
|
outlong = long || type == "json"
|
165
168
|
rows = keys.map() { |row|
|
data/lib/zillabyte/cli/nuke.rb
CHANGED
@@ -7,14 +7,12 @@ require "zillabyte/common"
|
|
7
7
|
#
|
8
8
|
class Zillabyte::Command::Nuke < Zillabyte::Command::Base
|
9
9
|
|
10
|
-
|
11
|
-
|
12
10
|
|
13
11
|
# apps:nuke
|
14
12
|
#
|
15
|
-
#
|
13
|
+
# Destroys all state associated with the current user.
|
16
14
|
#
|
17
|
-
# --force
|
15
|
+
# --force # This option must always be given
|
18
16
|
#
|
19
17
|
def index
|
20
18
|
|
@@ -34,9 +32,9 @@ class Zillabyte::Command::Nuke < Zillabyte::Command::Base
|
|
34
32
|
if options[:output_type] == "json"
|
35
33
|
display res.body.to_json
|
36
34
|
else
|
37
|
-
display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']}
|
35
|
+
display "Nuked #{res.body['deleted_apps']} app(s) and #{res.body['deleted_apps']} dataset(s)"
|
38
36
|
end
|
39
37
|
|
40
38
|
|
41
39
|
end
|
42
|
-
end
|
40
|
+
end
|
data/lib/zillabyte/cli/query.rb
CHANGED
@@ -9,12 +9,13 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
9
9
|
|
10
10
|
# query:sxp EXPRESSION
|
11
11
|
#
|
12
|
-
#
|
12
|
+
# Executes SXP queries against the zillabyte corpus.
|
13
13
|
#
|
14
|
-
# -o, --offset OFFSET
|
15
|
-
# -l, --limit LIMIT
|
16
|
-
# --
|
17
|
-
#
|
14
|
+
# -o, --offset OFFSET # Skips to the offset [default: 0]
|
15
|
+
# -l, --limit LIMIT # Sets the result limit [default: 20]
|
16
|
+
# -t, --tail TAIL # Continuously watches for new results
|
17
|
+
# --meta # Show meta columns (since, confidence, source)
|
18
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
18
19
|
#
|
19
20
|
#Examples:
|
20
21
|
#
|
@@ -26,6 +27,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
26
27
|
opts[:offset] = options[:offset] || 0
|
27
28
|
opts[:limit] = options[:limit] || 10
|
28
29
|
type = options[:output_type]
|
30
|
+
show_meta = options[:meta] || false
|
29
31
|
tail = options[:tail] || false
|
30
32
|
expression = options[:expression] || shift_argument
|
31
33
|
opts[:since] = options[:since]
|
@@ -79,6 +81,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
79
81
|
headings = []
|
80
82
|
if res['results'].first
|
81
83
|
headings = res['results'].first.keys
|
84
|
+
META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
|
82
85
|
rows = res['results'].each do |obj|
|
83
86
|
new_row = headings.map do |heading|
|
84
87
|
obj[heading]
|
@@ -110,14 +113,15 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
110
113
|
|
111
114
|
# query:sql EXPRESSION
|
112
115
|
#
|
113
|
-
#
|
116
|
+
# Executes SQL queries against the zillabyte corpus.
|
114
117
|
#
|
115
|
-
# -o, --offset OFFSET
|
116
|
-
# -l, --limit LIMIT
|
117
|
-
# -t, --tail TAIL
|
118
|
-
# -s, --since SINCE
|
119
|
-
# --no_truncation
|
120
|
-
# --
|
118
|
+
# -o, --offset OFFSET # Skips to the offset [default: 0]
|
119
|
+
# -l, --limit LIMIT # Sets the result limit [default: 20]
|
120
|
+
# -t, --tail TAIL # Continuously watches for new results
|
121
|
+
# -s, --since SINCE # Grab newer records since SINCE
|
122
|
+
# --no_truncation # Doesn't truncate long strings
|
123
|
+
# --meta # Show meta columns (since, confidence, source)
|
124
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
121
125
|
#
|
122
126
|
#Examples:
|
123
127
|
#
|
@@ -132,6 +136,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
132
136
|
tail = options[:tail] || false
|
133
137
|
expression = options[:expression] || shift_argument
|
134
138
|
opts[:since] = options[:since]
|
139
|
+
show_meta = options[:meta] || false
|
135
140
|
type = options[:output_type]
|
136
141
|
seen = {}
|
137
142
|
|
@@ -183,6 +188,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
183
188
|
filtered = []
|
184
189
|
if res['rows'].first
|
185
190
|
concrete_headings = res['rows'].first.keys
|
191
|
+
META_COLUMNS.each {|c| concrete_headings.delete c} if (!show_meta)
|
186
192
|
concrete_headings.each do |ch|
|
187
193
|
has_alias = false
|
188
194
|
col_aliases.each do |al|
|
@@ -242,9 +248,8 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
242
248
|
|
243
249
|
# query:pull QUERY FILE
|
244
250
|
#
|
245
|
-
#
|
251
|
+
# Executes a query and downloads the results to FILE.
|
246
252
|
#
|
247
|
-
# pulls query data to OUTPUT.gz
|
248
253
|
def pull
|
249
254
|
|
250
255
|
query = options[:query] || shift_argument
|
@@ -264,43 +269,12 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
264
269
|
)
|
265
270
|
res = res.body
|
266
271
|
|
267
|
-
|
268
|
-
display "Waiting for download." if type.nil?
|
269
|
-
File.open(file, "w") do |f|
|
270
|
-
f.write open(res["uri"]).read
|
271
|
-
end
|
272
|
-
elsif(res["s3_credentials"])
|
273
|
-
display "Pulling query results." if type.nil?
|
274
|
-
s3 = AWS::S3.new(res["s3_credentials"])
|
275
|
-
bucket = s3.buckets[res["s3_bucket"]]
|
276
|
-
obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
|
277
|
-
|
278
|
-
while(true)
|
279
|
-
keys = obj.collect(&:key)
|
280
|
-
if keys.length > 0 and keys.include?("#{res["s3_file_key"]}/manifest")
|
281
|
-
display "Starting to write to file..." if type.nil?
|
282
|
-
File.open(file, "w") do |f|
|
283
|
-
obj.each do |o|
|
284
|
-
if o.key == "#{res["s3_file_key"]}/manifest"
|
285
|
-
next
|
286
|
-
end
|
287
|
-
f.write(o.read)
|
288
|
-
end
|
289
|
-
end
|
290
|
-
break
|
291
|
-
else
|
292
|
-
sleep(3)
|
293
|
-
obj = bucket.objects.with_prefix("#{res["s3_file_key"]}/")
|
294
|
-
end
|
295
|
-
end
|
296
|
-
else
|
297
|
-
throw "bad response: #{res}"
|
298
|
-
end
|
272
|
+
handle_downloading_manifest(file, res, type)
|
299
273
|
|
300
274
|
if type == "json"
|
301
275
|
display "{}"
|
302
276
|
else
|
303
|
-
display "results
|
277
|
+
display "finished downloading results to file"
|
304
278
|
end
|
305
279
|
|
306
280
|
end
|
@@ -310,8 +284,10 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
310
284
|
|
311
285
|
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
|
312
286
|
#
|
313
|
-
#
|
314
|
-
#
|
287
|
+
# Executes a query and pulls data to S3_BUCKET/FILE_KEY/part***.gz.
|
288
|
+
#
|
289
|
+
# --output_type OUTPUT_TYPE # Specify an output type i.e. json #HIDDEN
|
290
|
+
#
|
315
291
|
def pull_to_s3
|
316
292
|
|
317
293
|
query = options[:query] || shift_argument
|
@@ -337,7 +313,7 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
337
313
|
display {}.to_json
|
338
314
|
else
|
339
315
|
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
340
|
-
display "if the
|
316
|
+
display "if the dataset is large, this may take a while, please check your s3 account after a few minutes"
|
341
317
|
end
|
342
318
|
end
|
343
319
|
alias_command "query:pull:s3", "query:pull_to_s3"
|
data/lib/zillabyte/cli/repl.rb
CHANGED
@@ -7,9 +7,11 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
|
|
7
7
|
|
8
8
|
# repl
|
9
9
|
#
|
10
|
-
#
|
11
|
-
#
|
12
|
-
# --
|
10
|
+
# Start a console session for zillabyte.
|
11
|
+
#
|
12
|
+
# --quiet # HIDDEN
|
13
|
+
# --history HISTORY # HIDDEN hack to allow history for readline
|
14
|
+
#
|
13
15
|
def index
|
14
16
|
if !options[:quiet]
|
15
17
|
Zillabyte::Command.run("version")
|
@@ -29,7 +31,7 @@ class Zillabyte::Command::Repl < Zillabyte::Command::Base
|
|
29
31
|
Readline::HISTORY << his
|
30
32
|
end
|
31
33
|
end
|
32
|
-
# TODO: Add tab completion for basic commands, app/
|
34
|
+
# TODO: Add tab completion for basic commands, app/dataset names etc.
|
33
35
|
buf = ""
|
34
36
|
while cmd = Readline.readline(prompt, true)
|
35
37
|
if cmd && cmd.length > 0
|
@@ -0,0 +1,123 @@
|
|
1
|
+
# Getting Started with Zillabyte Apps
|
2
|
+
|
3
|
+
Zillabyte Apps are chunks of code that can scalably analyze almost any type of data. You develop & test the app locally and then push to the Cloud when you are ready to scale.
|
4
|
+
|
5
|
+
Apps are fundamentally comprised of inputs (`sources`), outputs (`sinks`), and zero or more in-between `operations`. This style of programming is known as the [Pipe Programming Paradigm](http://blog.zillabyte.com/2014/05/14/the-pipe-programming-paradigm/) and it allows Zillabyte to scale your App in the cloud.
|
6
|
+
|
7
|
+
To learn more about Zillabyte Apps, please take a few moments and visit [docs.zillabyte.com](http://docs.zillabyte.com/)
|
8
|
+
|
9
|
+
## Step 1: Understand your Data Options
|
10
|
+
|
11
|
+
`Sources` are how you get data into your App.
|
12
|
+
|
13
|
+
#### Source from Data Inside Zillabyte
|
14
|
+
|
15
|
+
You can store your datasets inside Zillabyte. In this regard, Zillabyte acts like a large database, ready to feed your App. Consider the following:
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
require "zillabyte"
|
19
|
+
|
20
|
+
# Create an app
|
21
|
+
app = Zillabyte.app("hello_world")
|
22
|
+
|
23
|
+
# Source from the 'web_pages' dataset
|
24
|
+
stream = app.source("web_pages")
|
25
|
+
|
26
|
+
# ...
|
27
|
+
```
|
28
|
+
|
29
|
+
The above [example](http://docs.zillabyte.com/quickstart/hello_world/) will source data from the 'web_pages' dataset. This particular dataset happens to be an [Open Dataset](http://zillabyte.com/data) and is open to all users.
|
30
|
+
|
31
|
+
You are of course free to create private datasets that will not be accessible to the public. To do so, use the `zillabyte data` commands.
|
32
|
+
|
33
|
+
```bash
|
34
|
+
$ zillabyte data --help
|
35
|
+
```
|
36
|
+
|
37
|
+
|
38
|
+
#### Other Source Options
|
39
|
+
|
40
|
+
You are free to source from other datasets outside of Zillabyte. To do so, please see our documentation on [docs.zillabyte.com](http://docs.zillabyte.com).
|
41
|
+
|
42
|
+
|
43
|
+
## Step 2: Build & Test your App
|
44
|
+
|
45
|
+
Once your app is built, you will want to test it locally. The `zillabyte test` is intended to help you identify bugs before submitting to the cloud. `zillabyte test` simulates the cloud environment on your local machine. In your App's directory, run the following:
|
46
|
+
|
47
|
+
```bash
|
48
|
+
$ # in your app's directory
|
49
|
+
$ zillabyte test
|
50
|
+
|
51
|
+
inferring your app details...
|
52
|
+
app name: hello_world
|
53
|
+
app language: ruby
|
54
|
+
==================== operation #0
|
55
|
+
name: source_1
|
56
|
+
type: source
|
57
|
+
#...
|
58
|
+
```
|
59
|
+
|
60
|
+
|
61
|
+
## Step 3: Submit your App to the Zillabyte Cloud
|
62
|
+
|
63
|
+
Once you are confident in your code, you can push to the cloud and run at scale. If you wish to push the entire directory, run the following:
|
64
|
+
|
65
|
+
```bash
|
66
|
+
$ # in your app's directory
|
67
|
+
$ zillabyte push
|
68
|
+
packaging directory... done
|
69
|
+
uploading 31744 bytes...
|
70
|
+
#...
|
71
|
+
```
|
72
|
+
|
73
|
+
The `zillabyte push` command may take a few minutes to complete. During this time, you will see log output coming from the cloud servers. You will notice your operations are being parallelized automatically. For example, if you had an operation named `each_1`, you will see it parallelized into `each_1.1`, `each_1.2`, `each_1.3`, and so forth.
|
74
|
+
|
75
|
+
|
76
|
+
## Step 4: Examine the Your App's Logs
|
77
|
+
|
78
|
+
The `zillabyte push` will stop streaming logs once the push sequence is complete. If you wish to examine your app's logs further, run the `zillabyte logs` command
|
79
|
+
|
80
|
+
```bash
|
81
|
+
$ zillabyte logs
|
82
|
+
|
83
|
+
Retrieving logs for flow #hello_world...please wait...
|
84
|
+
2014-07-01T05:02:40.923+00:00 flow_1210[app] - [STARTUP] new app created.
|
85
|
+
2014-07-01T05:02:42.563+00:00 flow_1210[app] - [STARTUP] registering app.
|
86
|
+
2014-07-01T05:02:46.083+00:00 flow_1210[app] - [STARTUP] Beginning app deployment
|
87
|
+
2014-07-01T05:02:48.915+00:00 flow_1210[flow] - [STARTUP] Pulling the code from our servers...
|
88
|
+
2014-07-01T05:02:48.949+00:00 flow_1210[app] - [STARTUP] Initializing the app's environment...this might take a while.
|
89
|
+
2014-07-01T05:03:06.775+00:00 flow_1210[flow] - [STARTUP] Performing the initial setup...
|
90
|
+
```
|
91
|
+
|
92
|
+
The logs will stream indefinitely. To terminate, hit `ctrl+c`.
|
93
|
+
|
94
|
+
## Step 5: Optionally Kill & Resubmit your App
|
95
|
+
|
96
|
+
If you notice a problem with your app, you can make the code change and `zillabyte push` right away. Only one instance of your app can be active at a time, so Zillabyte will automatically terminate your existing app before proceeding.
|
97
|
+
|
98
|
+
If you wish to kill your app without re-pushing, simple run the `kill` command.
|
99
|
+
|
100
|
+
```bash
|
101
|
+
$ # in your app's directory
|
102
|
+
$ zillabyte apps:kill
|
103
|
+
```
|
104
|
+
|
105
|
+
## Step 6: Download Results
|
106
|
+
|
107
|
+
When your app is done processing, you will most likely have data to download from the cloud. This can be accomplished with the `zillabyte data:pull` command. For example:
|
108
|
+
|
109
|
+
```bash
|
110
|
+
# list my datasets
|
111
|
+
$ zillabyte data
|
112
|
+
+------+---------------------+
|
113
|
+
| id | name |
|
114
|
+
+------+---------------------+
|
115
|
+
| 1516 | hello_worlds |
|
116
|
+
+------+---------------------+
|
117
|
+
```
|
118
|
+
|
119
|
+
```bash
|
120
|
+
# download the entire dataset to a local file
|
121
|
+
$ zillabyte data:pull hello_worlds my_local_file
|
122
|
+
```
|
123
|
+
|
@@ -0,0 +1,81 @@
|
|
1
|
+
GEM
|
2
|
+
remote: https://rubygems.org/
|
3
|
+
specs:
|
4
|
+
actionpack (3.2.17)
|
5
|
+
activemodel (= 3.2.17)
|
6
|
+
activesupport (= 3.2.17)
|
7
|
+
builder (~> 3.0.0)
|
8
|
+
erubis (~> 2.7.0)
|
9
|
+
journey (~> 1.0.4)
|
10
|
+
rack (~> 1.4.5)
|
11
|
+
rack-cache (~> 1.2)
|
12
|
+
rack-test (~> 0.6.1)
|
13
|
+
sprockets (~> 2.2.1)
|
14
|
+
activemodel (3.2.17)
|
15
|
+
activesupport (= 3.2.17)
|
16
|
+
builder (~> 3.0.0)
|
17
|
+
activesupport (3.2.17)
|
18
|
+
i18n (~> 0.6, >= 0.6.4)
|
19
|
+
multi_json (~> 1.0)
|
20
|
+
ascii_charts (0.9.1)
|
21
|
+
aws-sdk (1.33.0)
|
22
|
+
json (~> 1.4)
|
23
|
+
nokogiri (>= 1.4.4)
|
24
|
+
uuidtools (~> 2.1)
|
25
|
+
builder (3.0.4)
|
26
|
+
chronic (0.10.2)
|
27
|
+
colorize (0.7.2)
|
28
|
+
erubis (2.7.0)
|
29
|
+
excon (0.33.0)
|
30
|
+
hike (1.2.3)
|
31
|
+
i18n (0.6.9)
|
32
|
+
indentation (0.1.1)
|
33
|
+
journey (1.0.4)
|
34
|
+
json (1.8.1)
|
35
|
+
mime-types (2.2)
|
36
|
+
mini_portile (0.5.3)
|
37
|
+
multi_json (1.9.2)
|
38
|
+
netrc (0.7.7)
|
39
|
+
nokogiri (1.6.1)
|
40
|
+
mini_portile (~> 0.5.0)
|
41
|
+
rack (1.4.5)
|
42
|
+
rack-cache (1.2)
|
43
|
+
rack (>= 0.4)
|
44
|
+
rack-test (0.6.2)
|
45
|
+
rack (>= 1.0)
|
46
|
+
rest-client (1.6.7)
|
47
|
+
mime-types (>= 1.16)
|
48
|
+
sprockets (2.2.2)
|
49
|
+
hike (~> 1.2)
|
50
|
+
multi_json (~> 1.0)
|
51
|
+
rack (~> 1.0)
|
52
|
+
tilt (~> 1.1, != 1.3.0)
|
53
|
+
terminal-table (1.4.5)
|
54
|
+
tilt (1.4.1)
|
55
|
+
time_difference (0.3.2)
|
56
|
+
activesupport
|
57
|
+
uuidtools (2.1.4)
|
58
|
+
zillabyte (0.0.20)
|
59
|
+
zillabyte-cli (~> 0.0.20)
|
60
|
+
zillabyte-cli (0.0.20)
|
61
|
+
actionpack (~> 3.2.13)
|
62
|
+
activesupport (~> 3.2.11)
|
63
|
+
ascii_charts (~> 0.9.1)
|
64
|
+
aws-sdk (~> 1.33.0)
|
65
|
+
bundler (~> 1.3)
|
66
|
+
chronic (~> 0.10)
|
67
|
+
colorize (~> 0.6)
|
68
|
+
excon (~> 0.31)
|
69
|
+
indentation (~> 0.1)
|
70
|
+
mini_portile (~> 0.5.0)
|
71
|
+
multi_json (~> 1.0)
|
72
|
+
netrc (~> 0.7.7)
|
73
|
+
rest-client (~> 1.6.1)
|
74
|
+
terminal-table (~> 1.4)
|
75
|
+
time_difference
|
76
|
+
|
77
|
+
PLATFORMS
|
78
|
+
ruby
|
79
|
+
|
80
|
+
DEPENDENCIES
|
81
|
+
zillabyte
|
data/lib/zillabyte/helpers.rb
CHANGED
@@ -153,6 +153,28 @@ module Zillabyte
|
|
153
153
|
return nil
|
154
154
|
end
|
155
155
|
|
156
|
+
|
157
|
+
def handle_downloading_manifest(file, res, type = nil)
|
158
|
+
if res["manifest"]
|
159
|
+
res["manifest"].each_with_index do |uri, index|
|
160
|
+
|
161
|
+
# Do we have multiple files? then suffix with numbers...
|
162
|
+
if res["manifest"].size <= 1
|
163
|
+
shard_file = file
|
164
|
+
else
|
165
|
+
shard_file = "#{File.basename(file, File.extname(file))}__#{'%04i' % index}#{File.extname(file)}"
|
166
|
+
end
|
167
|
+
|
168
|
+
display "downloading #{shard_file}" unless type == "json"
|
169
|
+
|
170
|
+
File.open(shard_file, "w") do |f|
|
171
|
+
f.write open(uri).read
|
172
|
+
end
|
173
|
+
end
|
174
|
+
else
|
175
|
+
error "expected remote server to return file manifest"
|
176
|
+
end
|
177
|
+
end
|
156
178
|
|
157
179
|
end
|
158
180
|
end
|
data/zillabyte-cli.gemspec
CHANGED
@@ -37,7 +37,6 @@ Gem::Specification.new do |spec|
|
|
37
37
|
spec.add_dependency "chronic", "~> 0.10"
|
38
38
|
spec.add_dependency "ascii_charts", "~> 0.9.1"
|
39
39
|
spec.add_dependency "indentation", "~> 0.1"
|
40
|
-
spec.add_dependency "aws-sdk", "~> 1.33.0"
|
41
40
|
spec.add_dependency "time_difference"
|
42
41
|
spec.add_dependency "mkfifo"
|
43
42
|
|