s3stream 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +12 -0
- data/bin/s3stream +7 -0
- data/lib/s3stream/main.rb +5 -28
- data/lib/s3stream/upload.rb +77 -0
- data/lib/s3stream/version.rb +1 -1
- data/lib/s3stream.rb +1 -6
- metadata +13 -12
data/README
CHANGED
@@ -33,6 +33,10 @@ Usage
|
|
33
33
|
% s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
|
34
34
|
% s3stream store mybucket myfile.dat < /tmp/myfile.dat
|
35
35
|
|
36
|
+
If you want to try uploading a bunch of data, but don't have any handy:
|
37
|
+
|
38
|
+
% base64 /dev/urandom | head -c $((10 * 1024 * 1024)) | s3stream store mybucket 10mb.txt
|
39
|
+
|
36
40
|
And if you want to get fancy:
|
37
41
|
|
38
42
|
% cat \
|
@@ -40,3 +44,11 @@ And if you want to get fancy:
|
|
40
44
|
<(s3stream fetch mybucket log1.log.gz | gunzip) \
|
41
45
|
| lzop -9 \
|
42
46
|
| s3stream store mybucket combined.log.lzo
|
47
|
+
|
48
|
+
|
49
|
+
If you want to stream an upload in ruby:
|
50
|
+
|
51
|
+
S3Stream::Upload.to(:s3object => s3object) do |stream|
|
52
|
+
stream.write("abc")
|
53
|
+
stream.write("123")
|
54
|
+
end
|
data/bin/s3stream
CHANGED
data/lib/s3stream/main.rb
CHANGED
@@ -18,36 +18,13 @@ module S3Stream
|
|
18
18
|
def store(bucket_name, filename)
|
19
19
|
$stdout.sync = true
|
20
20
|
object = s3object(bucket_name, filename)
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
(1..MAX_CHUNKS).each do |chunk|
|
27
|
-
if $stdin.eof?
|
28
|
-
puts "End of input."
|
29
|
-
break
|
30
|
-
end
|
31
|
-
if chunk < MAX_CHUNKS
|
32
|
-
print "Buffering input (up to #{buffer_size} bytes) ... "
|
33
|
-
$stdin.read(buffer_size, buffer)
|
34
|
-
puts "done."
|
35
|
-
print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
|
36
|
-
upload.add_part(buffer)
|
37
|
-
puts "done."
|
38
|
-
buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
|
39
|
-
else
|
40
|
-
print "Last part (#{chunk})! Buffering input (unlimited) ..."
|
41
|
-
buffer = $stdin.read
|
42
|
-
puts "done."
|
43
|
-
print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
|
44
|
-
upload.add_part(buffer)
|
45
|
-
puts "done."
|
46
|
-
end
|
47
|
-
total += buffer.size
|
21
|
+
S3Stream::Upload.to(:s3object => object, :log_to => $stdout) do |out|
|
22
|
+
buffer = ""
|
23
|
+
until $stdin.eof?
|
24
|
+
$stdin.readpartial(4096, buffer)
|
25
|
+
out.write(buffer)
|
48
26
|
end
|
49
27
|
end
|
50
|
-
puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
|
51
28
|
end
|
52
29
|
|
53
30
|
private
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module S3Stream
|
2
|
+
class Upload
|
3
|
+
|
4
|
+
# S3Stream::Upload.to(:s3object => s3object, :log_to => $stdout) do |out|
|
5
|
+
# out.write("abc")
|
6
|
+
# out.write("123")
|
7
|
+
# end
|
8
|
+
def self.to(args={})
|
9
|
+
stream = new(args)
|
10
|
+
begin
|
11
|
+
yield stream
|
12
|
+
stream.close
|
13
|
+
rescue => e
|
14
|
+
stream.cancel
|
15
|
+
raise
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
attr_reader :size
|
20
|
+
|
21
|
+
def initialize(args={})
|
22
|
+
@s3object = args[:s3object] || raise("Missing :s3object")
|
23
|
+
@log = args[:log_to] # optional
|
24
|
+
reset
|
25
|
+
@upload = @s3object.multipart_upload
|
26
|
+
end
|
27
|
+
|
28
|
+
def write(data)
|
29
|
+
@buffer << data
|
30
|
+
flush if @chunk < S3Stream::MAX_CHUNKS && @buffer.size >= @buffer_size
|
31
|
+
@size += data.size
|
32
|
+
end
|
33
|
+
|
34
|
+
def close
|
35
|
+
flush if @buffer.size > 0
|
36
|
+
unless @upload.close.nil?
|
37
|
+
log "Done uploading #{size} bytes to #{location}."
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def cancel
|
42
|
+
@upload.abort unless @upload.nil?
|
43
|
+
reset
|
44
|
+
log "Canceled upload to #{location}."
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def location
|
50
|
+
"s3://#{@s3object.bucket.name}/#{@s3object.key}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def reset
|
54
|
+
@buffer = ""
|
55
|
+
@buffer_size = S3Stream::INITIAL_BUFFER_SIZE
|
56
|
+
@size = 0
|
57
|
+
@chunk = 1
|
58
|
+
@upload = nil
|
59
|
+
end
|
60
|
+
|
61
|
+
def flush
|
62
|
+
log "Uploading part #{@chunk} (#{@buffer.size} bytes)."
|
63
|
+
@upload.add_part(@buffer)
|
64
|
+
@buffer_size = (@buffer_size * S3Stream::BUFFER_GROWTH_FACTOR).to_i
|
65
|
+
@buffer.clear
|
66
|
+
@chunk += 1
|
67
|
+
nil
|
68
|
+
end
|
69
|
+
|
70
|
+
def log(msg)
|
71
|
+
unless @log.nil?
|
72
|
+
@log.puts(msg)
|
73
|
+
@log.flush
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/lib/s3stream/version.rb
CHANGED
data/lib/s3stream.rb
CHANGED
@@ -2,11 +2,6 @@ require "thor"
|
|
2
2
|
require "aws-sdk"
|
3
3
|
require "s3stream/version"
|
4
4
|
require "s3stream/constants"
|
5
|
+
require "s3stream/upload"
|
5
6
|
require "s3stream/main"
|
6
7
|
require "s3stream/credentials"
|
7
|
-
|
8
|
-
if S3Stream::CREDENTIALS.nil?
|
9
|
-
puts "Environmental variables must be set."
|
10
|
-
puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
|
11
|
-
exit(1)
|
12
|
-
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3stream
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-03-
|
12
|
+
date: 2012-03-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: thor
|
16
|
-
requirement: &
|
16
|
+
requirement: &70361652001540 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 0.14.6
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70361652001540
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: aws-sdk
|
27
|
-
requirement: &
|
27
|
+
requirement: &70361651998340 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 1.3.5
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70361651998340
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: pry
|
38
|
-
requirement: &
|
38
|
+
requirement: &70361652012160 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :development
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70361652012160
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: pry-doc
|
49
|
-
requirement: &
|
49
|
+
requirement: &70361652011060 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70361652011060
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: pry-nav
|
60
|
-
requirement: &
|
60
|
+
requirement: &70361652010500 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,7 +65,7 @@ dependencies:
|
|
65
65
|
version: '0'
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70361652010500
|
69
69
|
description: Stream files on S3 to stdout or from stdin
|
70
70
|
email:
|
71
71
|
- chris@kindkid.com
|
@@ -85,6 +85,7 @@ files:
|
|
85
85
|
- lib/s3stream/constants.rb
|
86
86
|
- lib/s3stream/credentials.rb
|
87
87
|
- lib/s3stream/main.rb
|
88
|
+
- lib/s3stream/upload.rb
|
88
89
|
- lib/s3stream/version.rb
|
89
90
|
- s3stream.gemspec
|
90
91
|
homepage: https://github.com/kindkid/s3stream
|