s3stream 0.0.6 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README CHANGED
@@ -33,6 +33,10 @@ Usage
33
33
  % s3stream fetch mybucket myfile.dat > /tmp/myfile.dat
34
34
  % s3stream store mybucket myfile.dat < /tmp/myfile.dat
35
35
 
36
+ If you want to try uploading a bunch of data, but don't have any handy:
37
+
38
+ % base64 /dev/urandom | head -c $((10 * 1024 * 1024)) | s3stream store mybucket 10mb.txt
39
+
36
40
  And if you want to get fancy:
37
41
 
38
42
  % cat \
@@ -40,3 +44,11 @@ And if you want to get fancy:
40
44
  <(s3stream fetch mybucket log1.log.gz | gunzip) \
41
45
  | lzop -9 \
42
46
  | s3stream store mybucket combined.log.lzo
47
+
48
+
49
+ If you want to stream an upload in ruby:
50
+
51
+ S3Stream::Upload.to(:s3object => s3object) do |stream|
52
+ stream.write("abc")
53
+ stream.write("123")
54
+ end
data/bin/s3stream CHANGED
@@ -1,4 +1,11 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  require "s3stream"
4
+
5
+ if S3Stream::CREDENTIALS.nil?
6
+ puts "Environmental variables must be set."
7
+ puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
8
+ exit(1)
9
+ end
10
+
4
11
  S3Stream::Main.start
data/lib/s3stream/main.rb CHANGED
@@ -18,36 +18,13 @@ module S3Stream
18
18
  def store(bucket_name, filename)
19
19
  $stdout.sync = true
20
20
  object = s3object(bucket_name, filename)
21
- buffer = ""
22
- total = 0
23
- buffer_size = INITIAL_BUFFER_SIZE
24
- puts "Uploading, please be patient."
25
- object.multipart_upload do |upload|
26
- (1..MAX_CHUNKS).each do |chunk|
27
- if $stdin.eof?
28
- puts "End of input."
29
- break
30
- end
31
- if chunk < MAX_CHUNKS
32
- print "Buffering input (up to #{buffer_size} bytes) ... "
33
- $stdin.read(buffer_size, buffer)
34
- puts "done."
35
- print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
36
- upload.add_part(buffer)
37
- puts "done."
38
- buffer_size = (buffer_size * BUFFER_GROWTH_FACTOR).to_i
39
- else
40
- print "Last part (#{chunk})! Buffering input (unlimited) ..."
41
- buffer = $stdin.read
42
- puts "done."
43
- print "Uploading part #{chunk} (#{buffer.size} bytes) ... "
44
- upload.add_part(buffer)
45
- puts "done."
46
- end
47
- total += buffer.size
21
+ S3Stream::Upload.to(:s3object => object, :log_to => $stdout) do |out|
22
+ buffer = ""
23
+ until $stdin.eof?
24
+ $stdin.readpartial(4096, buffer)
25
+ out.write(buffer)
48
26
  end
49
27
  end
50
- puts "Done uploading to s3://#{bucket_name}/#{filename} (#{total} bytes)"
51
28
  end
52
29
 
53
30
  private
@@ -0,0 +1,77 @@
1
+ module S3Stream
2
+ class Upload
3
+
4
+ # S3Stream::Upload.to(:s3object => s3object, :log_to => $stdout) do |out|
5
+ # out.write("abc")
6
+ # out.write("123")
7
+ # end
8
+ def self.to(args={})
9
+ stream = new(args)
10
+ begin
11
+ yield stream
12
+ stream.close
13
+ rescue => e
14
+ stream.cancel
15
+ raise
16
+ end
17
+ end
18
+
19
+ attr_reader :size
20
+
21
+ def initialize(args={})
22
+ @s3object = args[:s3object] || raise("Missing :s3object")
23
+ @log = args[:log_to] # optional
24
+ reset
25
+ @upload = @s3object.multipart_upload
26
+ end
27
+
28
+ def write(data)
29
+ @buffer << data
30
+ flush if @chunk < S3Stream::MAX_CHUNKS && @buffer.size >= @buffer_size
31
+ @size += data.size
32
+ end
33
+
34
+ def close
35
+ flush if @buffer.size > 0
36
+ unless @upload.close.nil?
37
+ log "Done uploading #{size} bytes to #{location}."
38
+ end
39
+ end
40
+
41
+ def cancel
42
+ @upload.abort unless @upload.nil?
43
+ reset
44
+ log "Canceled upload to #{location}."
45
+ end
46
+
47
+ private
48
+
49
+ def location
50
+ "s3://#{@s3object.bucket.name}/#{@s3object.key}"
51
+ end
52
+
53
+ def reset
54
+ @buffer = ""
55
+ @buffer_size = S3Stream::INITIAL_BUFFER_SIZE
56
+ @size = 0
57
+ @chunk = 1
58
+ @upload = nil
59
+ end
60
+
61
+ def flush
62
+ log "Uploading part #{@chunk} (#{@buffer.size} bytes)."
63
+ @upload.add_part(@buffer)
64
+ @buffer_size = (@buffer_size * S3Stream::BUFFER_GROWTH_FACTOR).to_i
65
+ @buffer.clear
66
+ @chunk += 1
67
+ nil
68
+ end
69
+
70
+ def log(msg)
71
+ unless @log.nil?
72
+ @log.puts(msg)
73
+ @log.flush
74
+ end
75
+ end
76
+ end
77
+ end
@@ -1,3 +1,3 @@
1
1
  module S3Stream
2
- VERSION = "0.0.6"
2
+ VERSION = "0.0.7"
3
3
  end
data/lib/s3stream.rb CHANGED
@@ -2,11 +2,6 @@ require "thor"
2
2
  require "aws-sdk"
3
3
  require "s3stream/version"
4
4
  require "s3stream/constants"
5
+ require "s3stream/upload"
5
6
  require "s3stream/main"
6
7
  require "s3stream/credentials"
7
-
8
- if S3Stream::CREDENTIALS.nil?
9
- puts "Environmental variables must be set."
10
- puts "See https://github.com/kindkid/s3stream/blob/v#{S3Stream::VERSION}/README"
11
- exit(1)
12
- end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3stream
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.0.7
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-03-01 00:00:00.000000000 Z
12
+ date: 2012-03-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: thor
16
- requirement: &70192394775500 !ruby/object:Gem::Requirement
16
+ requirement: &70361652001540 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,10 +21,10 @@ dependencies:
21
21
  version: 0.14.6
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70192394775500
24
+ version_requirements: *70361652001540
25
25
  - !ruby/object:Gem::Dependency
26
26
  name: aws-sdk
27
- requirement: &70192394790820 !ruby/object:Gem::Requirement
27
+ requirement: &70361651998340 !ruby/object:Gem::Requirement
28
28
  none: false
29
29
  requirements:
30
30
  - - ~>
@@ -32,10 +32,10 @@ dependencies:
32
32
  version: 1.3.5
33
33
  type: :runtime
34
34
  prerelease: false
35
- version_requirements: *70192394790820
35
+ version_requirements: *70361651998340
36
36
  - !ruby/object:Gem::Dependency
37
37
  name: pry
38
- requirement: &70192394788600 !ruby/object:Gem::Requirement
38
+ requirement: &70361652012160 !ruby/object:Gem::Requirement
39
39
  none: false
40
40
  requirements:
41
41
  - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
43
43
  version: '0'
44
44
  type: :development
45
45
  prerelease: false
46
- version_requirements: *70192394788600
46
+ version_requirements: *70361652012160
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: pry-doc
49
- requirement: &70192394787200 !ruby/object:Gem::Requirement
49
+ requirement: &70361652011060 !ruby/object:Gem::Requirement
50
50
  none: false
51
51
  requirements:
52
52
  - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
54
54
  version: '0'
55
55
  type: :development
56
56
  prerelease: false
57
- version_requirements: *70192394787200
57
+ version_requirements: *70361652011060
58
58
  - !ruby/object:Gem::Dependency
59
59
  name: pry-nav
60
- requirement: &70192394786080 !ruby/object:Gem::Requirement
60
+ requirement: &70361652010500 !ruby/object:Gem::Requirement
61
61
  none: false
62
62
  requirements:
63
63
  - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
65
65
  version: '0'
66
66
  type: :development
67
67
  prerelease: false
68
- version_requirements: *70192394786080
68
+ version_requirements: *70361652010500
69
69
  description: Stream files on S3 to stdout or from stdin
70
70
  email:
71
71
  - chris@kindkid.com
@@ -85,6 +85,7 @@ files:
85
85
  - lib/s3stream/constants.rb
86
86
  - lib/s3stream/credentials.rb
87
87
  - lib/s3stream/main.rb
88
+ - lib/s3stream/upload.rb
88
89
  - lib/s3stream/version.rb
89
90
  - s3stream.gemspec
90
91
  homepage: https://github.com/kindkid/s3stream