s33r 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE.txt +9 -0
- data/MIT-LICENSE +21 -0
- data/README.txt +16 -0
- data/bin/config.yml +5 -0
- data/bin/s3cli.rb +96 -0
- data/lib/s33r.rb +7 -0
- data/lib/s33r/client.rb +188 -0
- data/lib/s33r/core.rb +131 -0
- data/lib/s33r/external/mimetypes.rb +1558 -0
- data/lib/s33r/list_bucket_result.rb +1 -0
- data/lib/s33r/named_bucket.rb +28 -0
- data/lib/s33r/net_http_overrides.rb +42 -0
- data/lib/s33r/s3_exception.rb +20 -0
- data/test/spec/spec_core.rb +87 -0
- metadata +61 -0
data/LICENSE.txt
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
This software is distributed under the MIT license (see MIT-LICENSE).
|
2
|
+
|
3
|
+
It incorporates code from the mimetypes library (http://raa.ruby-lang.org/project/mime-types/),
|
4
|
+
which is under a compatible license (same as Ruby).
|
5
|
+
|
6
|
+
It is also heavily based on the sample Ruby code provided by Amazon
|
7
|
+
(http://developer.amazonwebservices.com/connect/entry.jspa?externalID=135&categoryID=47).
|
8
|
+
|
9
|
+
I picked up a couple of ideas from http://rubyforge.org/projects/rsh3ll/ too.
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
Copyright (C) 2006 Elliot Smith
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
21
|
+
|
data/README.txt
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
This is a first pass at a Ruby library for talking to Amazon's S3 service.
|
2
|
+
|
3
|
+
It is less complete than my previous effort (http://townx.org/blog/elliot/s3_rails), but more solid, more elegant,
|
4
|
+
and with a test suite (which uses RSpec - see http://rubyforge.org/projects/rspec - I haven't added RSpec as a gem
|
5
|
+
dependency (you can ignore the tests if you like), but for reference I developed the tests using 0.5.12).
|
6
|
+
|
7
|
+
To use it from inside your Ruby script:
|
8
|
+
|
9
|
+
require 's33r'
|
10
|
+
|
11
|
+
If you want to see an example program which uses the library, have a look at bin/s3cli.rb. This is a simple command
|
12
|
+
line client which you can use to post a file to S3 and email a link to the file to someone. Useful for sending
|
13
|
+
big files to people without clogging their email inbox with enormous files. It is also intended as a demonstration
|
14
|
+
of how to use the library. Full instructions are included at the top of the file.
|
15
|
+
|
16
|
+
By the way, I've tested this on Linux, but not on Windows or Mac.
|
data/bin/config.yml
ADDED
data/bin/s3cli.rb
ADDED
@@ -0,0 +1,96 @@
|
|
1
|
+
# SIMPLE COMMAND LINE S3 CLIENT
|
2
|
+
# You need an Amazon S3 account to use this: get one at http://aws.amazon.com/
|
3
|
+
#
|
4
|
+
# Create a config. file called based on the config.yml example in this directory;
|
5
|
+
# then change the config_file setting below to set the path to your config. file.
|
6
|
+
# You will be prompted to create a bucket if your setting for bucket in the config.
|
7
|
+
# file is blank.
|
8
|
+
#
|
9
|
+
# Note that this script doesn't give you any means of deleting anything; you will
|
10
|
+
# need to write your own program to do this (I use my own s33r server,
|
11
|
+
# which I am currently re-implementing to work with this library).
|
12
|
+
#
|
13
|
+
# Call this script with two arguments:
|
14
|
+
# filename: file to upload; the file is uploaded to S3 and assigned a key
|
15
|
+
# based on filename supplied (including path)
|
16
|
+
# to_email: email address to send to (optional)
|
17
|
+
|
18
|
+
require 'yaml'
|
19
|
+
require 'rubygems'
|
20
|
+
require_gem 's33r'
|
21
|
+
require 'net/smtp'
|
22
|
+
|
23
|
+
filename = ARGV[0]
|
24
|
+
to_email = ARGV[1]
|
25
|
+
config_file = '/home/ell/.s33r'
|
26
|
+
if '/path/to/your/config/file' == config_file
|
27
|
+
puts 'Please set the config_file variable to the path to your config. file'
|
28
|
+
exit
|
29
|
+
end
|
30
|
+
|
31
|
+
# load config. file
|
32
|
+
options = YAML::load_file(config_file)
|
33
|
+
access_key = options['access_key']
|
34
|
+
secret_key = options['secret_key']
|
35
|
+
from_email = options['from_email']
|
36
|
+
to_email ||= options['to_email']
|
37
|
+
bucket = options['bucket']
|
38
|
+
|
39
|
+
# check for bucket
|
40
|
+
if !bucket
|
41
|
+
require 'readline'
|
42
|
+
bucket = Readline.readline('No bucket found; please enter name for new bucket: ', true)
|
43
|
+
client = S3::Client.new(access_key, secret_key)
|
44
|
+
response = client.create_bucket(bucket, client.canned_acl_header('public-read'))
|
45
|
+
if response.ok?
|
46
|
+
puts 'Created new bucket'
|
47
|
+
else
|
48
|
+
puts 'Could not create bucket (HTTP problem)'
|
49
|
+
exit
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# estimate upload time
|
54
|
+
filesize = FileTest.size(filename)
|
55
|
+
filesize_mb = (filesize / (1000*1000)).to_i
|
56
|
+
secs_per_mb = 12
|
57
|
+
str = "Transferring file; size %d bytes" % filesize
|
58
|
+
str += " (about %d Mb)" % filesize_mb if filesize_mb > 0
|
59
|
+
str += "\nUploading normally takes at least %d seconds per Mb\n" % secs_per_mb
|
60
|
+
|
61
|
+
if filesize_mb > 0
|
62
|
+
str += "So this should take about %d seconds" % (filesize_mb * secs_per_mb)
|
63
|
+
else
|
64
|
+
str += "So this should be done in no time"
|
65
|
+
end
|
66
|
+
|
67
|
+
puts str
|
68
|
+
|
69
|
+
# time the put
|
70
|
+
start_time_secs = Time.now.to_i
|
71
|
+
|
72
|
+
# a client pointing at the specified bucket
|
73
|
+
S3::NamedBucket.new(access_key, secret_key, bucket, :public_contents => true) do |client|
|
74
|
+
response = client.put_file(filename)
|
75
|
+
end
|
76
|
+
|
77
|
+
time_taken_secs = Time.now.to_i - start_time_secs
|
78
|
+
puts "Aah, it appears to have taken %d seconds" % time_taken_secs
|
79
|
+
|
80
|
+
# post-put report
|
81
|
+
if response.ok?
|
82
|
+
puts "File #{filename} transferred OK"
|
83
|
+
url = File.join("http://", S3::HOST, client.bucket_name, '/') + filename
|
84
|
+
puts "Available at URL:"
|
85
|
+
puts url
|
86
|
+
|
87
|
+
if to_email
|
88
|
+
message = "From:#{from_email}\r\nTo:#{to_email}\r\nSubject:You were sent a file\r\n\r\nFetch it from\n#{url}"
|
89
|
+
|
90
|
+
Net::SMTP.start('localhost') do |smtp|
|
91
|
+
smtp.send_message message, from_email, to_email
|
92
|
+
end
|
93
|
+
end
|
94
|
+
else
|
95
|
+
puts 'File transfer failed'
|
96
|
+
end
|
data/lib/s33r.rb
ADDED
data/lib/s33r/client.rb
ADDED
@@ -0,0 +1,188 @@
|
|
1
|
+
require 'net/https'
|
2
|
+
require 'cgi'
|
3
|
+
|
4
|
+
# this is a very thin layer over the S3 API
|
5
|
+
# TODO: need to wrap XML returned into object representation
|
6
|
+
module S3
|
7
|
+
include Net
|
8
|
+
|
9
|
+
class Client
|
10
|
+
include S3
|
11
|
+
attr_accessor :chunk_size, :default_headers
|
12
|
+
|
13
|
+
def initialize(aws_access_key, aws_secret_access_key)
|
14
|
+
@client = HTTP.new(HOST, PORT)
|
15
|
+
|
16
|
+
# turn off SSL certificate verification
|
17
|
+
@client.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
18
|
+
|
19
|
+
# always use SSL
|
20
|
+
@client.use_ssl = true
|
21
|
+
|
22
|
+
# set default chunk size for streaming request body (1 Mb)
|
23
|
+
@chunk_size = 1048576
|
24
|
+
|
25
|
+
# Amazon S3 developer keys
|
26
|
+
@aws_access_key = aws_access_key
|
27
|
+
@aws_secret_access_key = aws_secret_access_key
|
28
|
+
|
29
|
+
# headers sent with every request made by this client
|
30
|
+
@client_headers = {}
|
31
|
+
end
|
32
|
+
|
33
|
+
# send a request over the wire
|
34
|
+
def do_request(method, path, data=nil, headers={})
|
35
|
+
req = get_requester(method, path)
|
36
|
+
req.chunk_size = @chunk_size
|
37
|
+
|
38
|
+
# add the S3 headers which are always required
|
39
|
+
headers = add_default_headers(headers)
|
40
|
+
|
41
|
+
# add any client-specific default headers
|
42
|
+
headers = add_client_headers(headers)
|
43
|
+
|
44
|
+
headers['Authorization'] = generate_auth_header_value(method, path, headers,
|
45
|
+
@aws_access_key, @aws_secret_access_key)
|
46
|
+
|
47
|
+
headers.each do |key, value|
|
48
|
+
req[key] = value
|
49
|
+
end
|
50
|
+
|
51
|
+
@client.start do
|
52
|
+
if req.request_body_permitted?
|
53
|
+
# for streaming large files
|
54
|
+
if data.respond_to?(:read)
|
55
|
+
req.body_stream = data
|
56
|
+
req['Content-Length'] = data.stat.size.to_s
|
57
|
+
return @client.request(req, nil)
|
58
|
+
# simple text strings etc.
|
59
|
+
else
|
60
|
+
return @client.request(req, data)
|
61
|
+
end
|
62
|
+
else
|
63
|
+
return @client.request(req)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
# get
|
70
|
+
def do_get(path='/', headers={})
|
71
|
+
do_request('GET', path, headers)
|
72
|
+
end
|
73
|
+
|
74
|
+
# head
|
75
|
+
def do_head(path='/', headers={})
|
76
|
+
do_request('HEAD', path, headers)
|
77
|
+
end
|
78
|
+
|
79
|
+
# post
|
80
|
+
def do_post(path='/', data=nil, headers={})
|
81
|
+
do_request('POST', path, data, headers)
|
82
|
+
end
|
83
|
+
|
84
|
+
# put
|
85
|
+
def do_put(path='/', data=nil, headers={})
|
86
|
+
do_request('PUT', path, data, headers)
|
87
|
+
end
|
88
|
+
|
89
|
+
# return an instance of an appropriate request class
|
90
|
+
def get_requester(method, path)
|
91
|
+
raise S3Exception::UnsupportedHTTPMethod, "The #{method} HTTP method is not supported" if !(METHOD_VERBS.include?(method))
|
92
|
+
eval("HTTP::" + method[0,1].upcase + method[1..-1].downcase + ".new('#{path}')")
|
93
|
+
end
|
94
|
+
|
95
|
+
# convert a hash of name/value pairs to querystring variables
|
96
|
+
def get_querystring(pairs={})
|
97
|
+
str = ''
|
98
|
+
if pairs.size > 0
|
99
|
+
str += "?" + pairs.map { |key, value| "#{key}=#{CGI::escape(value.to_s)}" }.join('&')
|
100
|
+
end
|
101
|
+
str
|
102
|
+
end
|
103
|
+
|
104
|
+
# list all buckets
|
105
|
+
def list_all_buckets
|
106
|
+
do_get('/')
|
107
|
+
end
|
108
|
+
|
109
|
+
# create a bucket
|
110
|
+
def create_bucket(bucket_name, headers={})
|
111
|
+
bucket_name_valid?(bucket_name)
|
112
|
+
bucket_exists?(bucket_name)
|
113
|
+
do_put("/#{bucket_name}", nil, headers)
|
114
|
+
end
|
115
|
+
|
116
|
+
# list entries in a bucket
|
117
|
+
def list_bucket(bucket_name)
|
118
|
+
bucket_name_valid?(bucket_name)
|
119
|
+
bucket_exists?(bucket_name)
|
120
|
+
do_get("/#{bucket_name}")
|
121
|
+
end
|
122
|
+
|
123
|
+
# put some resource onto S3
|
124
|
+
def put_resource(data, bucket_name, resource_key, headers={})
|
125
|
+
do_put(File.join("/#{bucket_name}", "#{CGI::escape(resource_key)}"), data, headers)
|
126
|
+
end
|
127
|
+
|
128
|
+
# put a string onto S3
|
129
|
+
def put_text(string, bucket_name, resource_key, headers={})
|
130
|
+
headers["Content-Type"] = "text/plain"
|
131
|
+
put_resource(bucket_name, resource_key, string, headers)
|
132
|
+
end
|
133
|
+
|
134
|
+
# put a file onto S3
|
135
|
+
def put_file(filename, bucket_name, resource_key=nil, headers={})
|
136
|
+
# default to the file path as the resource key if none explicitly set
|
137
|
+
if resource_key.nil?
|
138
|
+
resource_key = filename
|
139
|
+
end
|
140
|
+
|
141
|
+
# content type is explicitly set in the headers
|
142
|
+
if headers[:content_type]
|
143
|
+
# use the first MIME type corresponding to this content type string
|
144
|
+
# (MIME::Types returns an array of possible MIME types)
|
145
|
+
mime_type = MIME::Types[headers[:content_type]][0]
|
146
|
+
else
|
147
|
+
mime_type = guess_mime_type(filename)
|
148
|
+
end
|
149
|
+
content_type = mime_type.simplified
|
150
|
+
headers['Content-Type'] = content_type
|
151
|
+
headers['Content-Transfer-Encoding'] = 'binary' if mime_type.binary?
|
152
|
+
|
153
|
+
# the data we want to put (handle to file, so we can stream from it)
|
154
|
+
File.open(filename) do |data|
|
155
|
+
# send the put request
|
156
|
+
put_resource(data, bucket_name, resource_key, headers)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
# guess a file's mime type
|
161
|
+
# NB if the mime_type for a file cannot be guessed, "text/plain" is used
|
162
|
+
def guess_mime_type(filename)
|
163
|
+
mime_type = MIME::Types.type_for(filename)[0]
|
164
|
+
mime_type ||= MIME::Types['text/plain'][0]
|
165
|
+
mime_type
|
166
|
+
end
|
167
|
+
|
168
|
+
# ensure that a bucket_name is well-formed
|
169
|
+
def bucket_name_valid?(bucket_name)
|
170
|
+
if '/' == bucket_name[0,1]
|
171
|
+
raise S3Exception::MalformedBucketName, "Bucket name cannot have a leading slash"
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
# TODO: proper check for existence of bucket;
|
176
|
+
# throw error if bucket does not exist (see bucket_name_valid? for example)
|
177
|
+
def bucket_exists?(bucket_name)
|
178
|
+
false
|
179
|
+
end
|
180
|
+
|
181
|
+
# add any default headers which should be sent with every request from the client;
|
182
|
+
# any headers passed into this method override the defaults in @client_headers
|
183
|
+
def add_client_headers(headers)
|
184
|
+
headers.merge!(@client_headers) { |key, arg, default| arg }
|
185
|
+
end
|
186
|
+
|
187
|
+
end
|
188
|
+
end
|
data/lib/s33r/core.rb
ADDED
@@ -0,0 +1,131 @@
|
|
1
|
+
require 'base64'
|
2
|
+
require 'time'
|
3
|
+
require 'net/https'
|
4
|
+
require 'openssl'
|
5
|
+
|
6
|
+
module S3
|
7
|
+
HOST = 's3.amazonaws.com'
|
8
|
+
PORT = 443
|
9
|
+
METADATA_PREFIX = 'x-amz-meta-'
|
10
|
+
AWS_HEADER_PREFIX = 'x-amz-'
|
11
|
+
AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
|
12
|
+
INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
|
13
|
+
REQUIRED_HEADERS = ['Content-Type', 'Date']
|
14
|
+
CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
|
15
|
+
METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST']
|
16
|
+
|
17
|
+
# builds the canonical string for signing;
|
18
|
+
# modified (slightly) from the Amazon sample code
|
19
|
+
def generate_canonical_string(method, path, headers={}, expires=nil)
|
20
|
+
interesting_headers = {}
|
21
|
+
headers.each do |key, value|
|
22
|
+
lk = key.downcase
|
23
|
+
if (INTERESTING_HEADERS.include?(lk) or lk =~ /^#{AWS_HEADER_PREFIX}/o)
|
24
|
+
interesting_headers[lk] = value
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# these fields get empty strings if they don't exist.
|
29
|
+
interesting_headers['content-type'] ||= ''
|
30
|
+
interesting_headers['content-md5'] ||= ''
|
31
|
+
|
32
|
+
# if you're using expires for query string auth, then it trumps date
|
33
|
+
if not expires.nil?
|
34
|
+
interesting_headers['date'] = expires
|
35
|
+
end
|
36
|
+
|
37
|
+
buf = "#{method}\n"
|
38
|
+
interesting_headers.sort { |a, b| a[0] <=> b[0] }.each do |key, value|
|
39
|
+
if key =~ /^#{AWS_HEADER_PREFIX}/o
|
40
|
+
buf << "#{key}:#{value}\n"
|
41
|
+
else
|
42
|
+
buf << "#{value}\n"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
# ignore everything after the question mark...
|
47
|
+
buf << path.gsub(/\?.*$/, '')
|
48
|
+
|
49
|
+
# ...unless there is an acl or torrent parameter
|
50
|
+
if path =~ /[&?]acl($|&|=)/
|
51
|
+
buf << '?acl'
|
52
|
+
elsif path =~ /[&?]torrent($|&|=)/
|
53
|
+
buf << '?torrent'
|
54
|
+
end
|
55
|
+
|
56
|
+
return buf
|
57
|
+
end
|
58
|
+
|
59
|
+
# get the header value for AWS authentication
|
60
|
+
def generate_auth_header_value(method, path, headers, aws_access_key, aws_secret_access_key)
|
61
|
+
raise S3Exception::MethodNotAvailable, "Method %s not available" % method if !METHOD_VERBS.include?(method)
|
62
|
+
|
63
|
+
# check the headers needed for authentication have been set
|
64
|
+
missing_headers = REQUIRED_HEADERS - headers.keys
|
65
|
+
if !(missing_headers.empty?)
|
66
|
+
raise S3Exception::MissingRequiredHeaders,
|
67
|
+
"Headers required for AWS auth value are missing: " + missing_headers.join(', ')
|
68
|
+
end
|
69
|
+
|
70
|
+
# get the AWS header
|
71
|
+
canonical_string = generate_canonical_string(method, path, headers)
|
72
|
+
signature = generate_signature(aws_secret_access_key, canonical_string)
|
73
|
+
AWS_AUTH_HEADER_VALUE % [aws_access_key, signature]
|
74
|
+
end
|
75
|
+
|
76
|
+
# encode the given string with the aws_secret_access_key, by taking the
|
77
|
+
# hmac sha1 sum, and then base64 encoding it
|
78
|
+
def generate_signature(aws_secret_access_key, str, urlencode=false)
|
79
|
+
digest = OpenSSL::HMAC::digest(OpenSSL::Digest::Digest.new("SHA1"), aws_secret_access_key, str)
|
80
|
+
Base64.encode64(digest).strip
|
81
|
+
end
|
82
|
+
|
83
|
+
# build the headers required with every S3 request
|
84
|
+
# options hash can contain extra header settings, as follows:
|
85
|
+
# :date and :content_type are required headers, and set to defaults if not supplied
|
86
|
+
def add_default_headers(headers, options={})
|
87
|
+
|
88
|
+
# set the default headers required by AWS
|
89
|
+
missing_headers = REQUIRED_HEADERS - headers.keys
|
90
|
+
|
91
|
+
if missing_headers.include?('Content-Type')
|
92
|
+
content_type = options[:content_type] || ''
|
93
|
+
headers['Content-Type'] = content_type
|
94
|
+
end
|
95
|
+
|
96
|
+
if missing_headers.include?('Date')
|
97
|
+
date = options[:date] || Time.now
|
98
|
+
headers['Date'] = date.httpdate
|
99
|
+
end
|
100
|
+
|
101
|
+
headers
|
102
|
+
end
|
103
|
+
|
104
|
+
# add metadata headers to the request, correctly prefixing them first
|
105
|
+
# returns the headers with the metadata headers appended
|
106
|
+
def metadata_headers(headers, metadata={})
|
107
|
+
unless metadata.empty?
|
108
|
+
metadata.each { |key, value| headers[METADATA_PREFIX + key] = value }
|
109
|
+
end
|
110
|
+
headers
|
111
|
+
end
|
112
|
+
|
113
|
+
# add a canned ACL setter
|
114
|
+
def canned_acl_header(canned_acl, headers={})
|
115
|
+
unless canned_acl.nil?
|
116
|
+
unless CANNED_ACLS.include?(canned_acl)
|
117
|
+
raise S3Exception::UnsupportedCannedACL, "The canned ACL #{canned_acl} is not supported"
|
118
|
+
end
|
119
|
+
headers[AWS_HEADER_PREFIX + 'acl'] = canned_acl
|
120
|
+
end
|
121
|
+
headers
|
122
|
+
end
|
123
|
+
|
124
|
+
# guess a file's mime type
|
125
|
+
# NB if the mime_type for a file cannot be guessed, "text/plain" is used
|
126
|
+
def guess_mime_type(file_name)
|
127
|
+
mime_type = MIME::Types.type_for(file_name)[0]
|
128
|
+
mime_type = MIME::Types['text/plain'][0] unless mime_type
|
129
|
+
mime_type
|
130
|
+
end
|
131
|
+
end
|