qoobaa-s3 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.gitignore +6 -0
- data/LICENSE +20 -0
- data/README.rdoc +7 -0
- data/Rakefile +58 -0
- data/VERSION +1 -0
- data/bin/s3cmd.rb +188 -0
- data/lib/s3.rb +23 -0
- data/lib/s3/bucket.rb +154 -0
- data/lib/s3/connection.rb +152 -0
- data/lib/s3/exceptions.rb +94 -0
- data/lib/s3/object.rb +153 -0
- data/lib/s3/roxy/moxie.rb +58 -0
- data/lib/s3/roxy/proxy.rb +72 -0
- data/lib/s3/service.rb +106 -0
- data/lib/s3/signature.rb +120 -0
- data/s3.gemspec +65 -0
- data/test/bucket_test.rb +22 -0
- data/test/connection_test.rb +164 -0
- data/test/s3_test.rb +5 -0
- data/test/service_test.rb +133 -0
- data/test/signature_test.rb +143 -0
- data/test/test_helper.rb +11 -0
- metadata +82 -0
data/.document
ADDED
data/.gitignore
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Jakub Kuźma, Mirosław Boruta
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'rake'
|
5
|
+
|
6
|
+
begin
|
7
|
+
require 'jeweler'
|
8
|
+
Jeweler::Tasks.new do |gem|
|
9
|
+
gem.name = "s3"
|
10
|
+
gem.summary = %Q{Library for accessing S3 objects and buckets, with command line tool}
|
11
|
+
gem.email = "qoobaa@gmail.com"
|
12
|
+
gem.homepage = "http://github.com/qoobaa/s3"
|
13
|
+
gem.authors = ["Jakub Kuźma", "Mirosław Boruta"]
|
14
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
15
|
+
end
|
16
|
+
|
17
|
+
rescue LoadError
|
18
|
+
puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'rake/testtask'
|
22
|
+
Rake::TestTask.new(:test) do |test|
|
23
|
+
test.libs << 'lib' << 'test'
|
24
|
+
test.pattern = 'test/**/*_test.rb'
|
25
|
+
test.verbose = true
|
26
|
+
end
|
27
|
+
|
28
|
+
begin
|
29
|
+
require 'rcov/rcovtask'
|
30
|
+
Rcov::RcovTask.new do |test|
|
31
|
+
test.libs << 'test'
|
32
|
+
test.pattern = 'test/**/*_test.rb'
|
33
|
+
test.verbose = true
|
34
|
+
end
|
35
|
+
rescue LoadError
|
36
|
+
task :rcov do
|
37
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
|
42
|
+
task :default => :test
|
43
|
+
|
44
|
+
require 'rake/rdoctask'
|
45
|
+
Rake::RDocTask.new do |rdoc|
|
46
|
+
if File.exist?('VERSION.yml')
|
47
|
+
config = YAML.load(File.read('VERSION.yml'))
|
48
|
+
version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
|
49
|
+
else
|
50
|
+
version = ""
|
51
|
+
end
|
52
|
+
|
53
|
+
rdoc.rdoc_dir = 'rdoc'
|
54
|
+
rdoc.title = "s3 #{version}"
|
55
|
+
rdoc.rdoc_files.include('README*')
|
56
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
57
|
+
end
|
58
|
+
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.3
|
data/bin/s3cmd.rb
ADDED
@@ -0,0 +1,188 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$: << File.expand_path(File.dirname(__FILE__) + "/../lib")
|
4
|
+
|
5
|
+
require "trollop"
|
6
|
+
require "s3"
|
7
|
+
|
8
|
+
# HELPER METHODS
|
9
|
+
|
10
|
+
include S3
|
11
|
+
|
12
|
+
def list_buckets(service)
|
13
|
+
service.buckets.each do |bucket|
|
14
|
+
puts bucket.name
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_bucket(service, name, location)
|
19
|
+
service.buckets.build(name).save(location)
|
20
|
+
end
|
21
|
+
|
22
|
+
def destroy_bucket(service, name)
|
23
|
+
service.buckets.find(name).destroy
|
24
|
+
end
|
25
|
+
|
26
|
+
def show_bucket(service, name, options = {})
|
27
|
+
service.buckets.find(name).objects.find_all.each do |object|
|
28
|
+
puts "#{name}/#{object.key}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def list_objects(service)
|
33
|
+
service.buckets.each do |bucket|
|
34
|
+
bucket.objects.each do |object|
|
35
|
+
puts "#{bucket.name}/#{object.key}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def create_object(service, name, file_name, options = {})
|
41
|
+
bucket_name, object_name = name.split("/", 2)
|
42
|
+
object = service.buckets.find(bucket_name).objects.build(object_name)
|
43
|
+
object.content_type = options[:type]
|
44
|
+
object.content_encoding = options[:encoding]
|
45
|
+
object.content_disposition = options[:disposition]
|
46
|
+
object.acl = options[:acl]
|
47
|
+
object.content = File.new(file_name)
|
48
|
+
object.save
|
49
|
+
end
|
50
|
+
|
51
|
+
def destroy_object(service, name)
|
52
|
+
bucket_name, object_name = name.split("/", 2)
|
53
|
+
object = service.buckets.find(bucket_name).objects.find(object_name)
|
54
|
+
object.destroy
|
55
|
+
end
|
56
|
+
|
57
|
+
def show_object(service, name, file_name = nil)
|
58
|
+
bucket_name, object_name = name.split("/", 2)
|
59
|
+
object = service.buckets.find(bucket_name).objects.find_first(object_name)
|
60
|
+
puts " object: #{object.name}/#{object.key}"
|
61
|
+
puts " content type: #{object.content_type}"
|
62
|
+
puts " size: #{object.size}"
|
63
|
+
puts " etag: #{object.etag}"
|
64
|
+
puts " last modified: #{object.last_modified}"
|
65
|
+
# puts "content disposition: #{object.content_disposition}"
|
66
|
+
if file_name
|
67
|
+
if file_name == "-"
|
68
|
+
puts object.content
|
69
|
+
else
|
70
|
+
File.open(file_name, "wb") do |file|
|
71
|
+
file.write(object.content)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# COMMAND LINE PARSER
|
78
|
+
|
79
|
+
ACCESS_KEY_ID = ENV["ACCESS_KEY_ID"]
|
80
|
+
SECRET_ACCESS_KEY = ENV["SECRET_ACCESS_KEY"]
|
81
|
+
COMMANDS = %w(bucket object)
|
82
|
+
BUCKET_SUBCOMMANDS = %w(add remove show)
|
83
|
+
OBJECT_SUBCOMMANDS = %w(add remove show)
|
84
|
+
|
85
|
+
global_options = Trollop::options do
|
86
|
+
banner "s3.rb"
|
87
|
+
opt :access_key_id, "Your access key id to AWS", :type => :string, :default => ACCESS_KEY_ID
|
88
|
+
opt :secret_access_key, "Your secret access key to AWS", :type => :string, :default => SECRET_ACCESS_KEY
|
89
|
+
opt :debug, "Debug mode", :type => :flag, :default => false
|
90
|
+
stop_on COMMANDS
|
91
|
+
end
|
92
|
+
|
93
|
+
Trollop::die "No access key id given" unless global_options[:access_key_id]
|
94
|
+
Trollop::die "No secret access key given" unless global_options[:secret_access_key]
|
95
|
+
|
96
|
+
service = Service.new(:access_key_id => global_options[:access_key_id],
|
97
|
+
:secret_access_key => global_options[:secret_access_key],
|
98
|
+
:debug => global_options[:debug])
|
99
|
+
|
100
|
+
command = ARGV.shift
|
101
|
+
|
102
|
+
begin
|
103
|
+
case command
|
104
|
+
when "bucket"
|
105
|
+
command_options = Trollop::options do
|
106
|
+
banner "manage buckets"
|
107
|
+
stop_on BUCKET_SUBCOMMANDS
|
108
|
+
end
|
109
|
+
subcommand = ARGV.shift
|
110
|
+
case subcommand
|
111
|
+
when "add"
|
112
|
+
subcommand_options = Trollop::options do
|
113
|
+
banner "add bucket"
|
114
|
+
opt :location, "Location of the bucket - EU or US", :default => "US", :type => :string
|
115
|
+
end
|
116
|
+
name = ARGV.shift
|
117
|
+
Trollop::die "Bucket has not been added because of unknown error" unless create_bucket(service, name, subcommand_options[:location])
|
118
|
+
when "remove"
|
119
|
+
subcommand_options = Trollop::options do
|
120
|
+
banner "remove bucket"
|
121
|
+
end
|
122
|
+
name = ARGV.shift
|
123
|
+
Trollop::die "Bucket name must be given" if name.nil? or name.empty?
|
124
|
+
Trollop::die "Bucket has not been removed because of unknown error" unless destroy_bucket(service, name)
|
125
|
+
when "show"
|
126
|
+
subcommand_options = Trollop::options do
|
127
|
+
banner "show bucket"
|
128
|
+
opt :prefix, "Limits the response to keys which begin with the indicated prefix", :type => :string
|
129
|
+
opt :marker, "Indicates where in the bucket to begin listing", :type => :string
|
130
|
+
opt :max_keys, "The maximum number of keys you'd like to see", :type => :integer
|
131
|
+
opt :delimiter, "Causes keys that contain the same string between the prefix and the first occurrence of the delimiter to be rolled up into a single result element", :type => :string
|
132
|
+
end
|
133
|
+
name = ARGV.shift
|
134
|
+
Trollop::die "Bucket name must be given" if name.nil? or name.empty?
|
135
|
+
show_bucket(service, name, subcommand_options)
|
136
|
+
when nil
|
137
|
+
list_buckets(service)
|
138
|
+
else
|
139
|
+
Trollop::die "Unknown subcommand: #{subcommand.inspect}"
|
140
|
+
end
|
141
|
+
when "object"
|
142
|
+
command_options = Trollop::options do
|
143
|
+
banner "manage objects"
|
144
|
+
stop_on OBJECT_SUBCOMMANDS
|
145
|
+
end
|
146
|
+
subcommand = ARGV.shift
|
147
|
+
case subcommand
|
148
|
+
when "add"
|
149
|
+
subcommand_options = Trollop::options do
|
150
|
+
banner "object add s3_object_name local_file_name"
|
151
|
+
opt :type, "A standard MIME type describing the format of the contents", :default => "binary/octet-stream"
|
152
|
+
opt :disposition, "Specifies presentational information for the object", :type => :string
|
153
|
+
opt :encoding, "Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied in order to obtain the media-type referenced by the Content-Type header field", :type => :string
|
154
|
+
opt :acl, "The canned ACL to apply to the object. Options include private, public-read, public-read-write, and authenticated-read", :type => :string
|
155
|
+
end
|
156
|
+
name = ARGV.shift
|
157
|
+
Trollop::die "No object name given" if name.nil? or name.empty?
|
158
|
+
file_name = ARGV.shift
|
159
|
+
Trollop::die "No file name given" if file_name.nil? or file_name.empty?
|
160
|
+
Trollop::die "Object has not been added because of unknown error" unless create_object(service, name, file_name, subcommand_options)
|
161
|
+
when "remove"
|
162
|
+
subcommand_options = Trollop::options do
|
163
|
+
banner "object remove s3_object_name"
|
164
|
+
end
|
165
|
+
name = ARGV.shift
|
166
|
+
Trollop::die "No object name given" if name.nil? or name.empty?
|
167
|
+
Trollop::die "Object has not been removed because of unknown error" unless destroy_object(service, name)
|
168
|
+
when "show"
|
169
|
+
subcommand_options = Trollop::options do
|
170
|
+
banner "object show s3_object_name optional_file_name"
|
171
|
+
end
|
172
|
+
name = ARGV.shift
|
173
|
+
Trollop::die "No object name given" if name.nil? or name.empty?
|
174
|
+
file_name = ARGV.shift
|
175
|
+
show_object(service, name, file_name)
|
176
|
+
when nil
|
177
|
+
list_objects(service)
|
178
|
+
else
|
179
|
+
Trollop::die "Unknown subcommand: #{subcommand.inspect}"
|
180
|
+
end
|
181
|
+
when nil
|
182
|
+
Trollop::die "No command given"
|
183
|
+
else
|
184
|
+
Trollop::die "Unknown command #{command.inspect}"
|
185
|
+
end
|
186
|
+
rescue Error::ResponseError => e
|
187
|
+
Trollop::die e.message.sub(/\.+\Z/, "")
|
188
|
+
end
|
data/lib/s3.rb
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
require "time"
|
2
|
+
require "openssl"
|
3
|
+
require "net/http"
|
4
|
+
require "net/https"
|
5
|
+
require "base64"
|
6
|
+
require "forwardable"
|
7
|
+
require "digest/md5"
|
8
|
+
|
9
|
+
require "xmlsimple"
|
10
|
+
|
11
|
+
require "s3/roxy/proxy"
|
12
|
+
require "s3/roxy/moxie"
|
13
|
+
|
14
|
+
require "s3/bucket"
|
15
|
+
require "s3/connection"
|
16
|
+
require "s3/exceptions"
|
17
|
+
require "s3/object"
|
18
|
+
require "s3/service"
|
19
|
+
require "s3/signature"
|
20
|
+
|
21
|
+
module S3
|
22
|
+
HOST = "s3.amazonaws.com"
|
23
|
+
end
|
data/lib/s3/bucket.rb
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
module S3
|
2
|
+
class Bucket
|
3
|
+
extend Roxy::Moxie
|
4
|
+
extend Forwardable
|
5
|
+
|
6
|
+
attr_reader :name, :service
|
7
|
+
|
8
|
+
def_instance_delegators :service, :service_request
|
9
|
+
|
10
|
+
def retrieve
|
11
|
+
bucket_request(:get, :params => { :max_keys => 0 })
|
12
|
+
self
|
13
|
+
end
|
14
|
+
|
15
|
+
def location(reload = false)
|
16
|
+
if reload or @location.nil?
|
17
|
+
response = bucket_request(:get, :params => { :location => nil })
|
18
|
+
@location = parse_location(response.body)
|
19
|
+
else
|
20
|
+
@location
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def ==(other)
|
25
|
+
self.name == other.name and self.service == other.service
|
26
|
+
end
|
27
|
+
|
28
|
+
def exists?
|
29
|
+
retrieve
|
30
|
+
true
|
31
|
+
rescue Error::NoSuchBucket
|
32
|
+
false
|
33
|
+
end
|
34
|
+
|
35
|
+
def destroy
|
36
|
+
bucket_request(:delete)
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def save(location = nil)
|
41
|
+
location = location.to_s.upcase if location
|
42
|
+
options = { :headers => {} }
|
43
|
+
if location and location != "US"
|
44
|
+
options[:body] = "<CreateBucketConfiguration><LocationConstraint>#{location}</LocationConstraint></CreateBucketConfiguration>"
|
45
|
+
options[:headers][:content_type] = "application/xml"
|
46
|
+
end
|
47
|
+
bucket_request(:put, options)
|
48
|
+
true
|
49
|
+
end
|
50
|
+
|
51
|
+
def vhost?
|
52
|
+
"#@name.#{HOST}" =~ /\A#{URI::REGEXP::PATTERN::HOSTNAME}\Z/
|
53
|
+
end
|
54
|
+
|
55
|
+
def host
|
56
|
+
vhost? ? "#@name.#{HOST}" : "#{HOST}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def path_prefix
|
60
|
+
vhost? ? "" : "#@name/"
|
61
|
+
end
|
62
|
+
|
63
|
+
def objects(reload = false, options = {})
|
64
|
+
if options.empty?
|
65
|
+
if reload or @objects.nil?
|
66
|
+
@objects = fetch_objects
|
67
|
+
else
|
68
|
+
@objects
|
69
|
+
end
|
70
|
+
else
|
71
|
+
fetch_objects(options)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
proxy :objects do
|
76
|
+
def build(key)
|
77
|
+
Object.new(proxy_owner, key)
|
78
|
+
end
|
79
|
+
|
80
|
+
def find_first(name)
|
81
|
+
object = build(name)
|
82
|
+
object.retrieve
|
83
|
+
end
|
84
|
+
alias :find :find_first
|
85
|
+
|
86
|
+
def find_all(options = {})
|
87
|
+
proxy_owner.objects(true, options)
|
88
|
+
end
|
89
|
+
|
90
|
+
def reload
|
91
|
+
proxy_owner.objects(true)
|
92
|
+
end
|
93
|
+
|
94
|
+
def destroy_all
|
95
|
+
proxy_target.each do |object|
|
96
|
+
object.destroy
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def inspect
|
102
|
+
"#<#{self.class}:#{name}>"
|
103
|
+
end
|
104
|
+
|
105
|
+
def initialize(service, name)
|
106
|
+
self.service = service
|
107
|
+
self.name = name
|
108
|
+
end
|
109
|
+
|
110
|
+
private
|
111
|
+
|
112
|
+
attr_writer :service
|
113
|
+
|
114
|
+
def name=(name)
|
115
|
+
raise ArgumentError.new("Invalid bucket name: #{name}") unless name_valid?(name)
|
116
|
+
@name = name
|
117
|
+
end
|
118
|
+
|
119
|
+
def bucket_request(method, options = {})
|
120
|
+
path = "#{path_prefix}#{options[:path]}"
|
121
|
+
service_request(method, options.merge(:host => host, :path => path))
|
122
|
+
end
|
123
|
+
|
124
|
+
def fetch_objects(options = {})
|
125
|
+
response = bucket_request(:get, options)
|
126
|
+
parse_objects(response.body)
|
127
|
+
end
|
128
|
+
|
129
|
+
def parse_objects(xml_body)
|
130
|
+
xml = XmlSimple.xml_in(xml_body)
|
131
|
+
objects_attributes = xml["Contents"]
|
132
|
+
if objects_attributes
|
133
|
+
objects_attributes.map do |object_attributes|
|
134
|
+
Object.new(self,
|
135
|
+
object_attributes["Key"].first,
|
136
|
+
:etag => object_attributes["ETag"].first,
|
137
|
+
:last_modified => object_attributes["LastModified"].first,
|
138
|
+
:size => object_attributes["Size"].first)
|
139
|
+
end
|
140
|
+
else
|
141
|
+
[]
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def parse_location(xml_body)
|
146
|
+
xml = XmlSimple.xml_in(xml_body)
|
147
|
+
xml["content"]
|
148
|
+
end
|
149
|
+
|
150
|
+
def name_valid?(name)
|
151
|
+
name =~ /\A[a-z0-9][a-z0-9\._-]{2,254}\Z/ and @name !~ /\A#{URI::REGEXP::PATTERN::IPV4ADDR}\Z/
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|