swift-storage 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,172 @@
1
+ require "time"
2
+
3
+ # @attr [String] content_length
4
+ # Content length of the Object, in bytes.
5
+ #
6
+ # @attr [String] content_type
7
+ # Content type of the Object, eg: `image/png`.
8
+ #
9
+ # @attr [String] expires
10
+ # When the object is set to expire.
11
+ #
12
+ # @attr [String] cache_control
13
+ # Object cache control header.
14
+ #
15
+ class SwiftStorage::Object < SwiftStorage::Node
16
+
17
+
18
+ parent_node :container
19
+
20
+
21
+ header_attributes :content_length,
22
+ :content_type,
23
+ :expires,
24
+ :cache_control
25
+
26
+ # Read the object data
27
+ #
28
+ # This will always make a request to the API server and will not use cache
29
+ #
30
+ # @param output_stream [IO]
31
+ # An optional stream to write the object's content to. This can be a `File` or and `IO` object.
32
+ # This method will **NEVER** rewind `output_stream`, either before writing and after.
33
+ #
34
+ # @return [String, output_stream]
35
+ # If `output_stream` is nil or ommited, it returns a string with the object content. If `output_stream`
36
+ # is given, returns it.
37
+ #
38
+ def read(output_stream=nil)
39
+ response = request(relative_path, :method => :get, :output_stream => output_stream)
40
+ if output_stream
41
+ output_stream
42
+ else
43
+ response.body
44
+ end
45
+ end
46
+
47
+
48
+ # Write the object
49
+ #
50
+ # This will always make a request to the API server and will not use cache
51
+ #
52
+ # @note If you want to only update the metadata, you may omit `input_stream`
53
+ # but you must specify all other options otherwise they will be overwritten.
54
+ #
55
+ # @note Some headers specified here may not work with a specific swift server
56
+ # as they must be enabled in the server configuration.
57
+ #
58
+ # @param input_stream [String, IO]
59
+ # The data to upload, if ommited, the write will not override the body and instead it will update
60
+ # the metadata and other options. If `input_stream` is an `IO` object, it must
61
+ # be seeked to the proper position, this method will **NEVER** seek or rewind the stream.
62
+ #
63
+ # @param content_type [String]
64
+ # The content type, eg: `image/png`.
65
+ #
66
+ # @param attachment [Boolean]
67
+ # If `true` the file will be served with `Content-Disposition: attachment`.
68
+ #
69
+ # @param delete_at [Time]
70
+ # If set, the server will delete the object at the specified time.
71
+ #
72
+ # @param delete_after [Time]
73
+ # If set, the server will delete the object after the specified time.
74
+ #
75
+ # @param cache_control [String]
76
+ # The value for the 'Cache-Control' header when serving the object. The value
77
+ # is not parsed and served unmodified as is. If you set max-age, it will
78
+ # always be served with the same max-age value. To have the resource expire
79
+ # at point of time, use the expires header.
80
+ #
81
+ # @param expires [Time]
82
+ # Set the Expires header.
83
+ #
84
+ # @param object_manifest [String]
85
+ # When set, this object acts as a large object manifest. The value should be
86
+ # `<container>/<prefix>` where `<container>` is the container the object
87
+ # segments are in and `<prefix>` is the common prefix for all the segments.
88
+ #
89
+ # @return [input_stream]
90
+ # Return the `input_stream` argument, or `nil` if `input_stream` is ommited.
91
+ #
92
+ def write(input_stream=nil,
93
+ content_type: nil,
94
+ attachment: false,
95
+ delete_at: nil,
96
+ delete_after: nil,
97
+ cache_control: nil,
98
+ expires: nil,
99
+ object_manifest: nil,
100
+ metadata: nil)
101
+
102
+ h = {}
103
+
104
+ input_stream.nil? or content_type or raise ArgumentError, 'Content_type is required if input_stream is given'
105
+
106
+ object_manifest.nil? or input_stream.nil? or raise ArgumentError, 'Input must be nil on object manigest'
107
+
108
+ h[H::CONTENT_DISPOSITION] = attachment ? 'attachment' : 'inline'
109
+ h[H::OBJECT_MANIFEST] = object_manifest if object_manifest
110
+ h[H::CONTENT_TYPE] = content_type if content_type
111
+ h[H::EXPIRES] = expires.httpdate if expires
112
+ h[H::CACHE_CONTROL] = cache_control if cache_control
113
+
114
+ if delete_at
115
+ h[H::DELETE_AT] = delete_at.to_i.to_s
116
+ elsif delete_after
117
+ h[H::DELETE_AFTER] = delete_after.to_i.to_s
118
+ end
119
+
120
+ merge_metadata(h, metadata)
121
+
122
+ method = input_stream || object_manifest ? :put : :post
123
+
124
+ request(relative_path,
125
+ :method => method,
126
+ :headers => h,
127
+ :input_stream => input_stream)
128
+ clear_cache
129
+ input_stream
130
+ end
131
+
132
+
133
+ # Generates a public URL with an expiration time
134
+ #
135
+ # @param expires [Time]
136
+ # The absolute time when the URL will expire.
137
+ #
138
+ # @param method [Symbol]
139
+ # The HTTP method to allow, can be `:get, :put, :head`.
140
+ #
141
+ # @return [String]
142
+ # A temporary URL to the object.
143
+ #
144
+ # @!parse def temp_url(expires=Time.now + 3600, method: :get);end
145
+ def temp_url(expires=nil, method: :get)
146
+ expires ||= Time.now + 3600
147
+ service.create_temp_url(container.name, name, expires, method)
148
+ end
149
+
150
+ # Returns the object's URL
151
+ #
152
+ # @note This URL is unsigneds and the container authorization will apply. If
153
+ # the container do not allow public access, this URL will require an
154
+ # authentication token.
155
+ #
156
+ # @return [String]
157
+ # The object URL.
158
+ #
159
+ def url
160
+ File.join(service.storage_url, relative_path)
161
+ end
162
+
163
+ private
164
+
165
+ H = SwiftStorage::Headers
166
+
167
+ def relative_path
168
+ File.join(container.name, name)
169
+ end
170
+
171
+ end
172
+
@@ -0,0 +1,34 @@
1
+ class SwiftStorage::ObjectCollection < SwiftStorage::Node
2
+
3
+ parent_node :container
4
+
5
+
6
+ # Return all objects
7
+ #
8
+ # @note This method will return only the first 1000 objects.
9
+ #
10
+ # @return [Array<SwiftStorage::Object>]
11
+ # Objects in this collection
12
+ #
13
+ def all
14
+ get_lines(container.name).map { |name| SwiftStorage::Object.new(container, name)}
15
+ end
16
+
17
+ # Return a particular object
18
+ #
19
+ # @note This always return an object, regadeless of it's existence
20
+ # on the server. This call do NOT contact the server.
21
+ #
22
+ # @param name [String]
23
+ # The name (sometimes named key) of the object
24
+ #
25
+ # @return [SwiftStorage::Object]
26
+ # Object with given name
27
+ #
28
+ def [](name)
29
+ SwiftStorage::Object.new(container, name)
30
+ end
31
+
32
+ end
33
+
34
+
@@ -0,0 +1,201 @@
1
+ require 'uri'
2
+
3
+ class SwiftStorage::Service
4
+
5
+ include SwiftStorage::Utils
6
+ include SwiftStorage
7
+
8
+ attr_reader :tenant,
9
+ :endpoint,
10
+ :storage_url,
11
+ :auth_token,
12
+ :storage_token,
13
+ :storage_scheme,
14
+ :storage_host,
15
+ :storage_port,
16
+ :storage_path,
17
+ :temp_url_key
18
+
19
+ def initialize(tenant: ENV['SWIFT_STORAGE_TENANT'],
20
+ username: ENV['SWIFT_STORAGE_USERNAME'],
21
+ password: ENV['SWIFT_STORAGE_PASSWORD'],
22
+ endpoint: ENV['SWIFT_STORAGE_ENDPOINT'],
23
+ temp_url_key: ENV['SWIFT_STORAGE_TEMP_URL_KEY'])
24
+ @temp_url_key = temp_url_key
25
+
26
+ %w(tenant username password endpoint).each do |n|
27
+ eval("#{n} or raise ArgumentError, '#{n} is required'")
28
+ eval("@#{n} = #{n}")
29
+ end
30
+ self.storage_url = File.join(endpoint, 'v1', "AUTH_#{tenant}")
31
+
32
+ @sessions = {}
33
+ end
34
+
35
+ def authenticate!
36
+ headers = {
37
+ Headers::AUTH_USER => "#{tenant}:#{username}",
38
+ Headers::AUTH_KEY => password
39
+ }
40
+ res = request(auth_url, :headers => headers)
41
+
42
+ h = res.header
43
+ self.storage_url = h[Headers::STORAGE_URL]
44
+ @auth_token = h[Headers::AUTH_TOKEN]
45
+ @storage_token = h[Headers::STORAGE_TOKEN]
46
+ end
47
+
48
+ def authenticated?
49
+ !!(storage_url && auth_token)
50
+ end
51
+
52
+ def containers
53
+ @container_collection ||= SwiftStorage::ContainerCollection.new(self)
54
+ end
55
+
56
+ def account
57
+ @account ||= SwiftStorage::Account.new(self, tenant)
58
+ end
59
+
60
+ def storage_url=(new_url)
61
+ uri = URI.parse(new_url)
62
+ @storage_url = new_url
63
+ @storage_scheme = uri.scheme
64
+ @storage_host = uri.host
65
+ @storage_port = uri.port
66
+ @storage_path = uri.path
67
+ end
68
+
69
+ def create_temp_url(container, object, expires, method, options = {})
70
+
71
+ scheme = options[:scheme] || storage_scheme
72
+
73
+ method = method.to_s.upcase
74
+ # Limit methods
75
+ %w{GET PUT HEAD}.include?(method) or raise ArgumentError, "Only GET, PUT, HEAD supported"
76
+
77
+ expires = expires.to_i
78
+ object_path_escaped = File.join(storage_path, escape(container), escape(object,"/"))
79
+ object_path_unescaped = File.join(storage_path, escape(container), object)
80
+
81
+ string_to_sign = "#{method}\n#{expires}\n#{object_path_unescaped}"
82
+
83
+ sig = sig_to_hex(hmac('sha1', temp_url_key, string_to_sign))
84
+
85
+ klass = scheme == 'http' ? URI::HTTP : URI::HTTPS
86
+
87
+ temp_url_options = {
88
+ :scheme => scheme,
89
+ :host => storage_host,
90
+ :port => storage_port,
91
+ :path => object_path_escaped,
92
+ :query => URI.encode_www_form(
93
+ :temp_url_sig => sig,
94
+ :temp_url_expires => expires
95
+ )
96
+ }
97
+ klass.build(temp_url_options).to_s
98
+ end
99
+
100
+
101
+ # CGI.escape, but without special treatment on spaces
102
+ def self.escape(str, extra_exclude_chars = '')
103
+ str.gsub(/([^a-zA-Z0-9_.-#{extra_exclude_chars}]+)/) do
104
+ '%' + $1.unpack('H2' * $1.bytesize).join('%').upcase
105
+ end
106
+ end
107
+
108
+ def escape(*args)
109
+ self.class.escape(*args)
110
+ end
111
+
112
+ def request(path_or_url,
113
+ method: :get,
114
+ headers: nil,
115
+ input_stream: nil,
116
+ output_stream: nil)
117
+ headers ||= {}
118
+ headers.merge!(Headers::AUTH_TOKEN => auth_token) if authenticated?
119
+ headers.merge!(Headers::CONNECTION => 'keep-alive', Headers::PROXY_CONNECTION => 'keep-alive')
120
+
121
+ if !(path_or_url =~ /^http/)
122
+ path_or_url = File.join(storage_url, path_or_url)
123
+ end
124
+
125
+ # Cache HTTP session as url with no path (scheme, host, port)
126
+ uri = URI.parse(path_or_url)
127
+ path = uri.path
128
+ uri.path = ''
129
+ key = uri.to_s
130
+
131
+ if sessions[key].nil?
132
+ s = sessions[key] = Net::HTTP.new(uri.host, uri.port)
133
+ s.use_ssl = uri.scheme == 'https'
134
+ #s.set_debug_output($stderr)
135
+ s.keep_alive_timeout = 30
136
+ s.start
137
+ end
138
+ s = sessions[key]
139
+
140
+ case method
141
+ when :get
142
+ req = Net::HTTP::Get.new(path, headers)
143
+ when :delete
144
+ req = Net::HTTP::Delete.new(path, headers)
145
+ when :head
146
+ req = Net::HTTP::Head.new(path, headers)
147
+ when :post
148
+ req = Net::HTTP::Post.new(path, headers)
149
+ when :put
150
+ req = Net::HTTP::Put.new(path, headers)
151
+ else
152
+ raise ArgumentError, "Method #{method} not supported"
153
+ end
154
+
155
+ if input_stream
156
+ if String === input_stream
157
+ input_stream = StringIO.new(input_stream)
158
+ end
159
+ req.body_stream = input_stream
160
+ req.content_length = input_stream.size
161
+ end
162
+
163
+ if output_stream
164
+ output_proc = proc do |response|
165
+ response.read_body do |chunk|
166
+ output_stream.write(chunk)
167
+ end
168
+ end
169
+ end
170
+
171
+ response = s.request(req, &output_proc)
172
+ check_response!(response)
173
+ response
174
+ end
175
+
176
+ private
177
+
178
+ attr_reader :sessions,
179
+ :username,
180
+ :password
181
+
182
+ def auth_url
183
+ File.join(endpoint, 'auth/v1.0')
184
+ end
185
+
186
+ def check_response!(response)
187
+ case response.code
188
+ when /^2/
189
+ return true
190
+ when '401'
191
+ raise AuthError
192
+ when '403'
193
+ raise ForbiddenError
194
+ when '404'
195
+ raise NotFoundError
196
+ else
197
+ raise ServerError
198
+ end
199
+ end
200
+
201
+ end
@@ -0,0 +1,23 @@
1
+ module SwiftStorage::Utils
2
+
3
+ include SwiftStorage::Errors
4
+
5
+ def hmac(type, key, data)
6
+ digest = OpenSSL::Digest.new(type)
7
+ OpenSSL::HMAC.digest(digest, key, data)
8
+ end
9
+
10
+ def sig_to_hex(str)
11
+ str.unpack("C*").map { |c|
12
+ c.to_s(16)
13
+ }.map { |h|
14
+ h.size == 1 ? "0#{h}" : h
15
+ }.join
16
+ end
17
+
18
+ def struct(h)
19
+ return nil if h.empty?
20
+ Struct.new(*h.keys.map(&:to_sym)).new(*h.values)
21
+ end
22
+
23
+ end
@@ -0,0 +1,3 @@
1
+ module SwiftStorage
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,139 @@
1
+ require "swift-storage"
2
+ require_relative "support/local_server"
3
+
4
+ module TestServerMixin
5
+
6
+ def h
7
+ SwiftStorage::Headers
8
+ end
9
+
10
+ def self.run
11
+ @server = LocalTestServer.new
12
+ @server.run
13
+ end
14
+
15
+ def self.server
16
+ @server
17
+ end
18
+
19
+ def self.reset
20
+ @server.app.reset
21
+ end
22
+
23
+ def server
24
+ TestServerMixin.server
25
+ end
26
+
27
+ def swift_service
28
+ service = SwiftStorage::Service.new(:tenant => 'test',
29
+ :username => 'testuser',
30
+ :password => 'testpassword',
31
+ :endpoint => server.base_url,
32
+ :temp_url_key => 'A1234'
33
+ )
34
+ end
35
+
36
+ def test_storage_url
37
+ File.join(server.base_url, 'v1/AUTH_test')
38
+ end
39
+
40
+ def headers(new_headers)
41
+ server.app.mock_headers = new_headers
42
+ end
43
+
44
+ def status(new_status)
45
+ server.app.mock_status = new_status
46
+ end
47
+
48
+ def body(new_body)
49
+ server.app.mock_body = new_body
50
+ end
51
+
52
+ def random_length
53
+ Random.rand(5000) + 1000
54
+ end
55
+
56
+
57
+ end
58
+
59
+ RSpec::Matchers.define :send_request do |method, path, headers, body|
60
+ match do |actual|
61
+ actual.call()
62
+ env = server.app.last_env
63
+
64
+ @actual_method = env['REQUEST_METHOD'].downcase
65
+ @actual_path = env['PATH_INFO']
66
+ @actual_body = env['rack.input'].read
67
+
68
+ @method_match = @actual_method == method.to_s.downcase
69
+
70
+ @path_match = @actual_path == path
71
+ @headers_match = true
72
+
73
+ headers.each_pair do |k,v|
74
+ k = k.gsub('-', '_').upcase
75
+ actual_value = env[k] || env["HTTP_#{k}"]
76
+ if v.to_s != actual_value.to_s
77
+ @unatched_header = "Header #{k} should be #{v}, got #{actual_value||'null'}"
78
+ @headers_match = false
79
+ break
80
+ end
81
+ end if headers
82
+ @body_match = true
83
+ @body_match = @actual_body == body if body
84
+
85
+ @method_match && @path_match && @headers_match && @body_match
86
+ end
87
+
88
+ failure_message do
89
+ r = []
90
+ r << "Method should be #{method}, got #{@actual_method}" if !@method_match
91
+ r << "Path should be #{path}, got #{@actual_path}" if !@path_match
92
+ r << "Unmatched headers #{@unatched_header}" if !@headers_match
93
+ r << "Body doesn't match, for #{@actual_body} expected #{body}" if !@body_match
94
+ r.join("\n")
95
+ end
96
+
97
+ def supports_block_expectations?
98
+ true
99
+ end
100
+
101
+ end
102
+
103
+ RSpec.configure do |config|
104
+
105
+ config.expect_with :rspec do |expectations|
106
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
107
+ end
108
+
109
+
110
+ config.mock_with :rspec do |mocks|
111
+ mocks.verify_partial_doubles = true
112
+ end
113
+
114
+ config.filter_run :focus
115
+ config.run_all_when_everything_filtered = true
116
+
117
+ config.disable_monkey_patching!
118
+
119
+ if config.files_to_run.one?
120
+ config.default_formatter = 'doc'
121
+ end
122
+
123
+ config.profile_examples = 10
124
+
125
+ config.order = :random
126
+
127
+ Kernel.srand(config.seed)
128
+
129
+ config.include(TestServerMixin)
130
+
131
+ config.before(:suite) do
132
+ TestServerMixin.run
133
+ end
134
+
135
+ config.before(:each) do
136
+ TestServerMixin.reset
137
+ end
138
+
139
+ end