visor-image 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/bin/visor +423 -0
  2. data/bin/visor-image +10 -0
  3. data/config/server.rb +14 -0
  4. data/lib/image/auth.rb +147 -0
  5. data/lib/image/cli.rb +397 -0
  6. data/lib/image/client.rb +490 -0
  7. data/lib/image/meta.rb +219 -0
  8. data/lib/image/routes/delete_all_images.rb +40 -0
  9. data/lib/image/routes/delete_image.rb +62 -0
  10. data/lib/image/routes/get_image.rb +78 -0
  11. data/lib/image/routes/get_images.rb +54 -0
  12. data/lib/image/routes/get_images_detail.rb +54 -0
  13. data/lib/image/routes/head_image.rb +51 -0
  14. data/lib/image/routes/post_image.rb +189 -0
  15. data/lib/image/routes/put_image.rb +205 -0
  16. data/lib/image/server.rb +307 -0
  17. data/lib/image/store/cumulus.rb +126 -0
  18. data/lib/image/store/file_system.rb +119 -0
  19. data/lib/image/store/hdfs.rb +149 -0
  20. data/lib/image/store/http.rb +78 -0
  21. data/lib/image/store/lunacloud.rb +126 -0
  22. data/lib/image/store/s3.rb +121 -0
  23. data/lib/image/store/store.rb +39 -0
  24. data/lib/image/store/walrus.rb +130 -0
  25. data/lib/image/version.rb +5 -0
  26. data/lib/visor-image.rb +30 -0
  27. data/spec/lib/client_spec.rb +0 -0
  28. data/spec/lib/meta_spec.rb +230 -0
  29. data/spec/lib/routes/delete_image_spec.rb +98 -0
  30. data/spec/lib/routes/get_image_spec.rb +78 -0
  31. data/spec/lib/routes/get_images_detail_spec.rb +104 -0
  32. data/spec/lib/routes/get_images_spec.rb +104 -0
  33. data/spec/lib/routes/head_image_spec.rb +51 -0
  34. data/spec/lib/routes/post_image_spec.rb +112 -0
  35. data/spec/lib/routes/put_image_spec.rb +109 -0
  36. data/spec/lib/server_spec.rb +62 -0
  37. data/spec/lib/store/cumulus_spec.rb +0 -0
  38. data/spec/lib/store/file_system_spec.rb +32 -0
  39. data/spec/lib/store/http_spec.rb +56 -0
  40. data/spec/lib/store/s3_spec.rb +37 -0
  41. data/spec/lib/store/store_spec.rb +36 -0
  42. data/spec/lib/store/walrus_spec.rb +0 -0
  43. metadata +217 -0
@@ -0,0 +1,307 @@
1
+ require 'goliath'
2
+ require 'tempfile'
3
+ require 'json'
4
+
5
+ module Visor
6
+ module Image
7
+
8
+ # The VISoR Image Server. This supports all image metadata manipulation
9
+ # operations, dispatched to the VISoR Meta Server and the image files storage operations.
10
+ #
11
+ # *The Server API is a RESTful web service for image meta as follows:*
12
+ #
13
+ # HEAD /images/<id> - Returns metadata about the image with the given id
14
+ # GET /images - Returns a set of brief metadata about all public images
15
+ # GET /images/detail - Returns a set of detailed metadata about all public images
16
+ # GET /images/<id> - Returns image data and metadata for the image with the given id
17
+ # POST /images - Stores a new image data and metadata and returns the registered metadata
18
+ # PUT /images/<id> - Update image metadata and/or data for the image with the given id
19
+ # DELETE /images/<id> - Delete the metadata and data of the image with the given id
20
+ #
21
+ # The Image is multi-format, most properly it supports response encoding in JSON and XML formats.
22
+ # It will auto negotiate and encode the response metadata and error messages in the proper format,
23
+ # based on the request *Accept* header, being it 'application/json' or 'application/xml'.
24
+ # If no Accept header is provided, Image will encode and render it as JSON by default.
25
+ #
26
+ # This server routes all metadata operations to the {Visor::Meta::Server Visor Meta Server}.
27
+ #
28
+ # The server will interact with the multiple supported backend store to manage the image files.
29
+ #
30
+ # *Currently we support the following store backend and operations:*
31
+ #
32
+ # Amazon Simple Storage (s3) - GET, POST, PUT, DELETE
33
+ # Nimbus Cumulus (cumulus) - GET, POST, PUT, DELETE
34
+ # Eucalyptus Walrus (walrus) - GET, POST, PUT, DELETE
35
+ # Local Filesystem (file) - GET, POST, PUT, DELETE
36
+ # Remote HTTP (http) - GET
37
+ #
38
+ class Server < Goliath::API
39
+
40
+ # Middleware
41
+ #
42
+ # Listen at /status for a heartbeat server message status
43
+ use Goliath::Rack::Heartbeat
44
+ # Auto parse and merge body and query parameters
45
+ use Goliath::Rack::Params
46
+ # Cleanup accepted media types
47
+ use Goliath::Rack::DefaultMimeType
48
+
49
+
50
+ # Routes
51
+ #
52
+
53
+ # @method head_image
54
+ # @overload head '/image/:id'
55
+ #
56
+ # Head metadata about the image with the given id, see {Visor::Image::HeadImage}.
57
+ #
58
+ # The image metadata is promptly passed as a set of HTTP headers, as the following example:
59
+ #
60
+ # x-image-meta-_id: <id>
61
+ # x-image-meta-uri: <uri>
62
+ # x-image-meta-name: <name>
63
+ # x-image-meta-architecture: <architecture>
64
+ # x-image-meta-access: <access>
65
+ # x-image-meta-status: <status>
66
+ # x-image-meta-created_at: <timestamp>
67
+ #
68
+ # @note Undefined attributes are ignored and not included into response's header. Also
69
+ # any raised error will be passed through HTTP headers as expected.
70
+ #
71
+ # @param [String] id The wanted image _id.
72
+ #
73
+ # @example Retrieve the image metadata with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
74
+ # 'HEAD /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
75
+ #
76
+ # @return [HTTP Headers] The image data file.
77
+ #
78
+ # @raise [HTTP Error 404] If image not found.
79
+ # @raise [HTTP Error 500] On internal server error.
80
+ #
81
+ head '/images/:id', HeadImage
82
+
83
+
84
+ # @method get_all_brief
85
+ # @overload get '/images'
86
+ #
87
+ # Get brief information about all public images, see {Visor::Image::GetImages}.
88
+ #
89
+ # { "images":
90
+ # [
91
+ # {
92
+ # "_id":<_id>,
93
+ # "uri":<uri>,
94
+ # "name":<name>,
95
+ # "architecture":<architecture>,
96
+ # "type":<type>,
97
+ # "format":<format>,
98
+ # "store":<type>,
99
+ # "size":<size>,
100
+ # "created_at":<timestamp>
101
+ # },
102
+ # ...
103
+ # ]
104
+ # }
105
+ #
106
+ # The following options can be passed as query parameters, plus any other additional
107
+ # image attribute not defined in the schema.
108
+ #
109
+ # @param [String, Integer, Date] parameter The image attribute to filter results based on its value.
110
+ # @param [String] sort ("_id") The image attribute to sort results.
111
+ # @param [String] dir ("asc") The sorting order ("asc"/"desc").
112
+ #
113
+ # @example Retrieve all public images brief metadata:
114
+ # 'GET /images'
115
+ #
116
+ # @example Retrieve all public `x86_64` images brief metadata:
117
+ # 'GET /images?architecture=x86_64'
118
+ #
119
+ # @example Retrieve all public `.iso` images brief metadata, descending sorted by `size`:
120
+ # 'GET /images?format=iso&sort=size&dir=desc'
121
+ #
122
+ # @return [JSON, XML] The public images brief metadata.
123
+ #
124
+ # @raise [HTTP Error 404] If there is no public images.
125
+ # @raise [HTTP Error 500] On internal server error.
126
+ #
127
+ get '/images', GetImages
128
+
129
+
130
+ # @method get_all_detail
131
+ # @overload get '/images/detail'
132
+ #
133
+ # Get detailed information about all public images, see {Visor::Image::GetImagesDetail}.
134
+ #
135
+ # { "images":
136
+ # [
137
+ # {
138
+ # "_id":<_id>,
139
+ # "uri":<uri>,
140
+ # "name":<name>,
141
+ # "architecture":<architecture>,
142
+ # "access":<access>,
143
+ # "status":<status>,
144
+ # "size":<size>,
145
+ # "type":<type>,
146
+ # "format":<format>,
147
+ # "store":<store>,
148
+ # "created_at":<timestamp>
149
+ # "updated_at":<timestamp>,
150
+ # "kernel":<associated kernel>,
151
+ # "ramdisk":<associated ramdisk>,
152
+ # },
153
+ # ...
154
+ # ]
155
+ # }
156
+ #
157
+ # The following options can be passed as query parameters, plus any other additional
158
+ # image attribute not defined in the schema.
159
+ #
160
+ # @param [String, Integer, Date] parameter The image attribute to filter results based on its value.
161
+ # @param [String] sort ("_id") The image attribute to sort results.
162
+ # @param [String] dir ("asc") The sorting order ("asc"/"desc").
163
+ #
164
+ # @example Retrieve all public images detailed metadata:
165
+ # 'GET /images/detail'
166
+ #
167
+ # @note Querying and ordering results are made as with #get_all_detail
168
+ #
169
+ # @return [JSON, XML] The public images detailed metadata.
170
+ #
171
+ # @raise [HTTP Error 404] If there is no public images.
172
+ # @raise [HTTP Error 500] On internal server error.
173
+ #
174
+ get '/images/detail', GetImagesDetail
175
+
176
+
177
+ # @method get_image
178
+ # @overload get '/images/:id'
179
+ #
180
+ # Get image data and detailed metadata for the given id, see {Visor::Image::GetImage}.
181
+ #
182
+ # The image data file is streamed as response's body. The server will return a 200 status code,
183
+ # with a special HTTP header, indicating that the response body will be streamed in chunks
184
+ # and connection shouldn't be closed until the transfer is complete.
185
+ #
186
+ # Also, the image metadata is promptly passed as a set of HTTP headers, as the following example:
187
+ #
188
+ # x-image-meta-_id: <id>
189
+ # x-image-meta-uri: <uri>
190
+ # x-image-meta-name: <name>
191
+ # x-image-meta-architecture: <architecture>
192
+ # x-image-meta-access: <access>
193
+ # x-image-meta-status: <status>
194
+ # x-image-meta-created_at: <timestamp>
195
+ #
196
+ # @note Undefined attributes are ignored and not included into response's header.
197
+ #
198
+ # @param [String] id The wanted image _id.
199
+ #
200
+ # @example Retrieve the image with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
201
+ # 'GET /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
202
+ #
203
+ # @return [HTTP Headers] The image data file.
204
+ # @return [HTTP Body] The image data file.
205
+ #
206
+ # @raise [HTTP Error 404] If image not found.
207
+ # @raise [HTTP Error 500] On internal server error.
208
+ #
209
+ get '/images/:id', GetImage
210
+
211
+
212
+ # @method post
213
+ # @overload post '/images'
214
+ #
215
+ # Post image data and metadata and returns the registered metadata, see {Visor::Image::PostImage}.
216
+ #
217
+ # The image metadata should be encoded as HTTP headers and passed with the request, as the following example:
218
+ #
219
+ # x-image-meta-name: Ubuntu 10.10 Desktop
220
+ # x-image-meta-architecture: i386
221
+ # x-image-meta-store: s3
222
+ #
223
+ # If wanted, the image data file should be streamed through the request body.
224
+ # The server will receive that data in chunks, buffering them to a properly secured temporary
225
+ # file, avoiding buffering all the data into memory. Server will then handle the upload of that
226
+ # data to the definitive store location, cleaning then the generated temporary file.
227
+ #
228
+ # Alternatively, a *x-image-meta-location* header can be passed, if the file is already stored in some
229
+ # provided location. If this header is present, no body content can be passed in the request.
230
+ #
231
+ # @return [JSON, XML] The already created image detailed metadata.
232
+ #
233
+ # @raise [HTTP Error 400] If the image metadata validation fails.
234
+ # @raise [HTTP Error 400] If the location header is present no file content can be provided.
235
+ # @raise [HTTP Error 400] If trying to post an image file to a HTTP backend.
236
+ # @raise [HTTP Error 400] If provided store is an unsupported store backend.
237
+ # @raise [HTTP Error 404] If no image data is found at the provided location.
238
+ # @raise [HTTP Error 409] If the provided image file already exists in the backend store.
239
+ # @raise [HTTP Error 500] On internal server error.
240
+ #
241
+ post '/images', PostImage
242
+
243
+
244
+ # @method put
245
+ # @overload put '/images/:id'
246
+ #
247
+ # Put image metadata and/or data for the image with the given id, see {Visor::Image::PutImage}.
248
+ #
249
+ # The image metadata should be encoded as HTTP headers and passed with the request, as the following example:
250
+ #
251
+ # x-image-meta-name: Ubuntu 10.10 Server
252
+ # x-image-meta-architecture: x86_64
253
+ # x-image-meta-location: http://www.ubuntu.com/start-download?distro=server&bits=64&release=latest
254
+ #
255
+ # If wanted, the image data file should be streamed through the request body.
256
+ # The server will receive that data in chunks, buffering them to a properly secured temporary
257
+ # file, avoiding buffering all the data into memory. Server will then handle the upload of that
258
+ # data to the definitive store location, cleaning then the generated temporary file.
259
+ #
260
+ # Alternatively, a *x-image-meta-location* header can be passed, if the file is already stored in some
261
+ # provided location. If this header is present, no body content can be passed in the request.
262
+ #
263
+ # @note Only images with status set to 'locked' or 'error' can be updated with an image data file.
264
+ #
265
+ # @return [JSON, XML] The already updated image detailed metadata.
266
+ #
267
+ # @raise [HTTP Error 400] If the image metadata validation fails.
268
+ # @raise [HTTP Error 400] If no headers neither body found for update.
269
+ # @raise [HTTP Error 400] If the location header is present no file content can be provided.
270
+ # @raise [HTTP Error 400] If trying to post an image file to a HTTP backend.
271
+ # @raise [HTTP Error 400] If provided store is an unsupported store backend.
272
+ # @raise [HTTP Error 404] If no image data is found at the provided location.
273
+ # @raise [HTTP Error 409] If trying to assign image file to a locked or uploading image.
274
+ # @raise [HTTP Error 409] If the provided image file already exists in the backend store.
275
+ # @raise [HTTP Error 500] On internal server error.
276
+ #
277
+ put '/images/:id', PutImage
278
+
279
+
280
+ # @method delete
281
+ # @overload delete '/images/:id'
282
+ #
283
+ # Delete the metadata and data of the image with the given id, see {Visor::Image::DeleteImage}.
284
+ #
285
+ # @param [String] id The image _id to delete.
286
+ #
287
+ # @example Delete the image with id '19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d':
288
+ # 'DELETE /images/19b39ed6-6c43-41cc-8d59-d1a1ed24ac9d'
289
+ #
290
+ # @return [JSON, XML] The already deleted image detailed metadata.
291
+ #
292
+ # @raise [HTTP Error 404] If image meta or data not found.
293
+ # @raise [HTTP Error 403] If user does not have permission to manipulate the image file.
294
+ # @raise [HTTP Error 500] On internal server error.
295
+ #
296
+ delete '/images/:id', DeleteImage
297
+
298
+ delete '/all', DeleteAllImages
299
+
300
+ # Not Found
301
+ not_found('/') do
302
+ run Proc.new { |env| [404, {}, {code: 404, message: "Invalid operation or path."}.to_json] }
303
+ end
304
+ end
305
+
306
+ end
307
+ end
@@ -0,0 +1,126 @@
1
+ require 'uri'
2
+ require 's3restful'
3
+ require 'em-synchrony'
4
+ require 'em-synchrony/em-http'
5
+
6
+ module Visor
7
+ module Image
8
+ module Store
9
+
10
+ # The Nimbus Cumulus (Cumulus) backend store.
11
+ #
12
+ # This class handles the management of image files located in the Cumulus storage system,
13
+ # based on a URI like *cumulus://<access_key>:<secret_key>@<host>:<port>/<bucket>/<image>*.
14
+ #
15
+ class Cumulus
16
+ include Visor::Common::Exception
17
+
18
+ attr_accessor :uri, :config, :access_key, :secret_key, :bucket, :file, :host, :port
19
+
20
+ # Initializes a new Cumulus store client object. Cumulus credentials are loaded from the URI,
21
+ # on GET and DELETE operations, or from the configuration file for POST and PUT operation.
22
+ #
23
+ # @param [String] uri The URI of the file location.
24
+ # @param config [Hash] A set of configurations for the wanted store, loaded from
25
+ # VISoR configuration file.
26
+ #
27
+ # @return [Object] An instantiated Cumulus store object ready to use.
28
+ #
29
+ def initialize(uri, config)
30
+ @uri = URI(uri)
31
+ @config = config[:cumulus]
32
+
33
+ if @uri.scheme
34
+ @access_key = @uri.user
35
+ @secret_key = @uri.password
36
+ @bucket = @uri.path.split('/')[1]
37
+ @file = @uri.path.split('/')[2]
38
+ @host = @uri.host
39
+ @port = @uri.port
40
+ else
41
+ @access_key = @config[:access_key]
42
+ @secret_key = @config[:secret_key]
43
+ @bucket = @config[:bucket]
44
+ @host = @config[:host]
45
+ @port = @config[:port]
46
+ end
47
+ end
48
+
49
+ # Returns a Happening library S3 Cumulus compatible connection object.
50
+ #
51
+ # @return [Happening::S3::Item] A new Cumulus connection object.
52
+ #
53
+ def connection
54
+ S3restful::S3::Item.new(bucket, file, server: host, port: port, protocol: 'http',
55
+ aws_access_key_id: access_key, aws_secret_access_key: secret_key)
56
+ end
57
+
58
+ # Returns the image file to clients, streamed in chunks.
59
+ #
60
+ # @return [Object] Yields the file, a chunk at time.
61
+ #
62
+ def get
63
+ s3 = connection.aget
64
+ finish = proc { yield nil }
65
+
66
+ s3.stream { |chunk| yield chunk }
67
+ s3.callback &finish
68
+ s3.errback &finish
69
+ end
70
+
71
+ # Saves the image file to the its final destination, based on the temporary file
72
+ # created by the server at data reception time.
73
+ #
74
+ # @param [String] id The image id.
75
+ # @param [File] tmp_file The temporary file descriptor.
76
+ # @param [String] format The image file format.
77
+ #
78
+ # @return [String, Integer] The generated file location URI and image file size.
79
+ #
80
+ # @raise [Duplicated] If the image file already exists.
81
+ #
82
+ def save(id, tmp_file, format)
83
+ @file = "#{id}.#{format}"
84
+ uri = "cumulus://#{access_key}:#{secret_key}@#{host}:#{port}/#{bucket}/#{file}"
85
+ size = tmp_file.size
86
+
87
+ raise Duplicated, "The image file #{fp} already exists" if file_exists?(false)
88
+ STDERR.puts "COPYING!!"
89
+
90
+ connection.store tmp_file.path
91
+
92
+ [uri, size]
93
+ end
94
+
95
+ # Deletes the image file from its location.
96
+ #
97
+ # @raise [NotFound] If the image file was not found.
98
+ #
99
+ def delete
100
+ connection.delete
101
+ end
102
+
103
+ # Check if the image file exists.
104
+ #
105
+ # @param [True, False] raise_exc (true) If it should raise exception or return
106
+ # true/false whether the file exists or not.
107
+ #
108
+ # @return [True, False] If raise_exc is false, return true/false whether the
109
+ # file exists or not.
110
+ #
111
+ # @raise [NotFound] If the image file was not found.
112
+ #
113
+ def file_exists?(raise_exc=true)
114
+ exist = nil
115
+ error = proc { exist = false }
116
+ success = proc { |res| exist = true if res.response_header.status == 200 }
117
+
118
+ connection.head(on_error: error, on_success: success)
119
+ raise NotFound, "No image file found at #{uri}" if raise_exc && !exist
120
+ exist
121
+ end
122
+ end
123
+
124
+ end
125
+ end
126
+ end
@@ -0,0 +1,119 @@
1
+ require 'uri'
2
+
3
+ module Visor
4
+ module Image
5
+ module Store
6
+
7
+ # The FileSystem backend store.
8
+ #
9
+ # This class handles the management of image files located in the local FileSystem,
10
+ # based on a URI like *file:///path/to/my_image.format*.
11
+ #
12
+ class FileSystem
13
+ include Visor::Common::Exception
14
+
15
+ # Size of the chunk to stream the files out.
16
+ CHUNKSIZE = 65536
17
+
18
+ attr_accessor :uri, :fp, :config
19
+
20
+ # Initializes a new FileSystem store client object.
21
+ #
22
+ # @param [String] uri The URI of the file location.
23
+ # @param config [Hash] A set of configurations for the wanted store, loaded from
24
+ # VISoR configuration file.
25
+ #
26
+ # @return [Object] An instantiated FileSystem store object ready to use.
27
+ #
28
+ def initialize(uri, config)
29
+ @uri = URI(uri)
30
+ @fp = @uri.path
31
+ @config = config[:file]
32
+ end
33
+
34
+ # Returns the image file to clients, streamed in chunks.
35
+ #
36
+ # @return [Object] Yields the file, a chunk at time.
37
+ #
38
+ def get
39
+ file_exists?
40
+ open(fp, "rb") do |file|
41
+ yield file.read(CHUNKSIZE) until file.eof?
42
+ yield nil
43
+ end
44
+ end
45
+
46
+ # Saves the image file to the its final destination, based on the temporary file
47
+ # created by the server at data reception time.
48
+ #
49
+ # @param [String] id The image id.
50
+ # @param [File] tmp_file The temporary file descriptor.
51
+ # @param [String] format The image file format.
52
+ #
53
+ # @return [String, Integer] The generated file location URI and image file size.
54
+ #
55
+ # @raise [Duplicated] If the image file already exists.
56
+ #
57
+ def save(id, tmp_file, format)
58
+ dir = File.expand_path config[:directory]
59
+ file = "#{id}.#{format}"
60
+ fp = File.join(dir, file)
61
+ uri = "file://#{fp}"
62
+ size = tmp_file.size
63
+
64
+ FileUtils.mkpath(dir) unless Dir.exists?(dir)
65
+ raise Duplicated, "The image file #{fp} already exists" if File.exists?(fp)
66
+ STDERR.puts "Copying image tempfile #{tmp_file.path} to definitive #{fp}"
67
+
68
+ tmp = File.open(tmp_file.path, "rb")
69
+ new = File.open(fp, "wb")
70
+
71
+ each_chunk(tmp, CHUNKSIZE) do |chunk|
72
+ new << chunk
73
+ end
74
+
75
+ [uri, size]
76
+ end
77
+
78
+ # Deletes the image file to from its location.
79
+ #
80
+ # @raise [Forbidden] If user does not have permission to manipulate the image file.
81
+ # @raise [NotFound] If the image file was not found.
82
+ #
83
+ def delete
84
+ file_exists?
85
+ begin
86
+ File.delete(fp)
87
+ rescue => e
88
+ raise Forbidden, "Error while trying to delete image file #{fp}: #{e.message}"
89
+ end
90
+ end
91
+
92
+ # Check if the image file exists.
93
+ #
94
+ # @raise [NotFound] If the image file was not found.
95
+ #
96
+ def file_exists?
97
+ raise NotFound, "No image file found at #{fp}" unless File.exists?(fp)
98
+ end
99
+
100
+ private
101
+
102
+ # Iterates over the image file yielding a chunk per reactor tick.
103
+ #
104
+ # @return [Object] Yielded image file chunk.
105
+ #
106
+ def each_chunk(file, chunk_size=CHUNKSIZE)
107
+ handler = lambda do
108
+ unless file.eof?
109
+ yield file.read(chunk_size)
110
+ EM.next_tick &handler
111
+ end
112
+ end
113
+ EM.next_tick &handler
114
+ end
115
+
116
+ end
117
+ end
118
+ end
119
+ end