s3_relay 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.md +209 -0
  4. data/Rakefile +31 -0
  5. data/app/assets/javascripts/s3_relay.coffee +112 -0
  6. data/app/assets/stylesheets/s3_relay.css +31 -0
  7. data/app/controllers/s3_relay/uploads_controller.rb +65 -0
  8. data/app/helpers/s3_relay/uploads_helper.rb +23 -0
  9. data/app/models/s3_relay/upload.rb +46 -0
  10. data/config/routes.rb +5 -0
  11. data/db/migrate/20141009000804_create_s3_relay_uploads.rb +19 -0
  12. data/lib/s3_relay.rb +4 -0
  13. data/lib/s3_relay/base.rb +35 -0
  14. data/lib/s3_relay/engine.rb +18 -0
  15. data/lib/s3_relay/model.rb +54 -0
  16. data/lib/s3_relay/private_url.rb +33 -0
  17. data/lib/s3_relay/s3_relay.rb +4 -0
  18. data/lib/s3_relay/upload_presigner.rb +60 -0
  19. data/lib/s3_relay/version.rb +3 -0
  20. data/test/controllers/s3_relay/uploads_controller_test.rb +138 -0
  21. data/test/dummy/README.rdoc +28 -0
  22. data/test/dummy/Rakefile +6 -0
  23. data/test/dummy/app/assets/javascripts/application.js +13 -0
  24. data/test/dummy/app/assets/stylesheets/application.css +15 -0
  25. data/test/dummy/app/controllers/application_controller.rb +5 -0
  26. data/test/dummy/app/helpers/application_helper.rb +2 -0
  27. data/test/dummy/app/models/product.rb +6 -0
  28. data/test/dummy/app/views/layouts/application.html.erb +14 -0
  29. data/test/dummy/bin/bundle +3 -0
  30. data/test/dummy/bin/rails +4 -0
  31. data/test/dummy/bin/rake +4 -0
  32. data/test/dummy/config.ru +4 -0
  33. data/test/dummy/config/application.rb +23 -0
  34. data/test/dummy/config/boot.rb +5 -0
  35. data/test/dummy/config/database.yml +22 -0
  36. data/test/dummy/config/environment.rb +5 -0
  37. data/test/dummy/config/environments/development.rb +37 -0
  38. data/test/dummy/config/environments/production.rb +78 -0
  39. data/test/dummy/config/environments/test.rb +39 -0
  40. data/test/dummy/config/initializers/assets.rb +8 -0
  41. data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
  42. data/test/dummy/config/initializers/cookies_serializer.rb +3 -0
  43. data/test/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  44. data/test/dummy/config/initializers/inflections.rb +16 -0
  45. data/test/dummy/config/initializers/mime_types.rb +4 -0
  46. data/test/dummy/config/initializers/session_store.rb +3 -0
  47. data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
  48. data/test/dummy/config/locales/en.yml +23 -0
  49. data/test/dummy/config/routes.rb +7 -0
  50. data/test/dummy/config/secrets.yml +22 -0
  51. data/test/dummy/db/migrate/20141021002149_create_products.rb +9 -0
  52. data/test/dummy/db/schema.rb +41 -0
  53. data/test/dummy/log/development.log +53 -0
  54. data/test/dummy/log/test.log +14631 -0
  55. data/test/dummy/public/404.html +67 -0
  56. data/test/dummy/public/422.html +67 -0
  57. data/test/dummy/public/500.html +66 -0
  58. data/test/dummy/public/favicon.ico +0 -0
  59. data/test/factories/products.rb +5 -0
  60. data/test/factories/uploads.rb +15 -0
  61. data/test/helpers/s3_relay/uploads_helper_test.rb +13 -0
  62. data/test/lib/s3_relay/model_test.rb +81 -0
  63. data/test/lib/s3_relay/private_url_test.rb +28 -0
  64. data/test/lib/s3_relay/upload_presigner_test.rb +38 -0
  65. data/test/models/s3_relay/upload_test.rb +128 -0
  66. data/test/support/database_cleaner.rb +14 -0
  67. data/test/test_helper.rb +28 -0
  68. metadata +302 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: dcccfe11a546093475043762594d3df9af34b075
4
+ data.tar.gz: 79acd9bdd1b6b6a6877341a08b9ec5c923fb3f3d
5
+ SHA512:
6
+ metadata.gz: 504414953661b8a7fc6bc069fe90eef08005f1fdd0d5a3918a585aa132f684a831bee1c286475c722d129d331890b0545eb0d5e601c9fd3f3a35e94c94a265e2
7
+ data.tar.gz: 94578b8eec4bbe0a24855359d0098aaac6bbbebcf8e239d9a3b811ab98fe973eef0b37d625be722d5cf7dd56e55a99c5f76ac8adc2300f3ecebd7375784b87f5
@@ -0,0 +1,20 @@
1
+ Copyright 2014 Kenny Johnston
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,209 @@
1
+ # s3_relay
2
+
3
+ Enables direct file uploads to Amazon S3 and provides a flexible pattern for
4
+ your Rails app to asynchronously ingest the files.
5
+
6
+ ## Overview
7
+
8
+ This Rails engine allows you to quickly implement direct uploads to Amazon S3
9
+ from your Rails 3.1+ / 4.x application. It does not depend on any specific file
10
+ upload libraries, UI frameworks or AWS gems, like other solutions tend to.
11
+
12
+ It works by utilizing Amazon S3's
13
+ [Cross-Origin Resource Sharing](http://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html)
14
+ to permit browser-based uploads directly to S3 with presigned URLs generated by
15
+ this gem with your application's API credentials. As each file is uploaded,
16
+ the gem persists detail about the uploaded file in your application's database.
17
+ This table should be thought of much like a queue - think
18
+ [DelayedJob](https://github.com/collectiveidea/delayed_job) for your
19
+ uploaded-but-not-yet-ingested file uploads.
20
+
21
+ How (and if) you choose to import each uploaded file into your processing
22
+ library of choice is completely up to you. The gem tracks the state of each
23
+ upload so that you may used the provided `.pending` scope and `mark_imported!`
24
+ method to fetch, process (via your background processor of choice), then
25
+ mark-off each upload record whose file has been successfully ingested by your
26
+ app.
27
+
28
+ ## Features
29
+
30
+ * Files can be uploaded before or after your parent object has been saved.
31
+ * File upload fields can be placed inside or outside of your parent object's
32
+ form.
33
+ * File uploads display completion progress.
34
+ * Boilerplate styling can be used or easily replaced.
35
+ * All uploads are set to private by default, however support for other ACLs
36
+ is in the plans.
37
+ * All uploads are marked for [Server-Side Encryption by AWS](http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingEncryption.html) so that they are encrypted upon
38
+ write to disk. S3 then decrypts and streams the files as they are downloaded.
39
+ * Models can have multiple types of uploads and specify for each if only one
40
+ file is permitted or if multiple are.
41
+ * Multiple files can upload concurrently to S3.
42
+
43
+ ## Technology & Requirements
44
+
45
+ Uploads are made possible by use of the `FormData` object, defined in
46
+ [XMLHttpRequest Level 2](http://dev.w3.org/2006/webapi/XMLHttpRequest-2/).
47
+ Many people are broadly referring to this as being provided by HTML5, but
48
+ technically it's part of the aforementioned spec that browsers have been
49
+ adhering to for a couple of major versions now. Even IE, wuh?
50
+
51
+ The latest versions of all of the following are ideal, but here are the gem's
52
+ minimum requirements:
53
+
54
+ * Ruby 1.9.3+
55
+ * Rails 3.1+
56
+ * Modern versions of Chrome, Safari, FireFox or IE 10+
57
+ * Note: Progress bars are currently disabled in IE
58
+ * Note: IE <= 9 users will be instructed to upgrade their browser upon
59
+ selecting a file
60
+
61
+ ## Demo
62
+
63
+ See a demo application using `s3_relay` [here](https://github.com/kjohnston/s3_relay-demo).
64
+
65
+ ## Configuring CORS
66
+
67
+ Edit your S3 bucket's CORS Configuration to resemble the following:
68
+
69
+ ```
70
+ <CORSConfiguration>
71
+ <CORSRule>
72
+ <AllowedOrigin>*</AllowedOrigin>
73
+ <AllowedMethod>POST</AllowedMethod>
74
+ <AllowedHeader>Content-Type</AllowedHeader>
75
+ <AllowedHeader>origin</AllowedHeader>
76
+ </CORSRule>
77
+ </CORSConfiguration>
78
+ ```
79
+
80
+ Note: The example above is a starting point for development. Obviously, you
81
+ don't want to permit requests from any domain to upload to your S3 bucket.
82
+ Please see the [AWS Documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/cors.html)
83
+ to learn how to lock it down further.
84
+
85
+ ## Installation
86
+
87
+ * Add `gem "s3_relay"` to your Gemfile and run `bundle`.
88
+ * Add migrations to your app with `rake s3_relay:install:migrations db:migrate`.
89
+ * Add `mount S3Relay::Engine => "/s3_relay"` to the top of your routes file.
90
+ * Add `require s3_relay` to your JavaScript manifest.
91
+ * [Optional] Add `require s3_relay` to your Style Sheet manifest.
92
+ * Add the following environment variables to your app:
93
+
94
+ ```
95
+ S3_RELAY_ACCESS_KEY_ID="abc123"
96
+ S3_RELAY_SECRET_ACCESS_KEY="xzy456"
97
+ S3_RELAY_REGION="us-west-2"
98
+ S3_RELAY_BUCKET="some-s3-bucket"
99
+ S3_RELAY_ACL="private"
100
+ ```
101
+
102
+ ## Use
103
+
104
+ ### Add upload definitions to your model
105
+
106
+ ```ruby
107
+ class Product < ActiveRecord::Base
108
+ s3_relay :icon_upload
109
+ s3_relay :photo_uploads, has_many: true
110
+ end
111
+ ```
112
+
113
+ ### Restricting uploads to authenticated users
114
+
115
+ If your app's file uploads need to be restricted to logged in users, simply
116
+ override the following method in your application controller to call any
117
+ authentication method you're currently using.
118
+
119
+ ```ruby
120
+ def authenticate_for_s3_relay
121
+ authenticate_user! # Devise example
122
+ end
123
+ ```
124
+
125
+ ### Add virtual attributes to your controller's Strong Parameters config
126
+
127
+ ```ruby
128
+ product_params = params.require(:product)
129
+ .permit(:name, :new_icon_upload_uuid, new_photo_uploads_uuids: [])
130
+
131
+ @product = Product.new(product_params)
132
+ ```
133
+
134
+ ### Add file upload fields to your views
135
+
136
+ ```erb
137
+ <%= s3_relay_field @product, :icon_upload %>
138
+ <%= s3_relay_field @product, :photo_uploads, multiple: true %>
139
+ ```
140
+
141
+ ### Process uploads asynchronously
142
+
143
+ Use your background job processor of choice to process uploads pending
144
+ ingestion (and image processing) by your app.
145
+
146
+ Say you're using [Resque](https://github.com/resque/resque) and [CarrierWave](https://github.com/carrierwaveuploader/carrierwave), you could define a job class:
147
+
148
+ ```ruby
149
+ class ProductPhotoImporter
150
+ @queue = :photo_import
151
+
152
+ def self.perform(product_id)
153
+ @product = Product.find(id)
154
+
155
+ @product.photo_uploads.pending.each do |upload|
156
+ @product.photos.create(remote_file_url: upload.private_url)
157
+ upload.mark_processed!
158
+ end
159
+ end
160
+ end
161
+ ```
162
+
163
+ Then, enqueue a job from your controller, callback, service object, etc.
164
+ whenever there may be new uploads to process:
165
+
166
+ ```ruby
167
+ Resque.enqueue(ProductPhotoImporter, product.id)
168
+ ```
169
+
170
+ ### Restricting the objects uploads can be associated with
171
+
172
+ Remember the time when that guy found a way to a submit Github form in such a
173
+ way that it linked a new SSH key he provided to DHH's user record? No bueno.
174
+ Don't let your users attach files to objects they don't have access to.
175
+
176
+ You can prevent this by defining a method in ApplicationController that
177
+ filters out the parent object params passed during upload creation if your logic
178
+ finds that the user doesn't have access to the parent object in question. Ex:
179
+
180
+ ```ruby
181
+ def order_file_uploads_params(parent)
182
+ if parent.user == current_user
183
+ # Yep, that's your order, you can add files to it
184
+ { parent: parent }
185
+ else
186
+ # Nope, you're trying to add a file to someone else's order, or whatever
187
+ { }
188
+ end
189
+ end
190
+ ```
191
+
192
+ ## Contributing
193
+
194
+ 1. [Fork it](https://github.com/kjohnston/s3_relay/fork)
195
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
196
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
197
+ 4. Push to the branch (`git push origin my-new-feature`)
198
+ 5. [Create a Pull Request](https://github.com/kjohnston/s3_relay/pull/new)
199
+
200
+ ## Contributors
201
+
202
+ Many thanks go to the following who have contributed to making this gem even better:
203
+
204
+ [your name here]
205
+
206
+ ## License
207
+
208
+ * Freely distributable and licensed under the [MIT license](http://kjohnston.mit-license.org/license.html).
209
+ * Copyright (c) 2014 Kenny Johnston
@@ -0,0 +1,31 @@
1
+ begin
2
+ require "bundler/setup"
3
+ rescue LoadError
4
+ puts "You must `gem install bundler` and `bundle install` to run rake tasks"
5
+ end
6
+
7
+ require "rdoc/task"
8
+
9
+ RDoc::Task.new(:rdoc) do |rdoc|
10
+ rdoc.rdoc_dir = "rdoc"
11
+ rdoc.title = "S3Relay"
12
+ rdoc.options << "--line-numbers"
13
+ rdoc.rdoc_files.include("README.rdoc")
14
+ rdoc.rdoc_files.include("lib/**/*.rb")
15
+ end
16
+
17
+ APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
18
+ load "rails/tasks/engine.rake"
19
+
20
+ Bundler::GemHelper.install_tasks
21
+
22
+ require "rake/testtask"
23
+
24
+ Rake::TestTask.new(:test) do |t|
25
+ t.libs << "lib"
26
+ t.libs << "test"
27
+ t.pattern = "test/**/*_test.rb"
28
+ t.verbose = false
29
+ end
30
+
31
+ task default: :test
@@ -0,0 +1,112 @@
1
+ displayFailedUpload = (progressColumn=null) ->
2
+ if progressColumn
3
+ progressColumn.text("File could not be uploaded")
4
+ else
5
+ alert("File could not be uploaded")
6
+
7
+ saveUrl = (container, uuid, filename, contentType, publicUrl) ->
8
+ privateUrl = null
9
+
10
+ $.ajax
11
+ type: "POST"
12
+ url: "/s3_relay/uploads"
13
+ async: false
14
+ data:
15
+ parent_type: container.data("parentType")
16
+ parent_id: container.data("parentId")
17
+ association: container.data("association")
18
+ uuid: uuid
19
+ filename: filename
20
+ content_type: contentType
21
+ public_url: publicUrl
22
+ success: (data, status, xhr) ->
23
+ privateUrl = data.private_url
24
+ error: (xhr) ->
25
+ console.log xhr.responseText
26
+
27
+ return privateUrl
28
+
29
+ uploadFiles = (container) ->
30
+ fileInput = $("input.s3r-field", container)
31
+ files = fileInput.get(0).files
32
+ uploadFile(container, file) for file in files
33
+ fileInput.val("")
34
+
35
+ uploadFile = (container, file) ->
36
+ fileName = file.name
37
+
38
+ $.ajax
39
+ type: "GET"
40
+ url: "/s3_relay/uploads/new"
41
+ async: false
42
+ success: (data, status, xhr) ->
43
+ formData = new FormData()
44
+ xhr = new XMLHttpRequest()
45
+ endpoint = data.endpoint
46
+
47
+ formData.append("AWSAccessKeyID", data.awsaccesskeyid)
48
+ formData.append("x-amz-server-side-encryption", data.x_amz_server_side_encryption)
49
+ formData.append("key", data.key)
50
+ formData.append("success_action_status", data.success_action_status)
51
+ formData.append("acl", data.acl)
52
+ formData.append("policy", data.policy)
53
+ formData.append("signature", data.signature)
54
+ formData.append("content-type", file.type)
55
+ formData.append("file", file)
56
+
57
+ uuid = data.uuid
58
+
59
+ uploadList = $(".s3r-upload-list", container)
60
+ uploadList.prepend("<tr id='#{uuid}'><td><div class='s3r-file-url'>#{fileName}</div></td><td class='s3r-progress'><div class='s3r-bar' style='display: none;'><div class='s3r-meter'></div></div></td></tr>")
61
+ fileColumn = $(".s3r-upload-list ##{uuid} .s3r-file-url", container)
62
+ progressColumn = $(".s3r-upload-list ##{uuid} .s3r-progress", container)
63
+ progressBar = $(".s3r-bar", progressColumn)
64
+ progressMeter = $(".s3r-meter", progressColumn)
65
+
66
+ xhr.upload.addEventListener "progress", (ev) ->
67
+ if ev.position
68
+ percentage = ((ev.position / ev.totalSize) * 100.0).toFixed(0)
69
+ progressBar.show()
70
+ progressMeter.css "width", "#{percentage}%"
71
+ else
72
+ progressColumn.text("Uploading...") # IE can't get position
73
+
74
+ xhr.onreadystatechange = (ev) ->
75
+ if xhr.readyState is 4
76
+ progressColumn.text("") # IE can't get position
77
+ progressBar.remove()
78
+
79
+ if xhr.status == 201
80
+ contentType = file.type
81
+ publicUrl = $("Location", xhr.responseXML).text()
82
+ privateUrl = saveUrl(container, uuid, fileName, contentType, publicUrl)
83
+
84
+ if privateUrl == null
85
+ displayFailedUpload(progressColumn)
86
+ else
87
+ fileColumn.html("<a href='#{privateUrl}'>#{fileName}</a>")
88
+
89
+ virtualAttr = "#{container.data('parentType')}[new_#{container.data('association')}_uuids]"
90
+ hiddenField = "<input type='hidden' name='#{virtualAttr}[]' value='#{uuid}' />"
91
+ container.append(hiddenField)
92
+
93
+ else
94
+ displayFailedUpload(progressColumn)
95
+ console.log $("Message", xhr.responseXML).text()
96
+
97
+ xhr.open "POST", endpoint, true
98
+ xhr.send formData
99
+ error: (xhr) ->
100
+ displayFailedUpload()
101
+ console.log xhr.responseText
102
+
103
+ jQuery ->
104
+
105
+ $(document).on "change", ".s3r-field", ->
106
+ $this = $(this)
107
+
108
+ if !!window.FormData
109
+ uploadFiles($this.parent())
110
+ else
111
+ $this.hide()
112
+ $this.parent().append("<p>Your browser can't handle file uploads, please switch to <a href='http://google.com/chrome'>Google Chrome</a>.</p>")
@@ -0,0 +1,31 @@
1
+ .s3r-upload-list {
2
+ border: 1px solid #ccc;
3
+ border-collapse: collapse;
4
+ margin: 15px 0;
5
+ }
6
+
7
+ .s3r-upload-list td {
8
+ border-bottom: 1px solid #ccc;
9
+ padding: .5em;
10
+ width: 180px;
11
+ }
12
+
13
+ .s3r-file-url {
14
+ overflow: hidden;
15
+ text-overflow: ellipsis;
16
+ white-space: nowrap;
17
+ width: 180px;
18
+ }
19
+
20
+ .s3r-bar {
21
+ background-color: #eee;
22
+ border-radius: 3px;
23
+ height: 15px;
24
+ width: 180px;
25
+ }
26
+
27
+ .s3r-meter {
28
+ background-color: #43ac6a;
29
+ border-radius: 3px;
30
+ height: 15px;
31
+ }
@@ -0,0 +1,65 @@
1
+ class S3Relay::UploadsController < ApplicationController
2
+
3
+ before_filter :authenticate
4
+ skip_before_filter :verify_authenticity_token
5
+
6
+ def new
7
+ render json: S3Relay::UploadPresigner.new.form_data
8
+ end
9
+
10
+ def create
11
+ @upload = S3Relay::Upload.new(upload_attrs)
12
+
13
+ if @upload.save
14
+ render json: { private_url: @upload.private_url }, status: 201
15
+ else
16
+ render json: { errors: @upload.errors }, status: 422
17
+ end
18
+ end
19
+
20
+ protected
21
+
22
+ def authenticate
23
+ if respond_to?(:authenticate_for_s3_relay)
24
+ authenticate_for_s3_relay
25
+ end
26
+ end
27
+
28
+ def parent_attrs
29
+ type = params[:parent_type].try(:classify)
30
+ id = params[:parent_id]
31
+
32
+ return {} unless type.present? && id.present? &&
33
+ parent = type.constantize.find_by_id(id)
34
+
35
+ begin
36
+ public_send(
37
+ "#{type.underscore.downcase}_#{params[:association]}_params",
38
+ parent
39
+ )
40
+ rescue NoMethodError
41
+ { parent: parent }
42
+ end
43
+ end
44
+
45
+ def upload_attrs
46
+ attrs = {
47
+ upload_type: params[:association].try(:classify),
48
+ uuid: params[:uuid],
49
+ filename: params[:filename],
50
+ content_type: params[:content_type]
51
+ }
52
+
53
+ attrs.merge!(parent_attrs)
54
+ attrs.merge!(user_attrs)
55
+ end
56
+
57
+ def user_attrs
58
+ if respond_to?(:current_user) && (id = current_user.try(:id))
59
+ { user_id: id }
60
+ else
61
+ {}
62
+ end
63
+ end
64
+
65
+ end