activestorage-storj 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 19bb5d6b6d62fa2f84ac8e7e919cc298d4c4dfa9b36f6780878f945df2d70091
4
+ data.tar.gz: 7dda9ebf7267dce28bbe2cddb58b09419274b3b64ce3682f75c89296c92ed5f9
5
+ SHA512:
6
+ metadata.gz: 53adbe942d89be9472ff353702330c28452759ff0b57190b074d48906296d43ed1146edba701607294a1cd37d437cf0fd86d9020094cedeeebb535c6672ca277
7
+ data.tar.gz: 14e45036c1a01f2be119cb5d6fe5a89756e4cee48568576dee8825977da449748e38ead43f67a62ba279dc61add8bcfabad2ce31313f0949b682321e2cae8a2c
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 Your Data
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,85 @@
1
+ # ActiveStorage-Storj
2
+ ActiveStorage-Storj is a ruby gem that provides [Storj](https://www.storj.io/) cloud storage support for [ActiveStorage](https://guides.rubyonrails.org/active_storage_overview.html) in [Rails](https://rubyonrails.org/).
3
+
4
+ Note: [direct upload](https://guides.rubyonrails.org/active_storage_overview.html#direct-uploads) is not supported in this gem. To enable direct upload support, install [activestorage-storj-s3](https://github.com/Your-Data/activestorage-storj-s3) gem.
5
+
6
+ ## Requirements
7
+ * [Rails v7+](https://guides.rubyonrails.org/getting_started.html)
8
+ * Install Active Storage on a Rails project.
9
+
10
+ ```bash
11
+ $ bin/rails active_storage:install
12
+ $ bin/rails db:migrate
13
+ ```
14
+ * Build and install [uplink-c](https://github.com/storj/uplink-c) library. Follow the guide at [Prerequisites](https://github.com/storj-thirdparty/uplink-ruby#prerequisites) section.
15
+
16
+ ## Installation
17
+ * Add this line to your Rails application's Gemfile:
18
+
19
+ ```ruby
20
+ gem 'activestorage-storj', '~> 1.0'
21
+ ```
22
+
23
+ And then execute:
24
+ ```bash
25
+ $ bundle install
26
+ ```
27
+
28
+ * Declare a Storj Cloud Storage service in `config/storage.yml`:
29
+
30
+ ```yaml
31
+ storj:
32
+ service: storj
33
+ access_grant: ""
34
+ bucket: ""
35
+ auth_service_address: auth.storjshare.io:7777
36
+ link_sharing_address: https://link.storjshare.io
37
+ ```
38
+
39
+ Optionally provide upload and download options:
40
+
41
+ ```yaml
42
+ storj:
43
+ service: storj
44
+ ...
45
+ upload_chunk_size: 0
46
+ download_chunk_size: 0
47
+ ```
48
+
49
+ Add `public: true` to prevent the generated URL from expiring. By default, the private generated URL will expire in 5 minutes.
50
+
51
+ ```yaml
52
+ storj:
53
+ service: storj
54
+ ...
55
+ public: true
56
+ ```
57
+
58
+ ## Running the Tests
59
+
60
+ * Create `configurations.yml` file in `test/dummy/config/environments/service` folder, or copy the existing `configurations.example.yml` as `configurations.yml`.
61
+
62
+ * Provide Storj configurations for both `storj` and `storj_public` services in `configurations.yml`:
63
+
64
+ ```yaml
65
+ storj:
66
+ service: storj
67
+ access_grant: ""
68
+ bucket: ""
69
+ auth_service_address: auth.storjshare.io:7777
70
+ link_sharing_address: https://link.storjshare.io
71
+
72
+ storj_public:
73
+ service: storj
74
+ access_grant: ""
75
+ bucket: ""
76
+ auth_service_address: auth.storjshare.io:7777
77
+ link_sharing_address: https://link.storjshare.io
78
+ public: true
79
+ ```
80
+
81
+ * Run the tests:
82
+
83
+ ```bash
84
+ $ bin/test
85
+ ```
data/Rakefile ADDED
@@ -0,0 +1,3 @@
1
+ require "bundler/setup"
2
+
3
+ require "bundler/gem_tasks"
@@ -0,0 +1,313 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem "uplink-ruby", "~> 1.0"
4
+
5
+ require "uplink"
6
+
7
+ module ActiveStorage
8
+ MULTIPART_UPLOAD_THRESHOLD = 5.megabytes
9
+ AUTH_SERVICE_ADDRESS = "auth.storjshare.io:7777"
10
+ LINK_SHARING_ADDRESS = "https://link.storjshare.io"
11
+
12
+ # = Active Storage \Storj \Service
13
+ #
14
+ # Wraps the Storj as an Active Storage service.
15
+ # See ActiveStorage::Service for the generic API documentation that applies to all services.
16
+ class Service::StorjService < Service
17
+ attr_reader :bucket
18
+
19
+ def initialize(access_grant:, bucket:, upload_chunk_size: nil, download_chunk_size: nil, multipart_upload_threshold: nil,
20
+ auth_service_address: nil, link_sharing_address: nil, public: false, **config)
21
+ @access_grant = access_grant
22
+ @bucket = bucket
23
+ @upload_chunk_size = upload_chunk_size || MULTIPART_UPLOAD_THRESHOLD
24
+ @download_chunk_size = download_chunk_size || MULTIPART_UPLOAD_THRESHOLD
25
+ @multipart_upload_threshold = [multipart_upload_threshold || MULTIPART_UPLOAD_THRESHOLD, MULTIPART_UPLOAD_THRESHOLD].max
26
+ @auth_service_address = auth_service_address || AUTH_SERVICE_ADDRESS
27
+ @link_sharing_address = link_sharing_address || LINK_SHARING_ADDRESS
28
+ @public = public
29
+ @config = config
30
+ end
31
+
32
+ def upload(key, io, checksum: nil, content_type: nil, disposition: nil, filename: nil, custom_metadata: {})
33
+ instrument :upload, key: key, checksum: checksum do
34
+ contents = io.read
35
+
36
+ if checksum.present?
37
+ md5_hash = OpenSSL::Digest::MD5.base64digest(contents)
38
+ raise ActiveStorage::IntegrityError if md5_hash != checksum
39
+ end
40
+
41
+ Uplink.parse_access(@access_grant) do |access|
42
+ access.open_project do |project|
43
+ project.ensure_bucket(@bucket)
44
+
45
+ content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
46
+
47
+ upload_object(project, key, contents, content_type, content_disposition, custom_metadata)
48
+ end
49
+ end
50
+ end
51
+ end
52
+
53
+ def update_metadata(key, content_type:, disposition: nil, filename: nil, custom_metadata: {})
54
+ instrument :update_metadata, key: key, content_type: content_type, disposition: disposition do
55
+ Uplink.parse_access(@access_grant) do |access|
56
+ access.open_project do |project|
57
+ content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
58
+
59
+ project.update_object_metadata(@bucket, key, custom_metadata.merge({ "content-type": content_type, "content-disposition": content_disposition }))
60
+ end
61
+ end
62
+ end
63
+ end
64
+
65
+ def download(key, &block)
66
+ if block_given?
67
+ instrument :streaming_download, key: key do
68
+ stream(key, &block)
69
+ end
70
+ else
71
+ instrument :download, key: key do
72
+ Uplink.parse_access(@access_grant) do |access|
73
+ access.open_project do |project|
74
+ download_object(project, key)
75
+ end
76
+ end
77
+ rescue Uplink::ObjectKeyNotFoundError
78
+ raise ActiveStorage::FileNotFoundError
79
+ end
80
+ end
81
+ end
82
+
83
+ def download_chunk(key, range)
84
+ instrument :download_chunk, key: key, range: range do
85
+ Uplink.parse_access(@access_grant) do |access|
86
+ access.open_project do |project|
87
+ download_object(project, key, range)
88
+ end
89
+ end
90
+ rescue Uplink::ObjectKeyNotFoundError
91
+ raise ActiveStorage::FileNotFoundError
92
+ end
93
+ end
94
+
95
+ def compose(source_keys, destination_key, filename: nil, content_type: nil, disposition: nil, custom_metadata: {})
96
+ Uplink.parse_access(@access_grant) do |access|
97
+ access.open_project do |project|
98
+
99
+ contents = ''
100
+ source_keys.each do |source_key|
101
+ contents += download_object(project, source_key)
102
+ end
103
+
104
+ content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
105
+
106
+ upload_object(project, destination_key, contents, content_type, content_disposition, custom_metadata)
107
+ end
108
+ end
109
+ end
110
+
111
+ def delete(key)
112
+ instrument :delete, key: key do
113
+ Uplink.parse_access(@access_grant) do |access|
114
+ access.open_project do |project|
115
+ project.delete_object(@bucket, key)
116
+ end
117
+ end
118
+ rescue Uplink::ObjectKeyNotFoundError
119
+ # Ignore files already deleted
120
+ end
121
+ end
122
+
123
+ def delete_prefixed(prefix)
124
+ instrument :delete_prefixed, prefix: prefix do
125
+ Uplink.parse_access(@access_grant) do |access|
126
+ access.open_project do |project|
127
+ objects = []
128
+
129
+ project.list_objects(@bucket, { prefix: prefix }) do |it|
130
+ while it.next?
131
+ object = it.item
132
+ project.delete_object(@bucket, object.key)
133
+ end
134
+ end
135
+ end
136
+ end
137
+ end
138
+ end
139
+
140
+ def exist?(key)
141
+ instrument :exist, key: key do |payload|
142
+ Uplink.parse_access(@access_grant) do |access|
143
+ access.open_project do |project|
144
+ object = project.stat_object(@bucket, key)
145
+ answer = object.key == key
146
+ payload[:exist] = answer
147
+ answer
148
+ end
149
+ end
150
+ rescue Uplink::ObjectKeyNotFoundError
151
+ answer = false
152
+ payload[:exist] = answer
153
+ answer
154
+ end
155
+ end
156
+
157
+ def object(key)
158
+ Uplink.parse_access(@access_grant) do |access|
159
+ access.open_project do |project|
160
+ project.stat_object(@bucket, key)
161
+ end
162
+ end
163
+ rescue Uplink::ObjectKeyNotFoundError
164
+ raise ActiveStorage::FileNotFoundError
165
+ end
166
+
167
+ def headers_for_direct_upload(key, content_type:, checksum:, filename: nil, disposition: nil, custom_metadata: {}, **)
168
+ content_disposition = content_disposition_with(type: disposition, filename: filename) if disposition && filename
169
+
170
+ { "Content-Type" => content_type, "Content-MD5" => checksum, "Content-Disposition" => content_disposition, **custom_metadata_headers(custom_metadata) }
171
+ end
172
+
173
+ private
174
+ def upload_object(project, key, contents, content_type, content_disposition, custom_metadata = {})
175
+ if contents.size <= @multipart_upload_threshold
176
+ upload_with_single_part(project, key, contents, content_type, content_disposition, custom_metadata)
177
+ else
178
+ upload_with_multipart(project, key, contents, content_type, content_disposition, custom_metadata)
179
+ end
180
+ end
181
+
182
+ def upload_with_single_part(project, key, contents, content_type, content_disposition, custom_metadata = {})
183
+ project.upload_object(@bucket, key) do |upload|
184
+ chunk_size = @upload_chunk_size
185
+
186
+ file_size = contents.size
187
+ uploaded_total = 0
188
+
189
+ while uploaded_total < file_size
190
+ upload_size_left = file_size - uploaded_total
191
+ len = chunk_size <= 0 ? upload_size_left : [chunk_size, upload_size_left].min
192
+
193
+ bytes_written = upload.write(contents[uploaded_total, len], len)
194
+ uploaded_total += bytes_written
195
+ end
196
+
197
+ upload.set_custom_metadata(custom_metadata.merge({ "content-type": content_type, "content-disposition": content_disposition }))
198
+
199
+ upload.commit
200
+ end
201
+ end
202
+
203
+ def upload_with_multipart(project, key, contents, content_type, content_disposition, custom_metadata = {})
204
+ file_size = contents.size
205
+ part_size = @multipart_upload_threshold
206
+ part_count = (file_size.to_f / @multipart_upload_threshold).ceil
207
+
208
+ chunk_size = @upload_chunk_size
209
+ uploaded_total = 0
210
+
211
+ upload_info = project.begin_upload(@bucket, key)
212
+
213
+ part_count.times do |i|
214
+ project.upload_part(@bucket, key, upload_info.upload_id, i + 1) do |part_upload|
215
+ upload_size = [(i + 1) * part_size, file_size].min
216
+
217
+ while uploaded_total < upload_size
218
+ upload_size_left = upload_size - uploaded_total
219
+ len = chunk_size <= 0 ? upload_size_left : [chunk_size, upload_size_left].min
220
+
221
+ bytes_written = part_upload.write(contents[uploaded_total, len], len)
222
+ uploaded_total += bytes_written
223
+ end
224
+
225
+ part_upload.commit
226
+ end
227
+ end
228
+
229
+ upload_options = {
230
+ custom_metadata: custom_metadata.merge({ "content-type": content_type, "content-disposition": content_disposition })
231
+ }
232
+ project.commit_upload(@bucket, key, upload_info.upload_id, upload_options)
233
+ end
234
+
235
+ def download_object(project, key, range = nil)
236
+ project.download_object(@bucket, key, range.present? ? { offset: range.begin, length: range.size } : nil) do |download|
237
+ downloaded_data = []
238
+
239
+ object = download.info
240
+ file_size = object.content_length
241
+
242
+ chunk_size = @download_chunk_size
243
+ downloaded_total = 0
244
+
245
+ loop do
246
+ download_size_left = file_size - downloaded_total
247
+ len = chunk_size <= 0 ? download_size_left : [chunk_size, download_size_left].min
248
+
249
+ bytes_read, is_eof = download.read(downloaded_data, len)
250
+ downloaded_total += bytes_read
251
+
252
+ break if is_eof
253
+ end
254
+
255
+ downloaded_data.pack('C*')
256
+ end
257
+ end
258
+
259
+ # Reads the object for the given key in chunks, yielding each to the block.
260
+ def stream(key)
261
+ Uplink.parse_access(@access_grant) do |access|
262
+ access.open_project do |project|
263
+ project.download_object(@bucket, key) do |download|
264
+ object = download.info
265
+ file_size = object.content_length
266
+
267
+ chunk_size = @download_chunk_size
268
+ downloaded_total = 0
269
+
270
+ loop do
271
+ download_size_left = file_size - downloaded_total
272
+ len = chunk_size <= 0 ? download_size_left : [chunk_size, download_size_left].min
273
+
274
+ downloaded_data = []
275
+ bytes_read, is_eof = download.read(downloaded_data, len)
276
+ downloaded_total += bytes_read
277
+
278
+ break if is_eof
279
+
280
+ yield downloaded_data.pack('C*')
281
+ end
282
+ end
283
+ end
284
+ end
285
+ rescue Uplink::ObjectKeyNotFoundError
286
+ raise ActiveStorage::FileNotFoundError
287
+ end
288
+
289
+ def private_url(key, expires_in:, **options)
290
+ linkshare_url(key, expires_in: expires_in, **options)
291
+ end
292
+
293
+ def public_url(key, **options)
294
+ linkshare_url(key, expires_in: nil, **options)
295
+ end
296
+
297
+ def linkshare_url(key, expires_in:, **options)
298
+ Uplink.parse_access(@access_grant) do |access|
299
+ permission = { allow_download: true, not_after: expires_in.present? ? Time.current + expires_in.to_i : nil }
300
+ prefixes = [ { bucket: @bucket } ]
301
+
302
+ access.share(permission, prefixes) do |shared_access|
303
+ edge_credential = shared_access.edge_register_access({ auth_service_address: @auth_service_address }, { is_public: true })
304
+ edge_credential.join_share_url(@link_sharing_address, @bucket, key, { raw: true })
305
+ end
306
+ end
307
+ end
308
+
309
+ def custom_metadata_headers(metadata)
310
+ metadata.transform_keys { |key| "x-amz-meta-#{key}" }
311
+ end
312
+ end
313
+ end
@@ -0,0 +1,6 @@
1
+ module ActiveStorage
2
+ module Storj
3
+ class Railtie < ::Rails::Railtie
4
+ end
5
+ end
6
+ end
@@ -0,0 +1,5 @@
1
+ module ActiveStorage
2
+ module Storj
3
+ VERSION = "1.0.0"
4
+ end
5
+ end
@@ -0,0 +1,4 @@
1
+ # desc "Explaining what the task does"
2
+ # task :activestorage_storj do
3
+ # # Task goes here
4
+ # end
metadata ADDED
@@ -0,0 +1,83 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: activestorage-storj
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Your Data Inc
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2023-06-30 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rails
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '7.0'
20
+ - - ">="
21
+ - !ruby/object:Gem::Version
22
+ version: 7.0.4
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - "~>"
28
+ - !ruby/object:Gem::Version
29
+ version: '7.0'
30
+ - - ">="
31
+ - !ruby/object:Gem::Version
32
+ version: 7.0.4
33
+ - !ruby/object:Gem::Dependency
34
+ name: uplink-ruby
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '1.0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '1.0'
47
+ description: Providing Storj Cloud Storage support for ActiveStorage in Rails
48
+ email:
49
+ executables: []
50
+ extensions: []
51
+ extra_rdoc_files: []
52
+ files:
53
+ - LICENSE
54
+ - README.md
55
+ - Rakefile
56
+ - lib/active_storage/service/storj_service.rb
57
+ - lib/active_storage/storj/railtie.rb
58
+ - lib/active_storage/storj/version.rb
59
+ - lib/tasks/activestorage/storj_tasks.rake
60
+ homepage: https://github.com/Your-Data/activestorage-storj
61
+ licenses:
62
+ - MIT
63
+ metadata: {}
64
+ post_install_message:
65
+ rdoc_options: []
66
+ require_paths:
67
+ - lib
68
+ required_ruby_version: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - ">="
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ required_rubygems_version: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - ">="
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ requirements: []
79
+ rubygems_version: 3.3.26
80
+ signing_key:
81
+ specification_version: 4
82
+ summary: Storj Cloud Storage support for ActiveStorage in Rails
83
+ test_files: []