saviour 0.6.6 → 0.6.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/.travis.yml +3 -2
- data/lib/saviour/life_cycle.rb +12 -28
- data/lib/saviour/s3_storage.rb +7 -1
- data/lib/saviour/uploader/processors_runner.rb +1 -1
- data/lib/saviour/version.rb +1 -1
- data/lib/saviour.rb +2 -1
- data/saviour.gemspec +3 -1
- data/spec/feature/memory_usage_spec.rb +1 -2
- metadata +37 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 94654aa75977e1fc1b51ac4635cf62d99daacfe0c63b94b27132176e1a89b2f8
|
4
|
+
data.tar.gz: 72c77578ddfc92a494b4397a11ee55d5427075158939a3ab82d46fa620dada04
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1261335b9ae8af1929e708769cbcb115cb73e05f458241467a8a93124b198cbb1c7e2faec51a70b79dd068cab016277f84d4478d323bd7b83b25419a4a883ef7
|
7
|
+
data.tar.gz: f9012382aacb98104c4e83fe7049c554dad19ee2fae77f6f9524c43b03f4a531ea432fbd615211b0f2c2dcf7fc6669b3cdf2908a653db2e6aac37c6b7c7cc2e6
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/lib/saviour/life_cycle.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
module Saviour
|
2
2
|
class LifeCycle
|
3
|
+
SHOULD_USE_INTERLOCK = defined?(Rails) && !Rails.env.test?
|
4
|
+
|
3
5
|
class FileCreator
|
4
6
|
def initialize(current_path, file, column, connection)
|
5
7
|
@file = file
|
@@ -84,26 +86,17 @@ module Saviour
|
|
84
86
|
|
85
87
|
def delete!
|
86
88
|
DbHelpers.run_after_commit do
|
87
|
-
pool = Concurrent::
|
89
|
+
pool = Concurrent::Throttle.new Saviour::Config.concurrent_workers
|
88
90
|
|
89
91
|
futures = attached_files.map do |column|
|
90
|
-
|
91
|
-
file = @model.send(column)
|
92
|
+
pool.future(@model.send(column)) do |file|
|
92
93
|
path = file.persisted_path
|
93
94
|
file.uploader.storage.delete(path) if path
|
94
95
|
file.delete
|
95
|
-
}
|
96
|
-
end
|
97
|
-
|
98
|
-
ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
|
99
|
-
futures.each do |future|
|
100
|
-
future.value
|
101
|
-
raise(future.reason) if future.rejected?
|
102
96
|
end
|
103
97
|
end
|
104
98
|
|
105
|
-
|
106
|
-
pool.wait_for_termination
|
99
|
+
futures.each(&:value!)
|
107
100
|
end
|
108
101
|
end
|
109
102
|
|
@@ -131,35 +124,26 @@ module Saviour
|
|
131
124
|
)
|
132
125
|
end.compact
|
133
126
|
|
134
|
-
pool = Concurrent::
|
127
|
+
pool = Concurrent::Throttle.new Saviour::Config.concurrent_workers
|
135
128
|
|
136
129
|
futures = uploaders.map { |uploader|
|
137
|
-
|
138
|
-
if
|
139
|
-
Rails.application.executor.wrap {
|
130
|
+
pool.future(uploader) { |given_uploader|
|
131
|
+
if SHOULD_USE_INTERLOCK
|
132
|
+
Rails.application.executor.wrap { given_uploader.upload }
|
140
133
|
else
|
141
|
-
|
134
|
+
given_uploader.upload
|
142
135
|
end
|
143
136
|
}
|
144
137
|
}
|
145
138
|
|
146
|
-
work = -> {
|
147
|
-
futures.map do |x|
|
148
|
-
x.value.tap do
|
149
|
-
raise(x.reason) if x.rejected?
|
150
|
-
end
|
151
|
-
end.compact
|
152
|
-
}
|
139
|
+
work = -> { futures.map(&:value!).compact }
|
153
140
|
|
154
|
-
result = if
|
141
|
+
result = if SHOULD_USE_INTERLOCK
|
155
142
|
ActiveSupport::Dependencies.interlock.permit_concurrent_loads(&work)
|
156
143
|
else
|
157
144
|
work.call
|
158
145
|
end
|
159
146
|
|
160
|
-
pool.shutdown
|
161
|
-
pool.wait_for_termination
|
162
|
-
|
163
147
|
attrs = result.to_h
|
164
148
|
|
165
149
|
uploaders.map(&:uploader).select { |x| x.class.after_upload_hooks.any? }.each do |uploader|
|
data/lib/saviour/s3_storage.rb
CHANGED
@@ -3,6 +3,8 @@ begin
|
|
3
3
|
rescue LoadError
|
4
4
|
end
|
5
5
|
|
6
|
+
require 'marcel'
|
7
|
+
|
6
8
|
module Saviour
|
7
9
|
class S3Storage
|
8
10
|
MissingPublicUrlPrefix = Class.new(StandardError)
|
@@ -27,8 +29,12 @@ module Saviour
|
|
27
29
|
raise(KeyTooLarge, "The key in S3 must be at max 1024 bytes, this key is too big: #{path}")
|
28
30
|
end
|
29
31
|
|
32
|
+
mime_type = Marcel::MimeType.for file_or_contents
|
33
|
+
|
30
34
|
# TODO: Use multipart api
|
31
|
-
client.put_object(@create_options.merge(
|
35
|
+
client.put_object(@create_options.merge(
|
36
|
+
body: file_or_contents, bucket: @bucket, key: path, content_type: mime_type
|
37
|
+
))
|
32
38
|
end
|
33
39
|
|
34
40
|
def write_from_file(file, path)
|
data/lib/saviour/version.rb
CHANGED
data/lib/saviour.rb
CHANGED
data/saviour.gemspec
CHANGED
@@ -15,9 +15,11 @@ Gem::Specification.new do |spec|
|
|
15
15
|
|
16
16
|
spec.required_ruby_version = ">= 2.5.0"
|
17
17
|
|
18
|
-
spec.add_dependency "activerecord", ">= 5.1"
|
18
|
+
spec.add_dependency "activerecord", ">= 5.1", "< 7.0.0"
|
19
19
|
spec.add_dependency "activesupport", ">= 5.1"
|
20
20
|
spec.add_dependency "concurrent-ruby", ">= 1.0.5"
|
21
|
+
spec.add_dependency "concurrent-ruby-edge", ">= 0.6.0"
|
22
|
+
spec.add_dependency "marcel", ">= 1.0.0"
|
21
23
|
|
22
24
|
spec.add_development_dependency "bundler"
|
23
25
|
spec.add_development_dependency "rspec"
|
@@ -56,8 +56,7 @@ describe "memory usage" do
|
|
56
56
|
|
57
57
|
a.update! file: f
|
58
58
|
|
59
|
-
|
60
|
-
expect(GetProcessMem.new.mb - base_line).to be < size_to_test / 10
|
59
|
+
expect(GetProcessMem.new.mb - base_line).to be < 3
|
61
60
|
end
|
62
61
|
end
|
63
62
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: saviour
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Roger Campos
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-12-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -17,6 +17,9 @@ dependencies:
|
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '5.1'
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 7.0.0
|
20
23
|
type: :runtime
|
21
24
|
prerelease: false
|
22
25
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -24,6 +27,9 @@ dependencies:
|
|
24
27
|
- - ">="
|
25
28
|
- !ruby/object:Gem::Version
|
26
29
|
version: '5.1'
|
30
|
+
- - "<"
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 7.0.0
|
27
33
|
- !ruby/object:Gem::Dependency
|
28
34
|
name: activesupport
|
29
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,6 +58,34 @@ dependencies:
|
|
52
58
|
- - ">="
|
53
59
|
- !ruby/object:Gem::Version
|
54
60
|
version: 1.0.5
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: concurrent-ruby-edge
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: 0.6.0
|
68
|
+
type: :runtime
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 0.6.0
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: marcel
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 1.0.0
|
82
|
+
type: :runtime
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 1.0.0
|
55
89
|
- !ruby/object:Gem::Dependency
|
56
90
|
name: bundler
|
57
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -241,8 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
241
275
|
- !ruby/object:Gem::Version
|
242
276
|
version: '0'
|
243
277
|
requirements: []
|
244
|
-
|
245
|
-
rubygems_version: 2.7.6
|
278
|
+
rubygems_version: 3.1.6
|
246
279
|
signing_key:
|
247
280
|
specification_version: 4
|
248
281
|
summary: File storage handler following active record model lifecycle
|