saviour 0.4.5 → 0.4.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +0 -18
- data/README.md +6 -9
- data/lib/saviour.rb +8 -0
- data/lib/saviour/base_uploader.rb +17 -4
- data/lib/saviour/config.rb +1 -1
- data/lib/saviour/db_helpers.rb +69 -0
- data/lib/saviour/file.rb +76 -26
- data/lib/saviour/integrator.rb +8 -3
- data/lib/saviour/life_cycle.rb +60 -12
- data/lib/saviour/local_storage.rb +35 -12
- data/lib/saviour/model.rb +12 -3
- data/lib/saviour/s3_storage.rb +46 -17
- data/lib/saviour/source_filename_extractor.rb +6 -1
- data/lib/saviour/uploader/processors_runner.rb +29 -2
- data/lib/saviour/url_source.rb +7 -3
- data/lib/saviour/validator.rb +34 -4
- data/lib/saviour/version.rb +1 -1
- data/saviour.gemspec +4 -3
- data/spec/feature/{allow_overriding_attached_as_method.rb → allow_overriding_attached_as_method_spec.rb} +0 -0
- data/spec/feature/crud_workflows_spec.rb +26 -10
- data/spec/feature/dirty_spec.rb +29 -21
- data/spec/feature/memory_usage_spec.rb +84 -0
- data/spec/feature/{processors_api.rb → processors_api_spec.rb} +22 -5
- data/spec/feature/{rewind_source_before_read.rb → rewind_source_before_read_spec.rb} +0 -0
- data/spec/feature/transactional_behavior_spec.rb +147 -0
- data/spec/feature/{uploader_declaration.rb → uploader_declaration_spec.rb} +0 -0
- data/spec/feature/validations_spec.rb +19 -0
- data/spec/feature/with_copy_spec.rb +43 -0
- data/spec/models/base_uploader_spec.rb +12 -33
- data/spec/models/config_spec.rb +2 -2
- data/spec/models/file_spec.rb +19 -47
- data/spec/models/local_storage_spec.rb +13 -34
- data/spec/models/s3_storage_spec.rb +12 -37
- data/spec/models/url_source_spec.rb +4 -4
- data/spec/spec_helper.rb +4 -0
- metadata +29 -14
- data/gemfiles/4.0.gemfile +0 -9
- data/gemfiles/4.1.gemfile +0 -9
- data/gemfiles/4.2.gemfile +0 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b1d3de91b6a03bd4e483d2012346575d1fb444d
|
4
|
+
data.tar.gz: eca43877819318d52cabb2bcc6a02f71f131ee03
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab1eabe24fdc96f7785c9a72d6da2250c6ae9362aca1da91817c08846d70443b8919ca41c14cc34afc5652d4081dcac80a2450a3e30cbeee5ce9c2b82ac5fd29
|
7
|
+
data.tar.gz: 9db4871ec035aa11f7d9fcce3cd72ed7eb7e96e44567a5f21552e02df9a520a9057b455941ead2513cac13cdf14ce44d1d9c42012ec4af1e01007e855feef6ef
|
data/.travis.yml
CHANGED
@@ -2,15 +2,11 @@ language: ruby
|
|
2
2
|
sudo: false
|
3
3
|
cache: bundler
|
4
4
|
rvm:
|
5
|
-
- 2.1.10
|
6
5
|
- 2.2.8
|
7
6
|
- 2.3.5
|
8
7
|
- 2.4.2
|
9
8
|
|
10
9
|
gemfile:
|
11
|
-
- gemfiles/4.0.gemfile
|
12
|
-
- gemfiles/4.1.gemfile
|
13
|
-
- gemfiles/4.2.gemfile
|
14
10
|
- gemfiles/5.0.gemfile
|
15
11
|
- gemfiles/5.1.gemfile
|
16
12
|
|
@@ -20,17 +16,3 @@ addons:
|
|
20
16
|
|
21
17
|
after_success:
|
22
18
|
- bundle exec codeclimate-test-reporter
|
23
|
-
|
24
|
-
matrix:
|
25
|
-
exclude:
|
26
|
-
# rails 5+ requires 2.2+
|
27
|
-
- rvm: 2.1.10
|
28
|
-
gemfile: gemfiles/5.0.gemfile
|
29
|
-
- rvm: 2.1.10
|
30
|
-
gemfile: gemfiles/5.1.gemfile
|
31
|
-
|
32
|
-
# Arel support for ruby 2.4 starts in 4.2
|
33
|
-
- rvm: 2.4.2
|
34
|
-
gemfile: gemfiles/4.0.gemfile
|
35
|
-
- rvm: 2.4.2
|
36
|
-
gemfile: gemfiles/4.1.gemfile
|
data/README.md
CHANGED
@@ -223,10 +223,6 @@ You can also assign a Proc instead of a String to dynamically manage this (for m
|
|
223
223
|
|
224
224
|
This storage will take care of removing empty folders after removing files.
|
225
225
|
|
226
|
-
This storage includes a feature of overwrite protection, raising an exception if an attempt is made of writing something
|
227
|
-
on a path that already exists. This behaviour in enabled by default, but you can turn it off by passing an additional
|
228
|
-
argument when instantiating the storage: `overwrite_protection: false`.
|
229
|
-
|
230
226
|
|
231
227
|
### S3Storage
|
232
228
|
|
@@ -262,11 +258,6 @@ Saviour::Config.storage = Saviour::S3Storage.new(
|
|
262
258
|
)
|
263
259
|
```
|
264
260
|
|
265
|
-
This storage includes a feature of overwrite protection, raising an exception if an attempt is made of writing something
|
266
|
-
on a path that already exists. This behaviour in enabled by default, but you can turn it off by passing an additional
|
267
|
-
argument when instantiating the storage: `overwrite_protection: false`. This feature requires an additional HEAD request
|
268
|
-
to verify existence for every write.
|
269
|
-
|
270
261
|
NOTE: Be aware that S3 has a limit of 1024 bytes for the keys (paths) used. Be sure to truncate to that maximum length
|
271
262
|
if you're using an s3 storage, for example with a processor like this:
|
272
263
|
|
@@ -368,6 +359,12 @@ end
|
|
368
359
|
Use `store_dir` to indicate the default directory under which the file will be stored. You can also use it under a
|
369
360
|
`version` to change the default directory for that specific version.
|
370
361
|
|
362
|
+
Note that it's very important that the full path to any attached file to any model is unique. This is typically
|
363
|
+
accomplished by using `model.id` and `attached_as` as part of either the `store_dir` or the `filename`, in any
|
364
|
+
combination you may want. If this is not satisfied, you may experience unexpected overwrite of files or files
|
365
|
+
having unexpected contents, for example if two different models write to the same storage path, and then one
|
366
|
+
of them is deleted.
|
367
|
+
|
371
368
|
|
372
369
|
### Accessing model and attached_as
|
373
370
|
|
data/lib/saviour.rb
CHANGED
@@ -12,8 +12,16 @@ require 'saviour/source_filename_extractor'
|
|
12
12
|
require 'saviour/life_cycle'
|
13
13
|
require 'saviour/persistence_layer'
|
14
14
|
require 'saviour/validator'
|
15
|
+
require 'saviour/db_helpers'
|
15
16
|
|
16
17
|
require 'tempfile'
|
18
|
+
require 'fileutils'
|
17
19
|
|
18
20
|
module Saviour
|
21
|
+
NoActiveRecordDetected = Class.new(StandardError)
|
22
|
+
FileNotPresent = Class.new(StandardError)
|
23
|
+
ConfigurationError = Class.new(StandardError)
|
24
|
+
SourceError = Class.new(StandardError)
|
25
|
+
CannotCopy = Class.new(StandardError)
|
26
|
+
MissingSource = Class.new(StandardError)
|
19
27
|
end
|
@@ -19,8 +19,8 @@ module Saviour
|
|
19
19
|
@data.key?(name) || super
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
23
|
-
raise
|
22
|
+
def _process_as_contents(contents, filename)
|
23
|
+
raise ConfigurationError, "Please use `store_dir` in the uploader" unless store_dir
|
24
24
|
|
25
25
|
catch(:halt_process) do
|
26
26
|
if Config.processing_enabled
|
@@ -28,9 +28,22 @@ module Saviour
|
|
28
28
|
end
|
29
29
|
|
30
30
|
path = ::File.join(store_dir, filename)
|
31
|
-
Config.storage.write(contents, path)
|
32
31
|
|
33
|
-
path
|
32
|
+
[contents, path]
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def _process_as_file(file, filename)
|
37
|
+
raise ConfigurationError, "Please use `store_dir` in the uploader" unless store_dir
|
38
|
+
|
39
|
+
catch(:halt_process) do
|
40
|
+
if Config.processing_enabled
|
41
|
+
file, filename = Uploader::ProcessorsRunner.new(self).run_as_file!(file, filename)
|
42
|
+
end
|
43
|
+
|
44
|
+
path = ::File.join(store_dir, filename)
|
45
|
+
|
46
|
+
[file, path]
|
34
47
|
end
|
35
48
|
end
|
36
49
|
|
data/lib/saviour/config.rb
CHANGED
@@ -4,7 +4,7 @@ module Saviour
|
|
4
4
|
class Config
|
5
5
|
class NotImplemented
|
6
6
|
def method_missing(*)
|
7
|
-
raise(
|
7
|
+
raise(ConfigurationError, "You need to provide a storage! Set Saviour::Config.storage = xxx")
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Saviour
|
4
|
+
module DbHelpers
|
5
|
+
NotInTransaction = Class.new(StandardError)
|
6
|
+
|
7
|
+
class CommitDummy
|
8
|
+
def initialize(block)
|
9
|
+
@block = block
|
10
|
+
end
|
11
|
+
|
12
|
+
def rolledback!(*)
|
13
|
+
close_transaction
|
14
|
+
end
|
15
|
+
|
16
|
+
def close_transaction(*)
|
17
|
+
end
|
18
|
+
|
19
|
+
def before_committed!(*)
|
20
|
+
end
|
21
|
+
|
22
|
+
def committed!(*)
|
23
|
+
@block.call
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class RollbackDummy
|
28
|
+
def initialize(block)
|
29
|
+
@block = block
|
30
|
+
end
|
31
|
+
|
32
|
+
def rolledback!(*)
|
33
|
+
@block.call
|
34
|
+
close_transaction
|
35
|
+
end
|
36
|
+
|
37
|
+
def close_transaction(*)
|
38
|
+
end
|
39
|
+
|
40
|
+
def before_committed!(*)
|
41
|
+
end
|
42
|
+
|
43
|
+
def committed!(*)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
class << self
|
49
|
+
|
50
|
+
def run_after_commit(&block)
|
51
|
+
unless ActiveRecord::Base.connection.current_transaction.open?
|
52
|
+
raise NotInTransaction, 'Trying to use `run_after_commit` but no transaction is currently open.'
|
53
|
+
end
|
54
|
+
|
55
|
+
dummy = CommitDummy.new(block)
|
56
|
+
ActiveRecord::Base.connection.add_transaction_record(dummy)
|
57
|
+
end
|
58
|
+
|
59
|
+
def run_after_rollback(&block)
|
60
|
+
unless ActiveRecord::Base.connection.current_transaction.open?
|
61
|
+
raise NotInTransaction, 'Trying to use `run_after_commit` but no transaction is currently open.'
|
62
|
+
end
|
63
|
+
|
64
|
+
dummy = RollbackDummy.new(block)
|
65
|
+
ActiveRecord::Base.connection.add_transaction_record(dummy)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/saviour/file.rb
CHANGED
@@ -3,6 +3,7 @@ require 'securerandom'
|
|
3
3
|
module Saviour
|
4
4
|
class File
|
5
5
|
attr_reader :persisted_path
|
6
|
+
attr_reader :source
|
6
7
|
|
7
8
|
def initialize(uploader_klass, model, attached_as)
|
8
9
|
@uploader_klass, @model, @attached_as = uploader_klass, model, attached_as
|
@@ -11,6 +12,7 @@ module Saviour
|
|
11
12
|
|
12
13
|
def set_path!(path)
|
13
14
|
@persisted_path = path
|
15
|
+
@persisted_path_before_last_save = path
|
14
16
|
end
|
15
17
|
|
16
18
|
def exists?
|
@@ -18,11 +20,11 @@ module Saviour
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def read
|
21
|
-
persisted? &&
|
23
|
+
persisted? && Config.storage.read(@persisted_path)
|
22
24
|
end
|
23
25
|
|
24
26
|
def delete
|
25
|
-
persisted? &&
|
27
|
+
persisted? && Config.storage.delete(@persisted_path)
|
26
28
|
end
|
27
29
|
|
28
30
|
def public_url
|
@@ -36,20 +38,34 @@ module Saviour
|
|
36
38
|
if persisted?
|
37
39
|
another_file.persisted_path == persisted_path
|
38
40
|
else
|
39
|
-
another_file.
|
41
|
+
another_file.source == @source
|
40
42
|
end
|
41
43
|
end
|
42
44
|
|
43
45
|
def clone
|
46
|
+
return nil unless persisted?
|
47
|
+
|
44
48
|
new_file = Saviour::File.new(@uploader_klass, @model, @attached_as)
|
45
|
-
new_file.set_path!
|
49
|
+
new_file.set_path! @persisted_path
|
50
|
+
new_file
|
51
|
+
end
|
52
|
+
|
53
|
+
def dup
|
54
|
+
new_file = Saviour::File.new(@uploader_klass, @model, @attached_as)
|
55
|
+
|
56
|
+
if persisted?
|
57
|
+
new_file.assign(Saviour::StringSource.new(read, filename))
|
58
|
+
else
|
59
|
+
new_file.assign(Saviour::StringSource.new(source_data, filename_to_be_assigned))
|
60
|
+
end
|
61
|
+
|
46
62
|
new_file
|
47
63
|
end
|
48
64
|
|
49
65
|
alias_method :url, :public_url
|
50
66
|
|
51
67
|
def assign(object)
|
52
|
-
raise(
|
68
|
+
raise(SourceError, "given object to #assign or #<attach_as>= must respond to `read`") if object && !object.respond_to?(:read)
|
53
69
|
|
54
70
|
followers = @model.class.attached_followers_per_leader[@attached_as]
|
55
71
|
followers.each { |x| @model.send(x).assign(object) unless @model.send(x).changed? } if followers
|
@@ -79,20 +95,16 @@ module Saviour
|
|
79
95
|
end
|
80
96
|
|
81
97
|
def with_copy
|
82
|
-
raise "must be persisted" unless persisted?
|
83
|
-
|
84
|
-
Tempfile.
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
ensure
|
93
|
-
file.close
|
94
|
-
file.delete
|
95
|
-
end
|
98
|
+
raise CannotCopy, "must be persisted" unless persisted?
|
99
|
+
|
100
|
+
temp_file = Tempfile.new([::File.basename(filename, ".*"), ::File.extname(filename)])
|
101
|
+
|
102
|
+
begin
|
103
|
+
Config.storage.read_to_file(@persisted_path, temp_file)
|
104
|
+
|
105
|
+
yield(temp_file)
|
106
|
+
ensure
|
107
|
+
temp_file.close!
|
96
108
|
end
|
97
109
|
end
|
98
110
|
|
@@ -100,15 +112,53 @@ module Saviour
|
|
100
112
|
changed? ? (SourceFilenameExtractor.new(@source).detected_filename || SecureRandom.hex) : nil
|
101
113
|
end
|
102
114
|
|
103
|
-
def
|
104
|
-
|
115
|
+
def __maybe_with_tmpfile(source_type, file)
|
116
|
+
return yield if source_type == :stream
|
117
|
+
|
118
|
+
tmpfile = Tempfile.new([::File.basename(file.path, ".*"), ::File.extname(file.path)])
|
119
|
+
FileUtils.cp(file.path, tmpfile.path)
|
120
|
+
|
121
|
+
begin
|
122
|
+
yield(tmpfile)
|
123
|
+
ensure
|
124
|
+
tmpfile.close!
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def write(before_write: nil)
|
129
|
+
raise(MissingSource, "You must provide a source to read from before trying to write") unless @source
|
130
|
+
|
131
|
+
__maybe_with_tmpfile(source_type, @source) do |tmpfile|
|
132
|
+
contents, path = case source_type
|
133
|
+
when :stream
|
134
|
+
uploader._process_as_contents(source_data, filename_to_be_assigned)
|
135
|
+
when :file
|
136
|
+
uploader._process_as_file(tmpfile, filename_to_be_assigned)
|
137
|
+
end
|
138
|
+
@source_was = @source
|
105
139
|
|
106
|
-
|
107
|
-
|
140
|
+
if path
|
141
|
+
before_write.call(path) if before_write
|
108
142
|
|
109
|
-
|
110
|
-
|
111
|
-
|
143
|
+
case source_type
|
144
|
+
when :stream
|
145
|
+
Config.storage.write(contents, path)
|
146
|
+
when :file
|
147
|
+
Config.storage.write_from_file(contents, path)
|
148
|
+
end
|
149
|
+
|
150
|
+
@persisted_path = path
|
151
|
+
@persisted_path_before_last_save = path
|
152
|
+
path
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def source_type
|
158
|
+
if @source.respond_to?(:path)
|
159
|
+
:file
|
160
|
+
else
|
161
|
+
:stream
|
112
162
|
end
|
113
163
|
end
|
114
164
|
|
data/lib/saviour/integrator.rb
CHANGED
@@ -6,7 +6,7 @@ module Saviour
|
|
6
6
|
end
|
7
7
|
|
8
8
|
def setup!
|
9
|
-
raise "You cannot include Saviour twice in the same class" if @klass.respond_to?(:attached_files)
|
9
|
+
raise(ConfigurationError, "You cannot include Saviour::Model twice in the same class") if @klass.respond_to?(:attached_files)
|
10
10
|
|
11
11
|
@klass.class_attribute :attached_files
|
12
12
|
@klass.attached_files = []
|
@@ -26,7 +26,7 @@ module Saviour
|
|
26
26
|
end
|
27
27
|
|
28
28
|
if uploader_klass.nil? && block.nil?
|
29
|
-
raise
|
29
|
+
raise ConfigurationError, "you must provide either an UploaderClass or a block to define it."
|
30
30
|
end
|
31
31
|
|
32
32
|
mod = Module.new do
|
@@ -58,7 +58,12 @@ module Saviour
|
|
58
58
|
|
59
59
|
@klass.define_singleton_method("attach_validation") do |attach_as, method_name = nil, &block|
|
60
60
|
klass.__saviour_validations ||= Hash.new { [] }
|
61
|
-
klass.__saviour_validations[attach_as] += [method_name || block]
|
61
|
+
klass.__saviour_validations[attach_as] += [{ method_or_block: method_name || block, type: :memory }]
|
62
|
+
end
|
63
|
+
|
64
|
+
@klass.define_singleton_method("attach_validation_with_file") do |attach_as, method_name = nil, &block|
|
65
|
+
klass.__saviour_validations ||= Hash.new { [] }
|
66
|
+
klass.__saviour_validations[attach_as] += [{ method_or_block: method_name || block, type: :file }]
|
62
67
|
end
|
63
68
|
end
|
64
69
|
end
|
data/lib/saviour/life_cycle.rb
CHANGED
@@ -1,23 +1,42 @@
|
|
1
1
|
module Saviour
|
2
2
|
class LifeCycle
|
3
|
-
def initialize(model, persistence_klass
|
4
|
-
raise "Please provide an object compatible with Saviour." unless model.class.respond_to?(:attached_files)
|
3
|
+
def initialize(model, persistence_klass)
|
4
|
+
raise ConfigurationError, "Please provide an object compatible with Saviour." unless model.class.respond_to?(:attached_files)
|
5
5
|
|
6
6
|
@persistence_klass = persistence_klass
|
7
7
|
@model = model
|
8
8
|
end
|
9
9
|
|
10
10
|
def delete!
|
11
|
+
DbHelpers.run_after_commit do
|
12
|
+
attached_files.each do |column|
|
13
|
+
@model.send(column).delete
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def create!
|
11
19
|
attached_files.each do |column|
|
12
|
-
|
20
|
+
next unless @model.send(column).changed?
|
21
|
+
|
22
|
+
persistence_layer = @persistence_klass.new(@model)
|
23
|
+
new_path = @model.send(column).write
|
24
|
+
|
25
|
+
if new_path
|
26
|
+
persistence_layer.write(column, new_path)
|
27
|
+
|
28
|
+
DbHelpers.run_after_rollback do
|
29
|
+
Config.storage.delete(new_path)
|
30
|
+
end
|
31
|
+
end
|
13
32
|
end
|
14
33
|
end
|
15
34
|
|
16
|
-
def
|
35
|
+
def update!
|
17
36
|
attached_files.each do |column|
|
18
|
-
|
37
|
+
next unless @model.send(column).changed?
|
19
38
|
|
20
|
-
|
39
|
+
update_file(column)
|
21
40
|
end
|
22
41
|
end
|
23
42
|
|
@@ -25,14 +44,43 @@ module Saviour
|
|
25
44
|
private
|
26
45
|
|
27
46
|
|
28
|
-
def
|
29
|
-
persistence_layer = @persistence_klass.new(@model)
|
30
|
-
current_path = persistence_layer.read(column)
|
47
|
+
def update_file(column)
|
48
|
+
persistence_layer = @persistence_klass.new(@model)
|
49
|
+
current_path = persistence_layer.read(column)
|
50
|
+
dup_temp_path = SecureRandom.hex
|
51
|
+
|
52
|
+
dup_file = proc do
|
53
|
+
Config.storage.cp current_path, dup_temp_path
|
31
54
|
|
32
|
-
|
55
|
+
DbHelpers.run_after_commit do
|
56
|
+
Config.storage.delete dup_temp_path
|
57
|
+
end
|
33
58
|
|
34
|
-
|
35
|
-
|
59
|
+
DbHelpers.run_after_rollback do
|
60
|
+
Config.storage.mv dup_temp_path, current_path
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
new_path = @model.send(column).write(
|
65
|
+
before_write: ->(path) { dup_file.call if current_path == path }
|
66
|
+
)
|
67
|
+
|
68
|
+
if new_path
|
69
|
+
persistence_layer.write(column, new_path)
|
70
|
+
|
71
|
+
if current_path && current_path != new_path
|
72
|
+
DbHelpers.run_after_commit do
|
73
|
+
Config.storage.delete(current_path)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# Delete the newly uploaded file only if it's an update in a different path
|
78
|
+
if current_path.nil? || current_path != new_path
|
79
|
+
DbHelpers.run_after_rollback do
|
80
|
+
Config.storage.delete(new_path)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
36
84
|
end
|
37
85
|
|
38
86
|
def attached_files
|