saviour 0.4.5 → 0.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -18
- data/README.md +6 -9
- data/lib/saviour.rb +8 -0
- data/lib/saviour/base_uploader.rb +17 -4
- data/lib/saviour/config.rb +1 -1
- data/lib/saviour/db_helpers.rb +69 -0
- data/lib/saviour/file.rb +76 -26
- data/lib/saviour/integrator.rb +8 -3
- data/lib/saviour/life_cycle.rb +60 -12
- data/lib/saviour/local_storage.rb +35 -12
- data/lib/saviour/model.rb +12 -3
- data/lib/saviour/s3_storage.rb +46 -17
- data/lib/saviour/source_filename_extractor.rb +6 -1
- data/lib/saviour/uploader/processors_runner.rb +29 -2
- data/lib/saviour/url_source.rb +7 -3
- data/lib/saviour/validator.rb +34 -4
- data/lib/saviour/version.rb +1 -1
- data/saviour.gemspec +4 -3
- data/spec/feature/{allow_overriding_attached_as_method.rb → allow_overriding_attached_as_method_spec.rb} +0 -0
- data/spec/feature/crud_workflows_spec.rb +26 -10
- data/spec/feature/dirty_spec.rb +29 -21
- data/spec/feature/memory_usage_spec.rb +84 -0
- data/spec/feature/{processors_api.rb → processors_api_spec.rb} +22 -5
- data/spec/feature/{rewind_source_before_read.rb → rewind_source_before_read_spec.rb} +0 -0
- data/spec/feature/transactional_behavior_spec.rb +147 -0
- data/spec/feature/{uploader_declaration.rb → uploader_declaration_spec.rb} +0 -0
- data/spec/feature/validations_spec.rb +19 -0
- data/spec/feature/with_copy_spec.rb +43 -0
- data/spec/models/base_uploader_spec.rb +12 -33
- data/spec/models/config_spec.rb +2 -2
- data/spec/models/file_spec.rb +19 -47
- data/spec/models/local_storage_spec.rb +13 -34
- data/spec/models/s3_storage_spec.rb +12 -37
- data/spec/models/url_source_spec.rb +4 -4
- data/spec/spec_helper.rb +4 -0
- metadata +29 -14
- data/gemfiles/4.0.gemfile +0 -9
- data/gemfiles/4.1.gemfile +0 -9
- data/gemfiles/4.2.gemfile +0 -9
@@ -2,17 +2,15 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module Saviour
|
4
4
|
class LocalStorage
|
5
|
+
MissingPublicUrlPrefix = Class.new(StandardError)
|
6
|
+
|
5
7
|
def initialize(opts = {})
|
6
8
|
@local_prefix = opts[:local_prefix]
|
7
9
|
@public_url_prefix = opts[:public_url_prefix]
|
8
|
-
@overwrite_protection = opts.fetch(:overwrite_protection, true)
|
9
10
|
end
|
10
11
|
|
11
12
|
def write(contents, path)
|
12
|
-
|
13
|
-
|
14
|
-
dir = ::File.dirname(real_path(path))
|
15
|
-
FileUtils.mkdir_p(dir) unless ::File.directory?(dir)
|
13
|
+
ensure_dir!(path)
|
16
14
|
|
17
15
|
::File.open(real_path(path), "w") do |f|
|
18
16
|
f.binmode
|
@@ -20,15 +18,27 @@ module Saviour
|
|
20
18
|
end
|
21
19
|
end
|
22
20
|
|
21
|
+
def write_from_file(file, path)
|
22
|
+
ensure_dir!(path)
|
23
|
+
|
24
|
+
FileUtils.cp file.path, real_path(path)
|
25
|
+
end
|
26
|
+
|
27
|
+
def read_to_file(path, dest_file)
|
28
|
+
FileUtils.cp real_path(path), dest_file.path
|
29
|
+
end
|
30
|
+
|
23
31
|
def read(path)
|
24
|
-
assert_exists(path)
|
25
32
|
::File.open(real_path(path)).read
|
33
|
+
rescue Errno::ENOENT
|
34
|
+
raise FileNotPresent, "Trying to read an unexisting path: #{path}"
|
26
35
|
end
|
27
36
|
|
28
37
|
def delete(path)
|
29
|
-
assert_exists(path)
|
30
38
|
::File.delete(real_path(path))
|
31
39
|
ensure_removed_empty_dir(path)
|
40
|
+
rescue Errno::ENOENT
|
41
|
+
raise FileNotPresent, "Trying to delete an unexisting path: #{path}"
|
32
42
|
end
|
33
43
|
|
34
44
|
def exists?(path)
|
@@ -36,13 +46,30 @@ module Saviour
|
|
36
46
|
end
|
37
47
|
|
38
48
|
def public_url(path)
|
39
|
-
raise(
|
49
|
+
raise(MissingPublicUrlPrefix, "You must provide a `public_url_prefix`") unless public_url_prefix
|
40
50
|
::File.join(public_url_prefix, path)
|
41
51
|
end
|
42
52
|
|
53
|
+
def cp(source_path, destination_path)
|
54
|
+
FileUtils.cp(real_path(source_path), real_path(destination_path))
|
55
|
+
rescue Errno::ENOENT
|
56
|
+
raise FileNotPresent, "Trying to cp an unexisting path: #{source_path}"
|
57
|
+
end
|
58
|
+
|
59
|
+
def mv(source_path, destination_path)
|
60
|
+
FileUtils.mv(real_path(source_path), real_path(destination_path))
|
61
|
+
rescue Errno::ENOENT
|
62
|
+
raise FileNotPresent, "Trying to mv an unexisting path: #{source_path}"
|
63
|
+
end
|
64
|
+
|
43
65
|
|
44
66
|
private
|
45
67
|
|
68
|
+
def ensure_dir!(path)
|
69
|
+
dir = ::File.dirname(real_path(path))
|
70
|
+
FileUtils.mkdir_p(dir) unless ::File.directory?(dir)
|
71
|
+
end
|
72
|
+
|
46
73
|
def public_url_prefix
|
47
74
|
if @public_url_prefix.respond_to?(:call)
|
48
75
|
@public_url_prefix.call
|
@@ -55,10 +82,6 @@ module Saviour
|
|
55
82
|
@local_prefix ? ::File.join(@local_prefix, path) : path
|
56
83
|
end
|
57
84
|
|
58
|
-
def assert_exists(path)
|
59
|
-
raise(RuntimeError, "File does not exists: #{path}") unless ::File.file?(real_path(path))
|
60
|
-
end
|
61
|
-
|
62
85
|
def ensure_removed_empty_dir(path)
|
63
86
|
basedir = ::File.dirname(path)
|
64
87
|
return if basedir == "."
|
data/lib/saviour/model.rb
CHANGED
@@ -1,6 +1,4 @@
|
|
1
1
|
module Saviour
|
2
|
-
NoActiveRecordDetected = Class.new(StandardError)
|
3
|
-
|
4
2
|
module Model
|
5
3
|
def self.included(klass)
|
6
4
|
Integrator.new(klass, PersistenceLayer).setup!
|
@@ -9,7 +7,8 @@ module Saviour
|
|
9
7
|
raise(NoActiveRecordDetected, "Error: ActiveRecord not detected in #{self}") unless self.ancestors.include?(ActiveRecord::Base)
|
10
8
|
|
11
9
|
after_destroy { Saviour::LifeCycle.new(self, PersistenceLayer).delete! }
|
12
|
-
|
10
|
+
after_update { Saviour::LifeCycle.new(self, PersistenceLayer).update! }
|
11
|
+
after_create { Saviour::LifeCycle.new(self, PersistenceLayer).create! }
|
13
12
|
validate { Saviour::Validator.new(self).validate! }
|
14
13
|
end
|
15
14
|
end
|
@@ -20,5 +19,15 @@ module Saviour
|
|
20
19
|
end
|
21
20
|
super
|
22
21
|
end
|
22
|
+
|
23
|
+
def dup
|
24
|
+
duped = super
|
25
|
+
|
26
|
+
self.class.attached_files.each do |attach_as|
|
27
|
+
duped.instance_variable_set("@__uploader_#{attach_as}", send(attach_as).dup)
|
28
|
+
end
|
29
|
+
|
30
|
+
duped
|
31
|
+
end
|
23
32
|
end
|
24
33
|
end
|
data/lib/saviour/s3_storage.rb
CHANGED
@@ -5,24 +5,24 @@ end
|
|
5
5
|
|
6
6
|
module Saviour
|
7
7
|
class S3Storage
|
8
|
+
MissingPublicUrlPrefix = Class.new(StandardError)
|
9
|
+
KeyTooLarge = Class.new(StandardError)
|
10
|
+
|
8
11
|
def initialize(conf = {})
|
9
12
|
@bucket = conf.delete(:bucket)
|
10
13
|
@public_url_prefix = conf.delete(:public_url_prefix)
|
11
14
|
@conf = conf
|
12
|
-
@overwrite_protection = conf.delete(:overwrite_protection) { true }
|
13
15
|
@create_options = conf.delete(:create_options) { {} }
|
14
|
-
conf.fetch(:aws_access_key_id) { raise
|
15
|
-
conf.fetch(:aws_secret_access_key) { raise
|
16
|
+
conf.fetch(:aws_access_key_id) { raise(ArgumentError, "aws_access_key_id is required") }
|
17
|
+
conf.fetch(:aws_secret_access_key) { raise(ArgumentError, "aws_secret_access_key is required") }
|
16
18
|
end
|
17
19
|
|
18
20
|
def write(contents, path)
|
19
|
-
raise(RuntimeError, "The path you're trying to write already exists!") if @overwrite_protection && exists?(path)
|
20
|
-
|
21
21
|
path = sanitize_leading_slash(path)
|
22
22
|
|
23
23
|
# http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
|
24
24
|
if path.bytesize > 1024
|
25
|
-
raise(
|
25
|
+
raise(KeyTooLarge, "The key in S3 must be at max 1024 bytes, this key is too big: #{path}")
|
26
26
|
end
|
27
27
|
|
28
28
|
directory.files.create({
|
@@ -33,30 +33,63 @@ module Saviour
|
|
33
33
|
)
|
34
34
|
end
|
35
35
|
|
36
|
+
def write_from_file(file, path)
|
37
|
+
write(file, path)
|
38
|
+
end
|
39
|
+
|
40
|
+
def read_to_file(path, dest_file)
|
41
|
+
dest_file.rewind
|
42
|
+
dest_file.write(read(path))
|
43
|
+
end
|
44
|
+
|
36
45
|
def read(path)
|
37
|
-
|
38
|
-
|
39
|
-
directory.files.get(
|
46
|
+
real_path = sanitize_leading_slash(path)
|
47
|
+
|
48
|
+
file = directory.files.get(real_path)
|
49
|
+
raise FileNotPresent, "Trying to read an unexisting path: #{path}" unless file
|
50
|
+
|
51
|
+
file.body
|
40
52
|
end
|
41
53
|
|
42
54
|
def delete(path)
|
43
|
-
|
44
|
-
|
45
|
-
directory.files.get(
|
55
|
+
real_path = sanitize_leading_slash(path)
|
56
|
+
|
57
|
+
file = directory.files.get(real_path)
|
58
|
+
raise FileNotPresent, "Trying to delete an unexisting path: #{path}" unless file
|
59
|
+
|
60
|
+
file.destroy
|
46
61
|
end
|
47
62
|
|
48
63
|
def exists?(path)
|
49
64
|
path = sanitize_leading_slash(path)
|
65
|
+
|
50
66
|
!!directory.files.head(path)
|
51
67
|
end
|
52
68
|
|
53
69
|
def public_url(path)
|
54
|
-
raise(
|
70
|
+
raise(MissingPublicUrlPrefix, "You must provide a `public_url_prefix`") unless public_url_prefix
|
55
71
|
|
56
72
|
path = sanitize_leading_slash(path)
|
57
73
|
::File.join(public_url_prefix, path)
|
58
74
|
end
|
59
75
|
|
76
|
+
def cp(source_path, destination_path)
|
77
|
+
source_path = sanitize_leading_slash(source_path)
|
78
|
+
destination_path = sanitize_leading_slash(destination_path)
|
79
|
+
|
80
|
+
connection.copy_object(
|
81
|
+
@bucket, source_path,
|
82
|
+
@bucket, destination_path,
|
83
|
+
@create_options
|
84
|
+
)
|
85
|
+
|
86
|
+
FileUtils.cp(real_path(source_path), real_path(destination_path))
|
87
|
+
end
|
88
|
+
|
89
|
+
def mv(source_path, destination_path)
|
90
|
+
cp(source_path, destination_path)
|
91
|
+
delete(source_path)
|
92
|
+
end
|
60
93
|
|
61
94
|
private
|
62
95
|
|
@@ -72,10 +105,6 @@ module Saviour
|
|
72
105
|
path.gsub(/\A\/*/, '')
|
73
106
|
end
|
74
107
|
|
75
|
-
def assert_exists(path)
|
76
|
-
raise RuntimeError, "File does not exists: #{path}" unless exists?(path)
|
77
|
-
end
|
78
|
-
|
79
108
|
def directory
|
80
109
|
@directory ||= connection.directories.new(key: @bucket)
|
81
110
|
end
|
@@ -5,7 +5,12 @@ module Saviour
|
|
5
5
|
end
|
6
6
|
|
7
7
|
def detected_filename
|
8
|
-
original_filename || path_filename
|
8
|
+
filename || original_filename || path_filename
|
9
|
+
end
|
10
|
+
|
11
|
+
def filename
|
12
|
+
value = @source.filename if @source.respond_to?(:filename)
|
13
|
+
value if !value.nil? && value != ''
|
9
14
|
end
|
10
15
|
|
11
16
|
def original_filename
|
@@ -35,6 +35,8 @@ module Saviour
|
|
35
35
|
self.contents = ::File.read(file)
|
36
36
|
else
|
37
37
|
file.rewind
|
38
|
+
file.truncate(0)
|
39
|
+
file.binmode
|
38
40
|
file.write(contents)
|
39
41
|
file.flush
|
40
42
|
file.rewind
|
@@ -75,14 +77,39 @@ module Saviour
|
|
75
77
|
end
|
76
78
|
|
77
79
|
if previous_type == :file
|
78
|
-
file.rewind
|
79
80
|
self.contents = ::File.read(file)
|
80
81
|
end
|
81
82
|
|
82
|
-
file.
|
83
|
+
file.close!
|
83
84
|
|
84
85
|
[contents, filename]
|
85
86
|
end
|
87
|
+
|
88
|
+
def run_as_file!(start_file, initial_filename)
|
89
|
+
@file = start_file
|
90
|
+
@contents = nil
|
91
|
+
@filename = initial_filename
|
92
|
+
|
93
|
+
previous_type = :file
|
94
|
+
|
95
|
+
matching_processors.each do |processor|
|
96
|
+
advance!(processor, previous_type)
|
97
|
+
|
98
|
+
run_processor(processor)
|
99
|
+
previous_type = processor[:type]
|
100
|
+
end
|
101
|
+
|
102
|
+
if previous_type == :memory
|
103
|
+
file.rewind
|
104
|
+
file.truncate(0)
|
105
|
+
file.binmode
|
106
|
+
file.write(contents)
|
107
|
+
file.flush
|
108
|
+
file.rewind
|
109
|
+
end
|
110
|
+
|
111
|
+
[file, filename]
|
112
|
+
end
|
86
113
|
end
|
87
114
|
end
|
88
115
|
end
|
data/lib/saviour/url_source.rb
CHANGED
@@ -3,6 +3,10 @@ require 'net/http'
|
|
3
3
|
|
4
4
|
module Saviour
|
5
5
|
class UrlSource
|
6
|
+
TooManyRedirects = Class.new(StandardError)
|
7
|
+
InvalidUrl = Class.new(StandardError)
|
8
|
+
ConnectionFailed = Class.new(StandardError)
|
9
|
+
|
6
10
|
MAX_REDIRECTS = 10
|
7
11
|
|
8
12
|
def initialize(url)
|
@@ -33,7 +37,7 @@ module Saviour
|
|
33
37
|
end
|
34
38
|
|
35
39
|
def resolve(uri, max_redirects = MAX_REDIRECTS)
|
36
|
-
raise
|
40
|
+
raise TooManyRedirects, "Max number of allowed redirects reached (#{MAX_REDIRECTS}) when resolving #{uri}" if max_redirects == 0
|
37
41
|
|
38
42
|
response = Net::HTTP.get_response(uri)
|
39
43
|
|
@@ -51,12 +55,12 @@ module Saviour
|
|
51
55
|
begin
|
52
56
|
URI(url)
|
53
57
|
rescue URI::InvalidURIError
|
54
|
-
raise
|
58
|
+
raise InvalidUrl, "'#{url}' is not a valid URI"
|
55
59
|
end
|
56
60
|
end
|
57
61
|
|
58
62
|
def with_retry(n = 3, &block)
|
59
|
-
raise(
|
63
|
+
raise(ConnectionFailed, "Connection to #{@uri} failed after 3 attempts.") if n == 0
|
60
64
|
|
61
65
|
block.call || with_retry(n - 1, &block)
|
62
66
|
end
|
data/lib/saviour/validator.rb
CHANGED
@@ -1,16 +1,26 @@
|
|
1
1
|
module Saviour
|
2
2
|
class Validator
|
3
3
|
def initialize(model)
|
4
|
-
raise "Please provide an object compatible with Saviour." unless model.class.respond_to?(:attached_files)
|
4
|
+
raise(ConfigurationError, "Please provide an object compatible with Saviour.") unless model.class.respond_to?(:attached_files)
|
5
5
|
|
6
6
|
@model = model
|
7
7
|
end
|
8
8
|
|
9
9
|
def validate!
|
10
|
-
validations.each do |column,
|
11
|
-
raise(
|
10
|
+
validations.each do |column, declared_validations|
|
11
|
+
raise(ConfigurationError, "There is no attachment defined as '#{column}'") unless attached_files.include?(column)
|
12
12
|
if @model.send(column).changed?
|
13
|
-
|
13
|
+
declared_validations.each do |data|
|
14
|
+
type = data[:type]
|
15
|
+
method_or_block = data[:method_or_block]
|
16
|
+
|
17
|
+
case type
|
18
|
+
when :memory
|
19
|
+
run_validation(column, method_or_block)
|
20
|
+
when :file
|
21
|
+
run_validation_as_file(column, method_or_block)
|
22
|
+
end
|
23
|
+
end
|
14
24
|
end
|
15
25
|
end
|
16
26
|
end
|
@@ -37,6 +47,26 @@ module Saviour
|
|
37
47
|
end
|
38
48
|
end
|
39
49
|
|
50
|
+
def run_validation_as_file(column, method_or_block)
|
51
|
+
file = @model.send(column).source
|
52
|
+
filename = @model.send(column).filename_to_be_assigned
|
53
|
+
opts = { attached_as: column }
|
54
|
+
|
55
|
+
if method_or_block.respond_to?(:call)
|
56
|
+
if method_or_block.arity == 2
|
57
|
+
@model.instance_exec(file, filename, &method_or_block)
|
58
|
+
else
|
59
|
+
@model.instance_exec(file, filename, opts, &method_or_block)
|
60
|
+
end
|
61
|
+
else
|
62
|
+
if @model.method(method_or_block).arity == 2
|
63
|
+
@model.send(method_or_block, file, filename)
|
64
|
+
else
|
65
|
+
@model.send(method_or_block, file, filename, opts)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
40
70
|
def attached_files
|
41
71
|
@model.class.attached_files
|
42
72
|
end
|
data/lib/saviour/version.rb
CHANGED
data/saviour.gemspec
CHANGED
@@ -13,10 +13,10 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.files = `git ls-files`.split($/)
|
14
14
|
spec.require_paths = ["lib"]
|
15
15
|
|
16
|
-
spec.required_ruby_version = ">= 2.
|
16
|
+
spec.required_ruby_version = ">= 2.2.0"
|
17
17
|
|
18
|
-
spec.add_dependency "activerecord", ">=
|
19
|
-
spec.add_dependency "activesupport", ">=
|
18
|
+
spec.add_dependency "activerecord", ">= 5.0"
|
19
|
+
spec.add_dependency "activesupport", ">= 5.0"
|
20
20
|
|
21
21
|
spec.add_development_dependency "bundler"
|
22
22
|
spec.add_development_dependency "rspec"
|
@@ -25,4 +25,5 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_development_dependency "appraisal"
|
26
26
|
spec.add_development_dependency "fog-aws"
|
27
27
|
spec.add_development_dependency "mime-types"
|
28
|
+
spec.add_development_dependency "get_process_mem"
|
28
29
|
end
|
File without changes
|
@@ -139,22 +139,38 @@ describe "saving a new file" do
|
|
139
139
|
end
|
140
140
|
end
|
141
141
|
end
|
142
|
+
|
143
|
+
it "does allow to update the same file to another contents in the same path" do
|
144
|
+
a = klass.create! file: Saviour::StringSource.new("contents", "file.txt")
|
145
|
+
|
146
|
+
a.update_attributes! file: Saviour::StringSource.new("foo", "file.txt")
|
147
|
+
expect(Saviour::Config.storage.read(a[:file])).to eq "foo"
|
148
|
+
end
|
142
149
|
end
|
143
150
|
|
144
151
|
describe "dupping" do
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
152
|
+
let(:uploader) {
|
153
|
+
Class.new(Saviour::BaseUploader) {
|
154
|
+
store_dir { "/store/dir/#{model.id}" }
|
155
|
+
}
|
156
|
+
}
|
149
157
|
|
150
|
-
|
158
|
+
it "creates a non persisted file attachment" do
|
159
|
+
a = klass.create! file: Saviour::StringSource.new("contents", "file.txt")
|
160
|
+
expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
|
151
161
|
|
152
|
-
|
153
|
-
|
162
|
+
b = a.dup
|
163
|
+
expect(b).to_not be_persisted
|
164
|
+
expect(b.file).to_not be_persisted
|
165
|
+
end
|
154
166
|
|
155
|
-
|
156
|
-
|
157
|
-
|
167
|
+
it "can be saved" do
|
168
|
+
a = klass.create! file: Saviour::StringSource.new("contents", "file.txt")
|
169
|
+
b = a.dup
|
170
|
+
b.save!
|
171
|
+
|
172
|
+
expect(Saviour::Config.storage.exists?(b[:file])).to be_truthy
|
173
|
+
expect(Saviour::Config.storage.read(a[:file])).to eq Saviour::Config.storage.read(b[:file])
|
158
174
|
end
|
159
175
|
end
|
160
176
|
end
|