saviour 0.4.5 → 0.4.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +0 -18
  3. data/README.md +6 -9
  4. data/lib/saviour.rb +8 -0
  5. data/lib/saviour/base_uploader.rb +17 -4
  6. data/lib/saviour/config.rb +1 -1
  7. data/lib/saviour/db_helpers.rb +69 -0
  8. data/lib/saviour/file.rb +76 -26
  9. data/lib/saviour/integrator.rb +8 -3
  10. data/lib/saviour/life_cycle.rb +60 -12
  11. data/lib/saviour/local_storage.rb +35 -12
  12. data/lib/saviour/model.rb +12 -3
  13. data/lib/saviour/s3_storage.rb +46 -17
  14. data/lib/saviour/source_filename_extractor.rb +6 -1
  15. data/lib/saviour/uploader/processors_runner.rb +29 -2
  16. data/lib/saviour/url_source.rb +7 -3
  17. data/lib/saviour/validator.rb +34 -4
  18. data/lib/saviour/version.rb +1 -1
  19. data/saviour.gemspec +4 -3
  20. data/spec/feature/{allow_overriding_attached_as_method.rb → allow_overriding_attached_as_method_spec.rb} +0 -0
  21. data/spec/feature/crud_workflows_spec.rb +26 -10
  22. data/spec/feature/dirty_spec.rb +29 -21
  23. data/spec/feature/memory_usage_spec.rb +84 -0
  24. data/spec/feature/{processors_api.rb → processors_api_spec.rb} +22 -5
  25. data/spec/feature/{rewind_source_before_read.rb → rewind_source_before_read_spec.rb} +0 -0
  26. data/spec/feature/transactional_behavior_spec.rb +147 -0
  27. data/spec/feature/{uploader_declaration.rb → uploader_declaration_spec.rb} +0 -0
  28. data/spec/feature/validations_spec.rb +19 -0
  29. data/spec/feature/with_copy_spec.rb +43 -0
  30. data/spec/models/base_uploader_spec.rb +12 -33
  31. data/spec/models/config_spec.rb +2 -2
  32. data/spec/models/file_spec.rb +19 -47
  33. data/spec/models/local_storage_spec.rb +13 -34
  34. data/spec/models/s3_storage_spec.rb +12 -37
  35. data/spec/models/url_source_spec.rb +4 -4
  36. data/spec/spec_helper.rb +4 -0
  37. metadata +29 -14
  38. data/gemfiles/4.0.gemfile +0 -9
  39. data/gemfiles/4.1.gemfile +0 -9
  40. data/gemfiles/4.2.gemfile +0 -9
@@ -3,34 +3,42 @@ require 'spec_helper'
3
3
  describe "dirty model" do
4
4
  before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
5
5
 
6
- context "provides changes and previous file" do
7
- it do
8
- uploader = Class.new(Saviour::BaseUploader) { store_dir { "/store/dir" } }
9
- klass = Class.new(Test) { include Saviour::Model }
10
- klass.attach_file :file, uploader
11
- a = klass.create!
6
+ it "provides changes and previous file" do
7
+ uploader = Class.new(Saviour::BaseUploader) { store_dir { "/store/dir" } }
8
+ klass = Class.new(Test) { include Saviour::Model }
9
+ klass.attach_file :file, uploader
10
+ a = klass.create!
12
11
 
13
- with_test_file("example.xml") do |xml_file|
14
- with_test_file("camaloon.jpg") do |jpg_file|
15
- a.update_attributes! file: xml_file
12
+ with_test_file("example.xml") do |xml_file|
13
+ with_test_file("camaloon.jpg") do |jpg_file|
14
+ a.update_attributes! file: xml_file
16
15
 
17
- expect(a.changed_attributes).to eq({})
16
+ expect(a.changed_attributes).to eq({})
18
17
 
19
- expect(a.file.exists?).to be_truthy
20
- expect(a.file_changed?).to be_falsey
21
- expect(a.file.persisted?).to be_truthy
18
+ expect(a.file.exists?).to be_truthy
19
+ expect(a.file_changed?).to be_falsey
20
+ expect(a.file.persisted?).to be_truthy
22
21
 
23
- a.file = jpg_file
24
- expect(a.file.persisted?).to be_falsey
22
+ a.file = jpg_file
23
+ expect(a.file.persisted?).to be_falsey
25
24
 
26
- expect(a.file_changed?).to be_truthy
27
- expect(a.file_was).to_not eq(a.file)
28
- expect(a.file_was.url).to match /\.xml$/
29
- expect(a.file_was.persisted?).to be_truthy
25
+ expect(a.file_changed?).to be_truthy
26
+ expect(a.file_was).to_not eq(a.file)
27
+ expect(a.file_was.url).to match /\.xml$/
28
+ expect(a.file_was.persisted?).to be_truthy
30
29
 
31
- expect(a.changed_attributes).to include("file" => a.file_was)
32
- end
30
+ expect(a.changed_attributes).to include("file" => a.file_was)
33
31
  end
34
32
  end
35
33
  end
34
+
35
+ it "changes are nil when not persisted" do
36
+ uploader = Class.new(Saviour::BaseUploader) { store_dir { "/store/dir" } }
37
+ klass = Class.new(Test) { include Saviour::Model }
38
+ klass.attach_file :file, uploader
39
+
40
+ a = klass.new file: Saviour::StringSource.new("contents", "file.txt")
41
+ expect(a.file_changed?).to be_truthy
42
+ expect(a.file_was).to be_nil
43
+ end
36
44
  end
@@ -0,0 +1,84 @@
1
+ require 'spec_helper'
2
+
3
+ describe "memory usage" do
4
+ before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
5
+
6
+ let(:base_klass) {
7
+ a = Class.new(Test) { include Saviour::Model }
8
+ a.attach_file :file, uploader
9
+ a
10
+ }
11
+
12
+ CHUNK = ("A" * 1024).freeze
13
+
14
+ let(:size_to_test) { 10 } # Test with 10Mb files
15
+
16
+ def with_tempfile
17
+ f = Tempfile.new "test"
18
+
19
+ size_to_test.times do
20
+ 1024.times do
21
+ f.write CHUNK
22
+ end
23
+ end
24
+ f.flush
25
+
26
+ begin
27
+ yield f
28
+ ensure
29
+ f.close!
30
+ end
31
+ end
32
+
33
+ describe "is kept low when using exclusively with_file processors" do
34
+ let(:uploader) {
35
+ Class.new(Saviour::BaseUploader) {
36
+ store_dir { "/store/dir" }
37
+
38
+ process_with_file do |file, filename|
39
+ digest = Digest::MD5.file(file.path).hexdigest
40
+ [file, "#{digest}-#{filename}"]
41
+ end
42
+ }
43
+ }
44
+
45
+ it do
46
+ a = base_klass.create!
47
+
48
+ with_tempfile do |f|
49
+ base_line = GetProcessMem.new.mb
50
+
51
+ a.update_attributes! file: f
52
+
53
+ # Expect memory usage to grow below 10% of the file size
54
+ expect(GetProcessMem.new.mb - base_line).to be < size_to_test / 10
55
+ end
56
+ end
57
+ end
58
+
59
+ describe "increases to file size when using in memory processors" do
60
+ let(:uploader) {
61
+ Class.new(Saviour::BaseUploader) {
62
+ store_dir { "/store/dir" }
63
+
64
+ process do |contents, filename|
65
+ digest = Digest::MD5.hexdigest(contents)
66
+ [contents, "#{digest}-#{filename}"]
67
+ end
68
+ }
69
+ }
70
+
71
+ it do
72
+ a = base_klass.create!
73
+
74
+ with_tempfile do |f|
75
+ base_line = GetProcessMem.new.mb
76
+
77
+ a.update_attributes! file: f
78
+
79
+ # Expect memory usage to grow at least the size of the file
80
+ expect(GetProcessMem.new.mb - base_line).to be > size_to_test
81
+ end
82
+ end
83
+ end
84
+ end
@@ -5,8 +5,8 @@ describe "processor's API" do
5
5
 
6
6
  let(:uploader) {
7
7
  Class.new(Saviour::BaseUploader) do
8
- store_dir { "/store/dir/#{model.id}" }
9
- process { |contents, name| [contents, "#{model.id}-#{attached_as}-#{name}"] }
8
+ store_dir { "/store/dir/#{model.value}" }
9
+ process { |contents, name| [contents, "#{model.value}-#{attached_as}-#{name}"] }
10
10
  end
11
11
  }
12
12
 
@@ -14,7 +14,7 @@ describe "processor's API" do
14
14
  klass = Class.new(Test) {
15
15
  include Saviour::Model
16
16
 
17
- def id
17
+ def value
18
18
  87
19
19
  end
20
20
  }
@@ -38,8 +38,8 @@ describe "processor's API" do
38
38
  describe "can access store_dir" do
39
39
  let(:uploader) {
40
40
  Class.new(Saviour::BaseUploader) do
41
- store_dir { "/store/dir/#{model.id}" }
42
- process { |contents, name| ["FAKE: #{store_dir}", name] }
41
+ store_dir { "/store/dir/#{model.value}" }
42
+ process { |_, name| ["FAKE: #{store_dir}", name] }
43
43
  end
44
44
  }
45
45
 
@@ -55,4 +55,21 @@ describe "processor's API" do
55
55
  end
56
56
  end
57
57
  end
58
+
59
+ describe "can access id on the model (after_create)" do
60
+ let(:uploader) {
61
+ Class.new(Saviour::BaseUploader) do
62
+ store_dir { "/store/dir/#{model.id}" }
63
+ end
64
+ }
65
+
66
+ it do
67
+ with_test_file("example.xml") do |example, name|
68
+ a = klass.create! file: example
69
+
70
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
71
+ expect(a[:file]).to eq "/store/dir/#{a.id}/#{name}"
72
+ end
73
+ end
74
+ end
58
75
  end
@@ -0,0 +1,147 @@
1
+ require 'spec_helper'
2
+
3
+ describe "transactional behavior" do
4
+ before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
5
+
6
+ let(:uploader) {
7
+ Class.new(Saviour::BaseUploader) {
8
+ store_dir { "/store/dir" }
9
+ }
10
+ }
11
+
12
+ let(:klass) {
13
+ a = Class.new(Test) { include Saviour::Model }
14
+ a.attach_file :file, uploader
15
+ a
16
+ }
17
+
18
+ describe "deletion" do
19
+ it "deletes on after commit" do
20
+ with_test_file("example.xml") do |example|
21
+ a = klass.create! file: example
22
+ expect(a.file.exists?).to be_truthy
23
+
24
+ ActiveRecord::Base.transaction do
25
+ expect(a.destroy).to be_truthy
26
+ expect(klass.count).to eq 0
27
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
28
+ end
29
+
30
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_falsey
31
+ end
32
+ end
33
+
34
+ it "doesn't delete on rollback" do
35
+ with_test_file("example.xml") do |example|
36
+ a = klass.create! file: example
37
+ expect(a.file.exists?).to be_truthy
38
+
39
+ ActiveRecord::Base.transaction do
40
+ expect(a.destroy).to be_truthy
41
+ expect(klass.count).to eq 0
42
+ raise ActiveRecord::Rollback
43
+ end
44
+
45
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
46
+ end
47
+ end
48
+ end
49
+
50
+ describe "creation" do
51
+ it "upload happens right after insert" do
52
+ with_test_file("example.xml") do |example|
53
+ a = nil
54
+
55
+ ActiveRecord::Base.transaction do
56
+ a = klass.create! file: example
57
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
58
+ end
59
+
60
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
61
+ end
62
+ end
63
+
64
+ it "upload is deleted on rollback" do
65
+ with_test_file("example.xml") do |example|
66
+ path = nil
67
+
68
+ ActiveRecord::Base.transaction do
69
+ a = klass.create! file: example
70
+ path = a[:file]
71
+ expect(Saviour::Config.storage.exists?(path)).to be_truthy
72
+ raise ActiveRecord::Rollback
73
+ end
74
+
75
+ expect(klass.count).to eq 0
76
+ expect(Saviour::Config.storage.exists?(path)).to be_falsey
77
+ end
78
+ end
79
+ end
80
+
81
+ describe "update with a different path" do
82
+ it "leaves the previous file alive until after commit, then deletes it" do
83
+ with_test_file("example.xml") do |example|
84
+ a = klass.create! file: example
85
+ path1 = a[:file]
86
+ expect(Saviour::Config.storage.exists?(path1)).to be_truthy
87
+
88
+ with_test_file("camaloon.jpg") do |file2|
89
+ ActiveRecord::Base.transaction do
90
+ a.update_attributes! file: file2
91
+
92
+ expect(Saviour::Config.storage.exists?(path1)).to be_truthy
93
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
94
+ end
95
+
96
+ expect(Saviour::Config.storage.exists?(path1)).to be_falsey
97
+ end
98
+ end
99
+ end
100
+
101
+ it "on rollback doesn't delete previous file and deletes new file" do
102
+ with_test_file("example.xml") do |example|
103
+ a = klass.create! file: example
104
+ path1 = a[:file]
105
+ path2 = nil
106
+ expect(Saviour::Config.storage.exists?(path1)).to be_truthy
107
+
108
+ with_test_file("camaloon.jpg") do |file2|
109
+ ActiveRecord::Base.transaction do
110
+ a.update_attributes! file: file2
111
+ path2 = a[:file]
112
+
113
+ expect(Saviour::Config.storage.exists?(path1)).to be_truthy
114
+ expect(Saviour::Config.storage.exists?(path2)).to be_truthy
115
+
116
+ raise ActiveRecord::Rollback
117
+ end
118
+
119
+ expect(a.reload[:file]).to eq path1
120
+
121
+ expect(Saviour::Config.storage.exists?(path1)).to be_truthy
122
+ expect(Saviour::Config.storage.exists?(path2)).to be_falsey
123
+ end
124
+ end
125
+ end
126
+ end
127
+
128
+ describe "update with the same path" do
129
+ before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
130
+
131
+ it "leaves the previous file contents on rollback" do
132
+ a = klass.create! file: Saviour::StringSource.new("original content", "file.txt")
133
+
134
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
135
+ expect(Saviour::Config.storage.read(a[:file])).to eq "original content"
136
+
137
+ ActiveRecord::Base.transaction do
138
+ a.update_attributes! file: Saviour::StringSource.new("new content", "file.txt")
139
+ expect(Saviour::Config.storage.read(a[:file])).to eq "new content"
140
+ raise ActiveRecord::Rollback
141
+ end
142
+
143
+ expect(Saviour::Config.storage.exists?(a[:file])).to be_truthy
144
+ expect(Saviour::Config.storage.read(a[:file])).to eq "original content"
145
+ end
146
+ end
147
+ end
@@ -150,4 +150,23 @@ describe "validations saving a new file" do
150
150
  expect(a.errors[:file][0]).to eq "Received error in file"
151
151
  end
152
152
  end
153
+
154
+ describe "validations with file" do
155
+ it "fails reading the contents" do
156
+ klass = Class.new(base_klass) do
157
+ attach_validation_with_file :file, :check_contents
158
+
159
+ def check_contents(file, _, opts)
160
+ errors.add(:file, "Error!") if ::File.read(file.path) == "X"
161
+ end
162
+ end
163
+
164
+ Tempfile.open("") do |f|
165
+ f.write("X")
166
+ f.flush
167
+ a = klass.new file: f
168
+ expect(a).not_to be_valid
169
+ end
170
+ end
171
+ end
153
172
  end
@@ -0,0 +1,43 @@
1
+ require 'spec_helper'
2
+
3
+ describe "#with_copy" do
4
+ before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
5
+
6
+ let(:uploader) do
7
+ Class.new(Saviour::BaseUploader) {
8
+ store_dir { "/store/dir" }
9
+ process { |contents, filename| [contents, "foo_#{filename}"] }
10
+ }
11
+ end
12
+
13
+ let(:klass) do
14
+ klass = Class.new(Test) { include Saviour::Model }
15
+ klass.attach_file :file, uploader
16
+ klass
17
+ end
18
+
19
+ it "provides a copy of the stored file" do
20
+ a = klass.create! file: Saviour::StringSource.new("some contents", "houhou.txt")
21
+ a.file.with_copy do |tmpfile|
22
+ expect(tmpfile.read).to eq "some contents"
23
+ end
24
+ end
25
+
26
+ it "deletes the copied file even on exception" do
27
+ test_exception = Class.new(StandardError)
28
+
29
+ a = klass.create! file: Saviour::StringSource.new("some contents", "houhou.txt")
30
+ path = nil
31
+
32
+ begin
33
+ a.file.with_copy do |f|
34
+ path = f.path
35
+ expect(File.file?(path)).to be_truthy
36
+ raise(test_exception, "some exception within the block")
37
+ end
38
+ rescue test_exception
39
+ end
40
+
41
+ expect(File.file?(path)).to be_falsey
42
+ end
43
+ end
@@ -1,16 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Saviour::BaseUploader do
4
- let(:mocked_storage) {
5
- Class.new {
6
- def write(content, filename)
7
- # pass
8
- end
9
- }.new
10
- }
11
- before { allow(Saviour::Config).to receive(:storage).and_return(mocked_storage) }
12
-
13
-
14
4
  describe "DSL" do
15
5
  subject { Class.new(Saviour::BaseUploader) }
16
6
 
@@ -81,14 +71,14 @@ describe Saviour::BaseUploader do
81
71
  end
82
72
  end
83
73
 
84
- describe "#write" do
74
+ describe "#_process_as_contents" do
85
75
  subject { uploader.new(data: { model: "model", attached_as: "attached_as" }) }
86
76
 
87
77
  context do
88
78
  let(:uploader) { Class.new(Saviour::BaseUploader) }
89
79
 
90
80
  it "error if no store_dir" do
91
- expect { subject.write("contents", "filename.jpg") }.to raise_error(RuntimeError)
81
+ expect { subject._process_as_contents("contents", "filename.jpg") }.to raise_error(Saviour::ConfigurationError)
92
82
  end
93
83
  end
94
84
 
@@ -97,13 +87,8 @@ describe Saviour::BaseUploader do
97
87
  store_dir { "/store/dir" }
98
88
  } }
99
89
 
100
- it "calls storage write" do
101
- expect(Saviour::Config.storage).to receive(:write).with("contents", "/store/dir/file.jpg")
102
- subject.write("contents", "file.jpg")
103
- end
104
-
105
- it "returns the fullpath" do
106
- expect(subject.write("contents", "file.jpg")).to eq '/store/dir/file.jpg'
90
+ it "returns the final contents and fullpath" do
91
+ expect(subject._process_as_contents("contents", "file.jpg")).to eq ["contents", '/store/dir/file.jpg']
107
92
  end
108
93
  end
109
94
 
@@ -120,8 +105,7 @@ describe Saviour::BaseUploader do
120
105
 
121
106
  it "calls the processors" do
122
107
  expect(subject).to receive(:resize).with("content", "output.png").and_call_original
123
- expect(Saviour::Config.storage).to receive(:write).with("content-x2", "/store/dir/output.png")
124
- subject.write("content", "output.png")
108
+ expect(subject._process_as_contents("content", "output.png")).to eq ["content-x2", "/store/dir/output.png"]
125
109
  end
126
110
  end
127
111
 
@@ -138,8 +122,7 @@ describe Saviour::BaseUploader do
138
122
  } }
139
123
 
140
124
  it "respects ordering on processor calling" do
141
- expect(Saviour::Config.storage).to receive(:write).with("content-x2_x9", "/store/dir/prefix-output.png")
142
- subject.write("content", "output.png")
125
+ expect(subject._process_as_contents("content", "output.png")).to eq ["content-x2_x9", "/store/dir/prefix-output.png"]
143
126
  end
144
127
  end
145
128
 
@@ -155,8 +138,7 @@ describe Saviour::BaseUploader do
155
138
  } }
156
139
 
157
140
  it "calls the method using the stored arguments" do
158
- expect(Saviour::Config.storage).to receive(:write).with("content-50-10", "/store/dir/output.png")
159
- subject.write("content", "output.png")
141
+ expect(subject._process_as_contents("content", "output.png")).to eq ["content-50-10", "/store/dir/output.png"]
160
142
  end
161
143
  end
162
144
 
@@ -176,8 +158,7 @@ describe Saviour::BaseUploader do
176
158
  subject { uploader.new(data: { model: model, attached_as: "attached_as" }) }
177
159
 
178
160
  it "can access model from processors" do
179
- expect(Saviour::Config.storage).to receive(:write).with("content", "/store/dir/Robert_8_output.png")
180
- subject.write("content", "output.png")
161
+ expect(subject._process_as_contents("content", "output.png")).to eq ["content", "/store/dir/Robert_8_output.png"]
181
162
  end
182
163
  end
183
164
 
@@ -188,8 +169,8 @@ describe Saviour::BaseUploader do
188
169
  } }
189
170
 
190
171
  it do
191
- expect(Saviour::Config.storage).to_not receive(:write)
192
- expect(subject.write("contents", "file.jpg")).to be_nil
172
+ expect(Saviour::Config.storage).to_not receive(:_process_as_contents)
173
+ expect(subject._process_as_contents("contents", "file.jpg")).to be_nil
193
174
  end
194
175
  end
195
176
  end
@@ -211,8 +192,7 @@ describe Saviour::BaseUploader do
211
192
 
212
193
  it "calls the processors" do
213
194
  expect(subject).to receive(:foo).with(an_instance_of(Tempfile), "output.png").and_call_original
214
- expect(Saviour::Config.storage).to receive(:write).with("modified-contents", "/store/dir/output.png")
215
- subject.write("contents", "output.png")
195
+ expect(subject._process_as_contents("contents", "output.png")).to eq ["modified-contents", "/store/dir/output.png"]
216
196
  end
217
197
  end
218
198
 
@@ -237,8 +217,7 @@ describe Saviour::BaseUploader do
237
217
  } }
238
218
 
239
219
  it "can mix types of runs between file and contents" do
240
- expect(Saviour::Config.storage).to receive(:write).with("pre-contents_first_run-modified-contents", "/store/dir/pre-aaa.png")
241
- subject.write("contents", "aaa.png")
220
+ expect(subject._process_as_contents("contents", "aaa.png")).to eq ["pre-contents_first_run-modified-contents", "/store/dir/pre-aaa.png"]
242
221
  end
243
222
  end
244
223
  end