tessa 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +5 -0
- data/lib/tasks/tessa.rake +7 -0
- data/lib/tessa/active_storage/asset_wrapper.rb +2 -1
- data/lib/tessa/jobs/migrate_assets_job.rb +216 -0
- data/lib/tessa/model/dynamic_extensions.rb +7 -3
- data/lib/tessa/model.rb +2 -0
- data/lib/tessa/version.rb +1 -1
- data/lib/tessa.rb +8 -0
- data/spec/rails_helper.rb +4 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/tessa/config_spec.rb +22 -22
- data/spec/tessa/jobs/migrate_assets_job_spec.rb +247 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d31c66f27d8c731d0df1e58f196bd6279f4f2603ab39e33f75f8cf46630745fe
|
4
|
+
data.tar.gz: fe6050f938245f6502771adff53f13bd3af40a0339a1098d96bbd2416d6e0a2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76047a38e882dd59fe104b8be03a2970d996e70eb5f469549d8529e384f46732f5f76dad54e0d7c3d158c60de032b01c467ed13202209d0c4fdad4faa0328b84
|
7
|
+
data.tar.gz: '085ac99b9dcce059741887f7852d4a2fd08412b35fa5e520ac61c79f49d5511f43485fe6836863910d07f9336d0c295b31b1bb691b51abc3ea00cefefac35a0b'
|
data/Gemfile
CHANGED
@@ -0,0 +1,216 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
class Tessa::MigrateAssetsJob < ActiveJob::Base
|
4
|
+
|
5
|
+
queue_as :low
|
6
|
+
|
7
|
+
def perform(*args)
|
8
|
+
options = args&.extract_options!
|
9
|
+
options = {
|
10
|
+
batch_size: 10,
|
11
|
+
interval: 10.minutes.to_i
|
12
|
+
}.merge!(options&.symbolize_keys || {})
|
13
|
+
|
14
|
+
interval = options[:interval].seconds
|
15
|
+
processing_state = args.first ? Marshal.load(args.first) : load_models_from_registry
|
16
|
+
|
17
|
+
if processing_state.fully_processed?
|
18
|
+
Rails.logger.info("Nothing to do - all models have transitioned to ActiveStorage")
|
19
|
+
return
|
20
|
+
end
|
21
|
+
|
22
|
+
processing_state.batch_count = 0
|
23
|
+
while processing_state.batch_count < options[:batch_size]
|
24
|
+
model_state = processing_state.next_model
|
25
|
+
|
26
|
+
process(processing_state, model_state, options)
|
27
|
+
|
28
|
+
break if processing_state.fully_processed?
|
29
|
+
end
|
30
|
+
|
31
|
+
if processing_state.fully_processed?
|
32
|
+
Rails.logger.info("Finished processing all Tessa assets")
|
33
|
+
else
|
34
|
+
remaining_batches = (processing_state.count / options[:batch_size].to_f).ceil
|
35
|
+
|
36
|
+
Rails.logger.info("Continuing processing in #{interval}, "\
|
37
|
+
"ETA #{(remaining_batches * interval).from_now}. "\
|
38
|
+
"Working on #{processing_state.next_model.next_field}")
|
39
|
+
|
40
|
+
processing_state.batch_count = 0
|
41
|
+
self.class.set(wait: interval)
|
42
|
+
.perform_later(Marshal.dump(processing_state), options)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def process(processing_state, model_state, options)
|
49
|
+
while processing_state.batch_count < options[:batch_size]
|
50
|
+
field_state = model_state.next_field
|
51
|
+
|
52
|
+
process_field(processing_state, field_state, options)
|
53
|
+
|
54
|
+
return if model_state.fully_processed?
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def process_field(processing_state, field_state, options)
|
59
|
+
while processing_state.batch_count < options[:batch_size]
|
60
|
+
remaining = options[:batch_size] - processing_state.batch_count
|
61
|
+
|
62
|
+
next_batch = field_state.query
|
63
|
+
.offset(field_state.offset)
|
64
|
+
.limit(remaining)
|
65
|
+
|
66
|
+
next_batch.each_with_index do |record, idx|
|
67
|
+
begin
|
68
|
+
# Wait 1 second in between records to slow things down and let the
|
69
|
+
# system process for a bit
|
70
|
+
sleep 1 if idx > 0
|
71
|
+
|
72
|
+
reupload(record, field_state)
|
73
|
+
Rails.logger.info("#{record.class}#{record.id}##{field_state.field_name}: success")
|
74
|
+
field_state.success_count += 1
|
75
|
+
rescue StandardError => ex
|
76
|
+
Rails.logger.error("#{record.class}#{record.id}##{field_state.field_name}: error - #{ex}")
|
77
|
+
field_state.offset += 1
|
78
|
+
ensure
|
79
|
+
processing_state.batch_count += 1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
return if field_state.fully_processed?
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def reupload(record, field_state)
|
88
|
+
if field_state.tessa_field.multiple?
|
89
|
+
reupload_multiple(record, field_state)
|
90
|
+
else
|
91
|
+
reupload_single(record, field_state)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def reupload_single(record, field_state)
|
96
|
+
# models with ActiveStorage uploads have nil in the column, but if you call
|
97
|
+
# the method it hits the dynamic extensions and gives you the blob key
|
98
|
+
database_id = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
|
99
|
+
return unless database_id
|
100
|
+
|
101
|
+
asset = Tessa::Asset.find(database_id)
|
102
|
+
|
103
|
+
attachable = {
|
104
|
+
io: URI.open(asset.private_download_url),
|
105
|
+
filename: asset.meta[:name]
|
106
|
+
}
|
107
|
+
|
108
|
+
record.public_send("#{field_state.tessa_field.name}=", attachable)
|
109
|
+
record.save!
|
110
|
+
end
|
111
|
+
|
112
|
+
def reupload_multiple(record, field_state)
|
113
|
+
database_ids = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
|
114
|
+
return unless database_ids
|
115
|
+
|
116
|
+
assets = Tessa::Asset.find(database_ids)
|
117
|
+
|
118
|
+
attachables = assets.map do |asset|
|
119
|
+
{
|
120
|
+
io: URI.open(asset.private_download_url),
|
121
|
+
filename: asset.meta[:name]
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
125
|
+
record.public_send("#{field_state.tessa_field.name}=", attachables)
|
126
|
+
record.save!
|
127
|
+
end
|
128
|
+
|
129
|
+
def load_models_from_registry
|
130
|
+
Rails.application.eager_load!
|
131
|
+
|
132
|
+
# Load all Tessa models that can have attachments (not form objects)
|
133
|
+
models = Tessa.model_registry
|
134
|
+
.select { |m| m.respond_to?(:has_one_attached) }
|
135
|
+
|
136
|
+
# Initialize our Record Keeping object
|
137
|
+
ProcessingState.initialize_from_models(models)
|
138
|
+
end
|
139
|
+
|
140
|
+
ProcessingState = Struct.new(:model_queue, :batch_count) do
|
141
|
+
def self.initialize_from_models(models)
|
142
|
+
new(
|
143
|
+
models.map do |model|
|
144
|
+
ModelProcessingState.initialize_from_model(model)
|
145
|
+
end,
|
146
|
+
0
|
147
|
+
)
|
148
|
+
end
|
149
|
+
|
150
|
+
def next_model
|
151
|
+
model_queue.detect { |i| !i.fully_processed? }
|
152
|
+
end
|
153
|
+
|
154
|
+
def fully_processed?
|
155
|
+
model_queue.all?(&:fully_processed?)
|
156
|
+
end
|
157
|
+
|
158
|
+
def count
|
159
|
+
model_queue.sum { |m| m.field_queue.sum { |f| f.count - f.offset } }
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
ModelProcessingState = Struct.new(:class_name, :field_queue) do
|
164
|
+
def self.initialize_from_model(model)
|
165
|
+
new(
|
166
|
+
model.name,
|
167
|
+
model.tessa_fields.map do |name, _|
|
168
|
+
FieldProcessingState.initialize_from_model(model, name)
|
169
|
+
end
|
170
|
+
)
|
171
|
+
end
|
172
|
+
|
173
|
+
def next_field
|
174
|
+
field_queue.detect { |i| !i.fully_processed? }
|
175
|
+
end
|
176
|
+
|
177
|
+
def model
|
178
|
+
@model ||= class_name.constantize
|
179
|
+
end
|
180
|
+
|
181
|
+
def fully_processed?
|
182
|
+
field_queue.all?(&:fully_processed?)
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
FieldProcessingState = Struct.new(:class_name, :field_name, :offset, :success_count) do
|
187
|
+
def self.initialize_from_model(model, field_name)
|
188
|
+
new(
|
189
|
+
model.name,
|
190
|
+
field_name,
|
191
|
+
0,
|
192
|
+
0
|
193
|
+
)
|
194
|
+
end
|
195
|
+
|
196
|
+
def model
|
197
|
+
@model ||= class_name.constantize
|
198
|
+
end
|
199
|
+
|
200
|
+
def tessa_field
|
201
|
+
model.tessa_fields[field_name]
|
202
|
+
end
|
203
|
+
|
204
|
+
def query
|
205
|
+
model.where.not(Hash[tessa_field.id_field, nil])
|
206
|
+
end
|
207
|
+
|
208
|
+
def count
|
209
|
+
query.count
|
210
|
+
end
|
211
|
+
|
212
|
+
def fully_processed?
|
213
|
+
offset >= count
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
@@ -62,7 +62,8 @@ class Tessa::DynamicExtensions
|
|
62
62
|
|
63
63
|
def attributes
|
64
64
|
super.merge({
|
65
|
-
'#{field.id_field}' => #{field.id_field}
|
65
|
+
'#{field.id_field}' => #{field.id_field},
|
66
|
+
'_tessa_#{field.id_field}' => super['#{field.id_field}']
|
66
67
|
})
|
67
68
|
end
|
68
69
|
CODE
|
@@ -105,7 +106,7 @@ class Tessa::DynamicExtensions
|
|
105
106
|
else
|
106
107
|
ids = self.#{field.id_field}
|
107
108
|
ids.delete(change.id.to_i)
|
108
|
-
self.#{field.id_field} = ids
|
109
|
+
self.#{field.id_field} = ids.any? ? ids : nil
|
109
110
|
end
|
110
111
|
end
|
111
112
|
attachables.changes.select(&:add?).each do |change|
|
@@ -115,14 +116,17 @@ class Tessa::DynamicExtensions
|
|
115
116
|
end
|
116
117
|
when nil
|
117
118
|
a.detach
|
119
|
+
self.#{field.id_field} = nil
|
118
120
|
else
|
119
121
|
a.attach(*attachables)
|
122
|
+
self.#{field.id_field} = nil
|
120
123
|
end
|
121
124
|
end
|
122
125
|
|
123
126
|
def attributes
|
124
127
|
super.merge({
|
125
|
-
'#{field.id_field}' => #{field.id_field}
|
128
|
+
'#{field.id_field}' => #{field.id_field},
|
129
|
+
'_tessa_#{field.id_field}' => super['#{field.id_field}']
|
126
130
|
})
|
127
131
|
end
|
128
132
|
CODE
|
data/lib/tessa/model.rb
CHANGED
@@ -10,6 +10,8 @@ module Tessa
|
|
10
10
|
base.extend ClassMethods
|
11
11
|
base.after_commit :apply_tessa_change_sets if base.respond_to?(:after_commit)
|
12
12
|
base.before_destroy :remove_all_tessa_assets if base.respond_to?(:before_destroy)
|
13
|
+
|
14
|
+
Tessa.model_registry << base
|
13
15
|
end
|
14
16
|
|
15
17
|
module InstanceMethods
|
data/lib/tessa/version.rb
CHANGED
data/lib/tessa.rb
CHANGED
@@ -15,6 +15,10 @@ require "tessa/rack_upload_proxy"
|
|
15
15
|
require "tessa/upload"
|
16
16
|
require "tessa/view_helpers"
|
17
17
|
|
18
|
+
if defined?(ActiveJob)
|
19
|
+
require "tessa/jobs/migrate_assets_job"
|
20
|
+
end
|
21
|
+
|
18
22
|
module Tessa
|
19
23
|
class << self
|
20
24
|
def config
|
@@ -31,6 +35,10 @@ module Tessa
|
|
31
35
|
return find_asset(ids)
|
32
36
|
end
|
33
37
|
|
38
|
+
def model_registry
|
39
|
+
@model_registry ||= []
|
40
|
+
end
|
41
|
+
|
34
42
|
private
|
35
43
|
|
36
44
|
def find_asset(id)
|
data/spec/rails_helper.rb
CHANGED
@@ -5,10 +5,14 @@ require 'spec_helper'
|
|
5
5
|
|
6
6
|
require File.expand_path('dummy/config/environment.rb', __dir__)
|
7
7
|
|
8
|
+
require 'rspec/rails'
|
9
|
+
|
8
10
|
RSpec.configure do |config|
|
9
11
|
ActiveStorage::Current.host = 'https://www.example.com'
|
10
12
|
Rails.application.routes.default_url_options = {
|
11
13
|
protocol: 'https',
|
12
14
|
host: "www.example.com"
|
13
15
|
}
|
16
|
+
|
17
|
+
config.use_transactional_fixtures = true
|
14
18
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'webmock/rspec'
|
1
2
|
require 'tessa'
|
2
3
|
require 'tempfile'
|
3
4
|
|
@@ -11,6 +12,8 @@ if ENV['SIMPLE_COV'] || ENV['CC_TEST_REPORTER_ID']
|
|
11
12
|
end
|
12
13
|
|
13
14
|
RSpec.configure do |config|
|
15
|
+
WebMock.disable_net_connect!
|
16
|
+
|
14
17
|
config.expect_with :rspec do |expectations|
|
15
18
|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
16
19
|
end
|
data/spec/tessa/config_spec.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe Tessa::Config do
|
4
|
-
|
4
|
+
let(:cfg) { Tessa::Config.new }
|
5
5
|
|
6
6
|
shared_examples_for "defaults to environment variable" do
|
7
7
|
around { |ex| swap_environment_var(variable_name, 'from-env') { ex.run } }
|
@@ -19,63 +19,63 @@ RSpec.describe Tessa::Config do
|
|
19
19
|
describe "#username" do
|
20
20
|
it_behaves_like "defaults to environment variable" do
|
21
21
|
let(:variable_name) { 'TESSA_USERNAME' }
|
22
|
-
subject
|
22
|
+
subject { cfg.username }
|
23
23
|
end
|
24
24
|
|
25
25
|
it "behaves like a normal accessor" do
|
26
|
-
|
27
|
-
expect(
|
26
|
+
cfg.username = "my-new-value"
|
27
|
+
expect(cfg.username).to eq("my-new-value")
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
31
|
describe "#password" do
|
32
32
|
it_behaves_like "defaults to environment variable" do
|
33
33
|
let(:variable_name) { 'TESSA_PASSWORD' }
|
34
|
-
subject
|
34
|
+
subject { cfg.password }
|
35
35
|
end
|
36
36
|
|
37
37
|
it "behaves like a normal accessor" do
|
38
|
-
|
39
|
-
expect(
|
38
|
+
cfg.password = "my-new-value"
|
39
|
+
expect(cfg.password).to eq("my-new-value")
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
43
|
describe "#url" do
|
44
44
|
it_behaves_like "defaults to environment variable" do
|
45
45
|
let(:variable_name) { 'TESSA_URL' }
|
46
|
-
subject
|
46
|
+
subject { cfg.url }
|
47
47
|
end
|
48
48
|
|
49
49
|
it "behaves like a normal accessor" do
|
50
|
-
|
51
|
-
expect(
|
50
|
+
cfg.url = "my-new-value"
|
51
|
+
expect(cfg.url).to eq("my-new-value")
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
55
|
describe "#strategy" do
|
56
56
|
it_behaves_like "defaults to environment variable" do
|
57
57
|
let(:variable_name) { 'TESSA_STRATEGY' }
|
58
|
-
subject
|
58
|
+
subject { cfg.strategy }
|
59
59
|
end
|
60
60
|
|
61
61
|
it "uses the string 'default' when no envvar passed" do
|
62
|
-
expect(
|
62
|
+
expect(cfg.strategy).to eq("default")
|
63
63
|
end
|
64
64
|
|
65
65
|
it "behaves like a normal accessor" do
|
66
|
-
|
67
|
-
expect(
|
66
|
+
cfg.strategy = "my-new-value"
|
67
|
+
expect(cfg.strategy).to eq("my-new-value")
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
71
|
describe "#connection" do
|
72
72
|
it "is a Faraday::Connection" do
|
73
|
-
expect(
|
73
|
+
expect(cfg.connection).to be_a(Faraday::Connection)
|
74
74
|
end
|
75
75
|
|
76
|
-
context "with values
|
77
|
-
subject
|
78
|
-
before { args.each { |k, v|
|
76
|
+
context "with values cfgured" do
|
77
|
+
subject { cfg.connection }
|
78
|
+
before { args.each { |k, v| cfg.send("#{k}=", v) } }
|
79
79
|
let(:args) {
|
80
80
|
{
|
81
81
|
url: "http://tessa.test",
|
@@ -85,28 +85,28 @@ RSpec.describe Tessa::Config do
|
|
85
85
|
}
|
86
86
|
|
87
87
|
it "sets faraday's url prefix to our url" do
|
88
|
-
expect(
|
88
|
+
expect(subject.url_prefix.to_s).to match(cfg.url)
|
89
89
|
end
|
90
90
|
|
91
91
|
context "with faraday spy" do
|
92
92
|
let(:spy) { instance_spy(Faraday::Connection) }
|
93
93
|
before do
|
94
94
|
expect(Faraday).to receive(:new).and_yield(spy)
|
95
|
-
|
95
|
+
subject
|
96
96
|
end
|
97
97
|
|
98
98
|
it "sets up url_encoded request handler" do
|
99
99
|
expect(spy).to have_received(:request).with(:url_encoded)
|
100
100
|
end
|
101
101
|
|
102
|
-
it "
|
102
|
+
it "cfgures the default adapter" do
|
103
103
|
expect(spy).to have_received(:adapter).with(:net_http)
|
104
104
|
end
|
105
105
|
end
|
106
106
|
end
|
107
107
|
|
108
108
|
it "caches the result" do
|
109
|
-
expect(
|
109
|
+
expect(cfg.connection.object_id).to eq(cfg.connection.object_id)
|
110
110
|
end
|
111
111
|
end
|
112
112
|
end
|
@@ -0,0 +1,247 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
|
3
|
+
require 'tessa/jobs/migrate_assets_job'
|
4
|
+
|
5
|
+
RSpec.describe Tessa::MigrateAssetsJob do
|
6
|
+
it 'does nothing if no db rows' do
|
7
|
+
|
8
|
+
expect {
|
9
|
+
subject.perform
|
10
|
+
}.to_not change { ActiveStorage::Attachment.count }
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'creates attachments for old tessa assets' do
|
14
|
+
allow(Tessa::Asset).to receive(:find)
|
15
|
+
.with(1)
|
16
|
+
.and_return(
|
17
|
+
Tessa::Asset.new(id: 1,
|
18
|
+
meta: { name: 'README.md' },
|
19
|
+
private_download_url: 'https://test.com/README.md')
|
20
|
+
)
|
21
|
+
|
22
|
+
allow(Tessa::Asset).to receive(:find)
|
23
|
+
.with(2)
|
24
|
+
.and_return(
|
25
|
+
Tessa::Asset.new(id: 2,
|
26
|
+
meta: { name: 'LICENSE.txt' },
|
27
|
+
private_download_url: 'https://test.com/LICENSE.txt'),
|
28
|
+
)
|
29
|
+
|
30
|
+
stub_request(:get, 'https://test.com/README.md')
|
31
|
+
.to_return(body: File.new('README.md'))
|
32
|
+
stub_request(:get, 'https://test.com/LICENSE.txt')
|
33
|
+
.to_return(body: File.new('LICENSE.txt'))
|
34
|
+
|
35
|
+
# DB models with those
|
36
|
+
models = [
|
37
|
+
SingleAssetModel.create!(avatar_id: 1),
|
38
|
+
SingleAssetModel.create!(avatar_id: 2)
|
39
|
+
]
|
40
|
+
|
41
|
+
expect {
|
42
|
+
subject.perform
|
43
|
+
}.to change { ActiveStorage::Attachment.count }.by(2)
|
44
|
+
|
45
|
+
models.each do |m|
|
46
|
+
expect(m.reload.avatar_id).to eq(m.avatar.key)
|
47
|
+
# Now it's in activestorage
|
48
|
+
expect(m.avatar.public_url).to start_with(
|
49
|
+
'https://www.example.com/rails/active_storage/blobs/')
|
50
|
+
end
|
51
|
+
|
52
|
+
expect(SingleAssetModel.where.not('avatar_id' => nil).count)
|
53
|
+
.to eq(0)
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'preserves ActiveStorage blobs' do
|
57
|
+
allow(Tessa::Asset).to receive(:find)
|
58
|
+
.with([1, 2])
|
59
|
+
.and_return([
|
60
|
+
Tessa::Asset.new(id: 1,
|
61
|
+
meta: { name: 'README.md' },
|
62
|
+
private_download_url: 'https://test.com/README.md'),
|
63
|
+
Tessa::Asset.new(id: 2,
|
64
|
+
meta: { name: 'LICENSE.txt' },
|
65
|
+
private_download_url: 'https://test.com/LICENSE.txt'),
|
66
|
+
])
|
67
|
+
stub_request(:get, 'https://test.com/README.md')
|
68
|
+
.to_return(body: File.new('README.md'))
|
69
|
+
stub_request(:get, 'https://test.com/LICENSE.txt')
|
70
|
+
.to_return(body: File.new('LICENSE.txt'))
|
71
|
+
|
72
|
+
file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
|
73
|
+
|
74
|
+
model = MultipleAssetModel.create!(
|
75
|
+
# The Tessa DB column has the one asset
|
76
|
+
another_place: [1, 2]
|
77
|
+
)
|
78
|
+
# But has already attached a second ActiveStorage blob
|
79
|
+
::ActiveStorage::Attached::Many.new("multiple_field", model, dependent: :purge_later)
|
80
|
+
.attach(file2)
|
81
|
+
model.save!
|
82
|
+
attachment = model.multiple_field_attachments.first
|
83
|
+
|
84
|
+
expect {
|
85
|
+
subject.perform
|
86
|
+
}.to change { ActiveStorage::Attachment.count }.by(2)
|
87
|
+
|
88
|
+
model = model.reload
|
89
|
+
# The IDs are now the keys of ActiveStorage objects
|
90
|
+
expect(model.another_place).to eq(
|
91
|
+
model.multiple_field.map(&:key))
|
92
|
+
# preserves the existing attachment
|
93
|
+
expect(model.multiple_field_attachments).to include(attachment)
|
94
|
+
expect(model.another_place).to include(attachment.key)
|
95
|
+
|
96
|
+
# all assets are in activestorage
|
97
|
+
model.multiple_field.each do |blob|
|
98
|
+
expect(blob.public_url).to start_with(
|
99
|
+
'https://www.example.com/rails/active_storage/blobs/')
|
100
|
+
end
|
101
|
+
|
102
|
+
# DB column is reset to nil
|
103
|
+
expect(MultipleAssetModel.where.not('another_place' => nil).count)
|
104
|
+
.to eq(0)
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'Stops after hitting the batch size' do
|
108
|
+
1.upto(11).each do |i|
|
109
|
+
asset = Tessa::Asset.new(id: i,
|
110
|
+
meta: { name: 'README.md' },
|
111
|
+
private_download_url: 'https://test.com/README.md')
|
112
|
+
allow(Tessa::Asset).to receive(:find)
|
113
|
+
.with(i)
|
114
|
+
.and_return(asset)
|
115
|
+
allow(Tessa::Asset).to receive(:find)
|
116
|
+
.with([i])
|
117
|
+
.and_return([asset])
|
118
|
+
end
|
119
|
+
stub_request(:get, 'https://test.com/README.md')
|
120
|
+
.to_return(body: File.new('README.md'))
|
121
|
+
|
122
|
+
# Mix of the two models...
|
123
|
+
models =
|
124
|
+
1.upto(11).map do |i|
|
125
|
+
if i % 2 == 0
|
126
|
+
MultipleAssetModel.create!(another_place: [i])
|
127
|
+
else
|
128
|
+
SingleAssetModel.create!(avatar_id: i)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
final_state = nil
|
133
|
+
final_options = nil
|
134
|
+
dbl = double('set')
|
135
|
+
expect(dbl).to receive(:perform_later) do |state, options|
|
136
|
+
final_state = Marshal.load(state)
|
137
|
+
final_options = options
|
138
|
+
end
|
139
|
+
expect(Tessa::MigrateAssetsJob).to receive(:set)
|
140
|
+
.with(wait: 10.minutes)
|
141
|
+
.and_return(dbl)
|
142
|
+
|
143
|
+
expect {
|
144
|
+
subject.perform
|
145
|
+
}.to change { ActiveStorage::Attachment.count }.by(10)
|
146
|
+
|
147
|
+
expect(final_state.fully_processed?).to be false
|
148
|
+
expect(final_options).to eq({
|
149
|
+
batch_size: 10,
|
150
|
+
interval: 10.minutes.to_i
|
151
|
+
})
|
152
|
+
# One of the two models was fully processed
|
153
|
+
expect(final_state.model_queue.count(&:fully_processed?))
|
154
|
+
.to eq(1)
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'Skips over Tessa errors' do
|
158
|
+
1.upto(11).each do |i|
|
159
|
+
if i % 2 == 0
|
160
|
+
allow(Tessa::Asset).to receive(:find)
|
161
|
+
.with(i)
|
162
|
+
.and_raise(Tessa::RequestFailed)
|
163
|
+
next
|
164
|
+
end
|
165
|
+
|
166
|
+
allow(Tessa::Asset).to receive(:find)
|
167
|
+
.with(i)
|
168
|
+
.and_return(
|
169
|
+
Tessa::Asset.new(id: i,
|
170
|
+
meta: { name: 'README.md' },
|
171
|
+
private_download_url: 'https://test.com/README.md')
|
172
|
+
)
|
173
|
+
end
|
174
|
+
stub_request(:get, 'https://test.com/README.md')
|
175
|
+
.to_return(body: File.new('README.md'))
|
176
|
+
|
177
|
+
# Mix of the two models...
|
178
|
+
models =
|
179
|
+
1.upto(11).map do |i|
|
180
|
+
SingleAssetModel.create!(avatar_id: i)
|
181
|
+
end
|
182
|
+
|
183
|
+
final_state = nil
|
184
|
+
final_options = nil
|
185
|
+
dbl = double('set')
|
186
|
+
expect(dbl).to receive(:perform_later) do |state, options|
|
187
|
+
final_state = Marshal.load(state)
|
188
|
+
final_options = options
|
189
|
+
end
|
190
|
+
expect(Tessa::MigrateAssetsJob).to receive(:set)
|
191
|
+
.with(wait: 10.minutes)
|
192
|
+
.and_return(dbl)
|
193
|
+
|
194
|
+
expect {
|
195
|
+
subject.perform
|
196
|
+
}.to change { ActiveStorage::Attachment.count }.by(5)
|
197
|
+
|
198
|
+
expect(final_state.fully_processed?).to be false
|
199
|
+
field_state = final_state.next_model.next_field
|
200
|
+
expect(field_state.offset).to eq(5)
|
201
|
+
end
|
202
|
+
|
203
|
+
it 'Resumes from marshalled state' do
|
204
|
+
|
205
|
+
file = Rack::Test::UploadedFile.new('README.md')
|
206
|
+
|
207
|
+
state = Tessa::MigrateAssetsJob::ProcessingState.initialize_from_models(
|
208
|
+
[SingleAssetModel])
|
209
|
+
field_state = state.model_queue
|
210
|
+
.detect { |m| m.class_name == 'SingleAssetModel' }
|
211
|
+
.field_queue
|
212
|
+
.detect { |m| m.field_name == :avatar }
|
213
|
+
|
214
|
+
1.upto(10).each do |i|
|
215
|
+
if i % 2 == 0
|
216
|
+
# This one failed
|
217
|
+
SingleAssetModel.create!(avatar_id: i).tap do |r|
|
218
|
+
field_state.offset += 1
|
219
|
+
end
|
220
|
+
else
|
221
|
+
# This one succeeded and is in ActiveStorage
|
222
|
+
SingleAssetModel.create!(avatar: file).tap do |r|
|
223
|
+
field_state.success_count += 1
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
# This one still needs to transition
|
229
|
+
model = SingleAssetModel.create!(avatar_id: 11)
|
230
|
+
|
231
|
+
asset = Tessa::Asset.new(id: 11,
|
232
|
+
meta: { name: 'README.md' },
|
233
|
+
private_download_url: 'https://test.com/README.md')
|
234
|
+
allow(Tessa::Asset).to receive(:find)
|
235
|
+
.with(11)
|
236
|
+
.and_return(asset)
|
237
|
+
stub_request(:get, 'https://test.com/README.md')
|
238
|
+
.to_return(body: File.new('README.md'))
|
239
|
+
|
240
|
+
# Doesn't reenqueue since we finished processing
|
241
|
+
expect(Tessa::MigrateAssetsJob).to_not receive(:set)
|
242
|
+
|
243
|
+
expect {
|
244
|
+
subject.perform(Marshal.dump(state), { batch_size: 2, interval: 3 })
|
245
|
+
}.to change { ActiveStorage::Attachment.count }.by(1)
|
246
|
+
end
|
247
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tessa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Powell
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-06-
|
12
|
+
date: 2022-06-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: faraday
|
@@ -151,6 +151,7 @@ files:
|
|
151
151
|
- app/javascript/tessa/index.js.coffee
|
152
152
|
- bin/rspec
|
153
153
|
- config/routes.rb
|
154
|
+
- lib/tasks/tessa.rake
|
154
155
|
- lib/tessa.rb
|
155
156
|
- lib/tessa/active_storage/asset_wrapper.rb
|
156
157
|
- lib/tessa/asset.rb
|
@@ -160,6 +161,7 @@ files:
|
|
160
161
|
- lib/tessa/config.rb
|
161
162
|
- lib/tessa/controller_helpers.rb
|
162
163
|
- lib/tessa/engine.rb
|
164
|
+
- lib/tessa/jobs/migrate_assets_job.rb
|
163
165
|
- lib/tessa/model.rb
|
164
166
|
- lib/tessa/model/dynamic_extensions.rb
|
165
167
|
- lib/tessa/model/field.rb
|
@@ -240,6 +242,7 @@ files:
|
|
240
242
|
- spec/tessa/asset_spec.rb
|
241
243
|
- spec/tessa/config_spec.rb
|
242
244
|
- spec/tessa/controller_helpers_spec.rb
|
245
|
+
- spec/tessa/jobs/migrate_assets_job_spec.rb
|
243
246
|
- spec/tessa/model_field_spec.rb
|
244
247
|
- spec/tessa/model_spec.rb
|
245
248
|
- spec/tessa/rack_upload_proxy_spec.rb
|
@@ -340,6 +343,7 @@ test_files:
|
|
340
343
|
- spec/tessa/asset_spec.rb
|
341
344
|
- spec/tessa/config_spec.rb
|
342
345
|
- spec/tessa/controller_helpers_spec.rb
|
346
|
+
- spec/tessa/jobs/migrate_assets_job_spec.rb
|
343
347
|
- spec/tessa/model_field_spec.rb
|
344
348
|
- spec/tessa/model_spec.rb
|
345
349
|
- spec/tessa/rack_upload_proxy_spec.rb
|