tessa 1.0.2 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +5 -0
- data/lib/tasks/tessa.rake +7 -0
- data/lib/tessa/active_storage/asset_wrapper.rb +2 -1
- data/lib/tessa/jobs/migrate_assets_job.rb +216 -0
- data/lib/tessa/model/dynamic_extensions.rb +7 -3
- data/lib/tessa/model.rb +2 -0
- data/lib/tessa/version.rb +1 -1
- data/lib/tessa.rb +8 -0
- data/spec/rails_helper.rb +4 -0
- data/spec/spec_helper.rb +3 -0
- data/spec/tessa/config_spec.rb +22 -22
- data/spec/tessa/jobs/migrate_assets_job_spec.rb +247 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d31c66f27d8c731d0df1e58f196bd6279f4f2603ab39e33f75f8cf46630745fe
|
4
|
+
data.tar.gz: fe6050f938245f6502771adff53f13bd3af40a0339a1098d96bbd2416d6e0a2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76047a38e882dd59fe104b8be03a2970d996e70eb5f469549d8529e384f46732f5f76dad54e0d7c3d158c60de032b01c467ed13202209d0c4fdad4faa0328b84
|
7
|
+
data.tar.gz: '085ac99b9dcce059741887f7852d4a2fd08412b35fa5e520ac61c79f49d5511f43485fe6836863910d07f9336d0c295b31b1bb691b51abc3ea00cefefac35a0b'
|
data/Gemfile
CHANGED
@@ -0,0 +1,216 @@
|
|
1
|
+
require 'open-uri'
|
2
|
+
|
3
|
+
class Tessa::MigrateAssetsJob < ActiveJob::Base
|
4
|
+
|
5
|
+
queue_as :low
|
6
|
+
|
7
|
+
def perform(*args)
|
8
|
+
options = args&.extract_options!
|
9
|
+
options = {
|
10
|
+
batch_size: 10,
|
11
|
+
interval: 10.minutes.to_i
|
12
|
+
}.merge!(options&.symbolize_keys || {})
|
13
|
+
|
14
|
+
interval = options[:interval].seconds
|
15
|
+
processing_state = args.first ? Marshal.load(args.first) : load_models_from_registry
|
16
|
+
|
17
|
+
if processing_state.fully_processed?
|
18
|
+
Rails.logger.info("Nothing to do - all models have transitioned to ActiveStorage")
|
19
|
+
return
|
20
|
+
end
|
21
|
+
|
22
|
+
processing_state.batch_count = 0
|
23
|
+
while processing_state.batch_count < options[:batch_size]
|
24
|
+
model_state = processing_state.next_model
|
25
|
+
|
26
|
+
process(processing_state, model_state, options)
|
27
|
+
|
28
|
+
break if processing_state.fully_processed?
|
29
|
+
end
|
30
|
+
|
31
|
+
if processing_state.fully_processed?
|
32
|
+
Rails.logger.info("Finished processing all Tessa assets")
|
33
|
+
else
|
34
|
+
remaining_batches = (processing_state.count / options[:batch_size].to_f).ceil
|
35
|
+
|
36
|
+
Rails.logger.info("Continuing processing in #{interval}, "\
|
37
|
+
"ETA #{(remaining_batches * interval).from_now}. "\
|
38
|
+
"Working on #{processing_state.next_model.next_field}")
|
39
|
+
|
40
|
+
processing_state.batch_count = 0
|
41
|
+
self.class.set(wait: interval)
|
42
|
+
.perform_later(Marshal.dump(processing_state), options)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def process(processing_state, model_state, options)
|
49
|
+
while processing_state.batch_count < options[:batch_size]
|
50
|
+
field_state = model_state.next_field
|
51
|
+
|
52
|
+
process_field(processing_state, field_state, options)
|
53
|
+
|
54
|
+
return if model_state.fully_processed?
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def process_field(processing_state, field_state, options)
|
59
|
+
while processing_state.batch_count < options[:batch_size]
|
60
|
+
remaining = options[:batch_size] - processing_state.batch_count
|
61
|
+
|
62
|
+
next_batch = field_state.query
|
63
|
+
.offset(field_state.offset)
|
64
|
+
.limit(remaining)
|
65
|
+
|
66
|
+
next_batch.each_with_index do |record, idx|
|
67
|
+
begin
|
68
|
+
# Wait 1 second in between records to slow things down and let the
|
69
|
+
# system process for a bit
|
70
|
+
sleep 1 if idx > 0
|
71
|
+
|
72
|
+
reupload(record, field_state)
|
73
|
+
Rails.logger.info("#{record.class}#{record.id}##{field_state.field_name}: success")
|
74
|
+
field_state.success_count += 1
|
75
|
+
rescue StandardError => ex
|
76
|
+
Rails.logger.error("#{record.class}#{record.id}##{field_state.field_name}: error - #{ex}")
|
77
|
+
field_state.offset += 1
|
78
|
+
ensure
|
79
|
+
processing_state.batch_count += 1
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
return if field_state.fully_processed?
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def reupload(record, field_state)
|
88
|
+
if field_state.tessa_field.multiple?
|
89
|
+
reupload_multiple(record, field_state)
|
90
|
+
else
|
91
|
+
reupload_single(record, field_state)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def reupload_single(record, field_state)
|
96
|
+
# models with ActiveStorage uploads have nil in the column, but if you call
|
97
|
+
# the method it hits the dynamic extensions and gives you the blob key
|
98
|
+
database_id = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
|
99
|
+
return unless database_id
|
100
|
+
|
101
|
+
asset = Tessa::Asset.find(database_id)
|
102
|
+
|
103
|
+
attachable = {
|
104
|
+
io: URI.open(asset.private_download_url),
|
105
|
+
filename: asset.meta[:name]
|
106
|
+
}
|
107
|
+
|
108
|
+
record.public_send("#{field_state.tessa_field.name}=", attachable)
|
109
|
+
record.save!
|
110
|
+
end
|
111
|
+
|
112
|
+
def reupload_multiple(record, field_state)
|
113
|
+
database_ids = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
|
114
|
+
return unless database_ids
|
115
|
+
|
116
|
+
assets = Tessa::Asset.find(database_ids)
|
117
|
+
|
118
|
+
attachables = assets.map do |asset|
|
119
|
+
{
|
120
|
+
io: URI.open(asset.private_download_url),
|
121
|
+
filename: asset.meta[:name]
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
125
|
+
record.public_send("#{field_state.tessa_field.name}=", attachables)
|
126
|
+
record.save!
|
127
|
+
end
|
128
|
+
|
129
|
+
def load_models_from_registry
|
130
|
+
Rails.application.eager_load!
|
131
|
+
|
132
|
+
# Load all Tessa models that can have attachments (not form objects)
|
133
|
+
models = Tessa.model_registry
|
134
|
+
.select { |m| m.respond_to?(:has_one_attached) }
|
135
|
+
|
136
|
+
# Initialize our Record Keeping object
|
137
|
+
ProcessingState.initialize_from_models(models)
|
138
|
+
end
|
139
|
+
|
140
|
+
ProcessingState = Struct.new(:model_queue, :batch_count) do
|
141
|
+
def self.initialize_from_models(models)
|
142
|
+
new(
|
143
|
+
models.map do |model|
|
144
|
+
ModelProcessingState.initialize_from_model(model)
|
145
|
+
end,
|
146
|
+
0
|
147
|
+
)
|
148
|
+
end
|
149
|
+
|
150
|
+
def next_model
|
151
|
+
model_queue.detect { |i| !i.fully_processed? }
|
152
|
+
end
|
153
|
+
|
154
|
+
def fully_processed?
|
155
|
+
model_queue.all?(&:fully_processed?)
|
156
|
+
end
|
157
|
+
|
158
|
+
def count
|
159
|
+
model_queue.sum { |m| m.field_queue.sum { |f| f.count - f.offset } }
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
ModelProcessingState = Struct.new(:class_name, :field_queue) do
|
164
|
+
def self.initialize_from_model(model)
|
165
|
+
new(
|
166
|
+
model.name,
|
167
|
+
model.tessa_fields.map do |name, _|
|
168
|
+
FieldProcessingState.initialize_from_model(model, name)
|
169
|
+
end
|
170
|
+
)
|
171
|
+
end
|
172
|
+
|
173
|
+
def next_field
|
174
|
+
field_queue.detect { |i| !i.fully_processed? }
|
175
|
+
end
|
176
|
+
|
177
|
+
def model
|
178
|
+
@model ||= class_name.constantize
|
179
|
+
end
|
180
|
+
|
181
|
+
def fully_processed?
|
182
|
+
field_queue.all?(&:fully_processed?)
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
FieldProcessingState = Struct.new(:class_name, :field_name, :offset, :success_count) do
|
187
|
+
def self.initialize_from_model(model, field_name)
|
188
|
+
new(
|
189
|
+
model.name,
|
190
|
+
field_name,
|
191
|
+
0,
|
192
|
+
0
|
193
|
+
)
|
194
|
+
end
|
195
|
+
|
196
|
+
def model
|
197
|
+
@model ||= class_name.constantize
|
198
|
+
end
|
199
|
+
|
200
|
+
def tessa_field
|
201
|
+
model.tessa_fields[field_name]
|
202
|
+
end
|
203
|
+
|
204
|
+
def query
|
205
|
+
model.where.not(Hash[tessa_field.id_field, nil])
|
206
|
+
end
|
207
|
+
|
208
|
+
def count
|
209
|
+
query.count
|
210
|
+
end
|
211
|
+
|
212
|
+
def fully_processed?
|
213
|
+
offset >= count
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
@@ -62,7 +62,8 @@ class Tessa::DynamicExtensions
|
|
62
62
|
|
63
63
|
def attributes
|
64
64
|
super.merge({
|
65
|
-
'#{field.id_field}' => #{field.id_field}
|
65
|
+
'#{field.id_field}' => #{field.id_field},
|
66
|
+
'_tessa_#{field.id_field}' => super['#{field.id_field}']
|
66
67
|
})
|
67
68
|
end
|
68
69
|
CODE
|
@@ -105,7 +106,7 @@ class Tessa::DynamicExtensions
|
|
105
106
|
else
|
106
107
|
ids = self.#{field.id_field}
|
107
108
|
ids.delete(change.id.to_i)
|
108
|
-
self.#{field.id_field} = ids
|
109
|
+
self.#{field.id_field} = ids.any? ? ids : nil
|
109
110
|
end
|
110
111
|
end
|
111
112
|
attachables.changes.select(&:add?).each do |change|
|
@@ -115,14 +116,17 @@ class Tessa::DynamicExtensions
|
|
115
116
|
end
|
116
117
|
when nil
|
117
118
|
a.detach
|
119
|
+
self.#{field.id_field} = nil
|
118
120
|
else
|
119
121
|
a.attach(*attachables)
|
122
|
+
self.#{field.id_field} = nil
|
120
123
|
end
|
121
124
|
end
|
122
125
|
|
123
126
|
def attributes
|
124
127
|
super.merge({
|
125
|
-
'#{field.id_field}' => #{field.id_field}
|
128
|
+
'#{field.id_field}' => #{field.id_field},
|
129
|
+
'_tessa_#{field.id_field}' => super['#{field.id_field}']
|
126
130
|
})
|
127
131
|
end
|
128
132
|
CODE
|
data/lib/tessa/model.rb
CHANGED
@@ -10,6 +10,8 @@ module Tessa
|
|
10
10
|
base.extend ClassMethods
|
11
11
|
base.after_commit :apply_tessa_change_sets if base.respond_to?(:after_commit)
|
12
12
|
base.before_destroy :remove_all_tessa_assets if base.respond_to?(:before_destroy)
|
13
|
+
|
14
|
+
Tessa.model_registry << base
|
13
15
|
end
|
14
16
|
|
15
17
|
module InstanceMethods
|
data/lib/tessa/version.rb
CHANGED
data/lib/tessa.rb
CHANGED
@@ -15,6 +15,10 @@ require "tessa/rack_upload_proxy"
|
|
15
15
|
require "tessa/upload"
|
16
16
|
require "tessa/view_helpers"
|
17
17
|
|
18
|
+
if defined?(ActiveJob)
|
19
|
+
require "tessa/jobs/migrate_assets_job"
|
20
|
+
end
|
21
|
+
|
18
22
|
module Tessa
|
19
23
|
class << self
|
20
24
|
def config
|
@@ -31,6 +35,10 @@ module Tessa
|
|
31
35
|
return find_asset(ids)
|
32
36
|
end
|
33
37
|
|
38
|
+
def model_registry
|
39
|
+
@model_registry ||= []
|
40
|
+
end
|
41
|
+
|
34
42
|
private
|
35
43
|
|
36
44
|
def find_asset(id)
|
data/spec/rails_helper.rb
CHANGED
@@ -5,10 +5,14 @@ require 'spec_helper'
|
|
5
5
|
|
6
6
|
require File.expand_path('dummy/config/environment.rb', __dir__)
|
7
7
|
|
8
|
+
require 'rspec/rails'
|
9
|
+
|
8
10
|
RSpec.configure do |config|
|
9
11
|
ActiveStorage::Current.host = 'https://www.example.com'
|
10
12
|
Rails.application.routes.default_url_options = {
|
11
13
|
protocol: 'https',
|
12
14
|
host: "www.example.com"
|
13
15
|
}
|
16
|
+
|
17
|
+
config.use_transactional_fixtures = true
|
14
18
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'webmock/rspec'
|
1
2
|
require 'tessa'
|
2
3
|
require 'tempfile'
|
3
4
|
|
@@ -11,6 +12,8 @@ if ENV['SIMPLE_COV'] || ENV['CC_TEST_REPORTER_ID']
|
|
11
12
|
end
|
12
13
|
|
13
14
|
RSpec.configure do |config|
|
15
|
+
WebMock.disable_net_connect!
|
16
|
+
|
14
17
|
config.expect_with :rspec do |expectations|
|
15
18
|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
16
19
|
end
|
data/spec/tessa/config_spec.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe Tessa::Config do
|
4
|
-
|
4
|
+
let(:cfg) { Tessa::Config.new }
|
5
5
|
|
6
6
|
shared_examples_for "defaults to environment variable" do
|
7
7
|
around { |ex| swap_environment_var(variable_name, 'from-env') { ex.run } }
|
@@ -19,63 +19,63 @@ RSpec.describe Tessa::Config do
|
|
19
19
|
describe "#username" do
|
20
20
|
it_behaves_like "defaults to environment variable" do
|
21
21
|
let(:variable_name) { 'TESSA_USERNAME' }
|
22
|
-
subject
|
22
|
+
subject { cfg.username }
|
23
23
|
end
|
24
24
|
|
25
25
|
it "behaves like a normal accessor" do
|
26
|
-
|
27
|
-
expect(
|
26
|
+
cfg.username = "my-new-value"
|
27
|
+
expect(cfg.username).to eq("my-new-value")
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
31
|
describe "#password" do
|
32
32
|
it_behaves_like "defaults to environment variable" do
|
33
33
|
let(:variable_name) { 'TESSA_PASSWORD' }
|
34
|
-
subject
|
34
|
+
subject { cfg.password }
|
35
35
|
end
|
36
36
|
|
37
37
|
it "behaves like a normal accessor" do
|
38
|
-
|
39
|
-
expect(
|
38
|
+
cfg.password = "my-new-value"
|
39
|
+
expect(cfg.password).to eq("my-new-value")
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
43
|
describe "#url" do
|
44
44
|
it_behaves_like "defaults to environment variable" do
|
45
45
|
let(:variable_name) { 'TESSA_URL' }
|
46
|
-
subject
|
46
|
+
subject { cfg.url }
|
47
47
|
end
|
48
48
|
|
49
49
|
it "behaves like a normal accessor" do
|
50
|
-
|
51
|
-
expect(
|
50
|
+
cfg.url = "my-new-value"
|
51
|
+
expect(cfg.url).to eq("my-new-value")
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
55
|
describe "#strategy" do
|
56
56
|
it_behaves_like "defaults to environment variable" do
|
57
57
|
let(:variable_name) { 'TESSA_STRATEGY' }
|
58
|
-
subject
|
58
|
+
subject { cfg.strategy }
|
59
59
|
end
|
60
60
|
|
61
61
|
it "uses the string 'default' when no envvar passed" do
|
62
|
-
expect(
|
62
|
+
expect(cfg.strategy).to eq("default")
|
63
63
|
end
|
64
64
|
|
65
65
|
it "behaves like a normal accessor" do
|
66
|
-
|
67
|
-
expect(
|
66
|
+
cfg.strategy = "my-new-value"
|
67
|
+
expect(cfg.strategy).to eq("my-new-value")
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
71
|
describe "#connection" do
|
72
72
|
it "is a Faraday::Connection" do
|
73
|
-
expect(
|
73
|
+
expect(cfg.connection).to be_a(Faraday::Connection)
|
74
74
|
end
|
75
75
|
|
76
|
-
context "with values
|
77
|
-
subject
|
78
|
-
before { args.each { |k, v|
|
76
|
+
context "with values cfgured" do
|
77
|
+
subject { cfg.connection }
|
78
|
+
before { args.each { |k, v| cfg.send("#{k}=", v) } }
|
79
79
|
let(:args) {
|
80
80
|
{
|
81
81
|
url: "http://tessa.test",
|
@@ -85,28 +85,28 @@ RSpec.describe Tessa::Config do
|
|
85
85
|
}
|
86
86
|
|
87
87
|
it "sets faraday's url prefix to our url" do
|
88
|
-
expect(
|
88
|
+
expect(subject.url_prefix.to_s).to match(cfg.url)
|
89
89
|
end
|
90
90
|
|
91
91
|
context "with faraday spy" do
|
92
92
|
let(:spy) { instance_spy(Faraday::Connection) }
|
93
93
|
before do
|
94
94
|
expect(Faraday).to receive(:new).and_yield(spy)
|
95
|
-
|
95
|
+
subject
|
96
96
|
end
|
97
97
|
|
98
98
|
it "sets up url_encoded request handler" do
|
99
99
|
expect(spy).to have_received(:request).with(:url_encoded)
|
100
100
|
end
|
101
101
|
|
102
|
-
it "
|
102
|
+
it "cfgures the default adapter" do
|
103
103
|
expect(spy).to have_received(:adapter).with(:net_http)
|
104
104
|
end
|
105
105
|
end
|
106
106
|
end
|
107
107
|
|
108
108
|
it "caches the result" do
|
109
|
-
expect(
|
109
|
+
expect(cfg.connection.object_id).to eq(cfg.connection.object_id)
|
110
110
|
end
|
111
111
|
end
|
112
112
|
end
|
@@ -0,0 +1,247 @@
|
|
1
|
+
require 'rails_helper'
|
2
|
+
|
3
|
+
require 'tessa/jobs/migrate_assets_job'
|
4
|
+
|
5
|
+
RSpec.describe Tessa::MigrateAssetsJob do
|
6
|
+
it 'does nothing if no db rows' do
|
7
|
+
|
8
|
+
expect {
|
9
|
+
subject.perform
|
10
|
+
}.to_not change { ActiveStorage::Attachment.count }
|
11
|
+
end
|
12
|
+
|
13
|
+
it 'creates attachments for old tessa assets' do
|
14
|
+
allow(Tessa::Asset).to receive(:find)
|
15
|
+
.with(1)
|
16
|
+
.and_return(
|
17
|
+
Tessa::Asset.new(id: 1,
|
18
|
+
meta: { name: 'README.md' },
|
19
|
+
private_download_url: 'https://test.com/README.md')
|
20
|
+
)
|
21
|
+
|
22
|
+
allow(Tessa::Asset).to receive(:find)
|
23
|
+
.with(2)
|
24
|
+
.and_return(
|
25
|
+
Tessa::Asset.new(id: 2,
|
26
|
+
meta: { name: 'LICENSE.txt' },
|
27
|
+
private_download_url: 'https://test.com/LICENSE.txt'),
|
28
|
+
)
|
29
|
+
|
30
|
+
stub_request(:get, 'https://test.com/README.md')
|
31
|
+
.to_return(body: File.new('README.md'))
|
32
|
+
stub_request(:get, 'https://test.com/LICENSE.txt')
|
33
|
+
.to_return(body: File.new('LICENSE.txt'))
|
34
|
+
|
35
|
+
# DB models with those
|
36
|
+
models = [
|
37
|
+
SingleAssetModel.create!(avatar_id: 1),
|
38
|
+
SingleAssetModel.create!(avatar_id: 2)
|
39
|
+
]
|
40
|
+
|
41
|
+
expect {
|
42
|
+
subject.perform
|
43
|
+
}.to change { ActiveStorage::Attachment.count }.by(2)
|
44
|
+
|
45
|
+
models.each do |m|
|
46
|
+
expect(m.reload.avatar_id).to eq(m.avatar.key)
|
47
|
+
# Now it's in activestorage
|
48
|
+
expect(m.avatar.public_url).to start_with(
|
49
|
+
'https://www.example.com/rails/active_storage/blobs/')
|
50
|
+
end
|
51
|
+
|
52
|
+
expect(SingleAssetModel.where.not('avatar_id' => nil).count)
|
53
|
+
.to eq(0)
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'preserves ActiveStorage blobs' do
|
57
|
+
allow(Tessa::Asset).to receive(:find)
|
58
|
+
.with([1, 2])
|
59
|
+
.and_return([
|
60
|
+
Tessa::Asset.new(id: 1,
|
61
|
+
meta: { name: 'README.md' },
|
62
|
+
private_download_url: 'https://test.com/README.md'),
|
63
|
+
Tessa::Asset.new(id: 2,
|
64
|
+
meta: { name: 'LICENSE.txt' },
|
65
|
+
private_download_url: 'https://test.com/LICENSE.txt'),
|
66
|
+
])
|
67
|
+
stub_request(:get, 'https://test.com/README.md')
|
68
|
+
.to_return(body: File.new('README.md'))
|
69
|
+
stub_request(:get, 'https://test.com/LICENSE.txt')
|
70
|
+
.to_return(body: File.new('LICENSE.txt'))
|
71
|
+
|
72
|
+
file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
|
73
|
+
|
74
|
+
model = MultipleAssetModel.create!(
|
75
|
+
# The Tessa DB column has the one asset
|
76
|
+
another_place: [1, 2]
|
77
|
+
)
|
78
|
+
# But has already attached a second ActiveStorage blob
|
79
|
+
::ActiveStorage::Attached::Many.new("multiple_field", model, dependent: :purge_later)
|
80
|
+
.attach(file2)
|
81
|
+
model.save!
|
82
|
+
attachment = model.multiple_field_attachments.first
|
83
|
+
|
84
|
+
expect {
|
85
|
+
subject.perform
|
86
|
+
}.to change { ActiveStorage::Attachment.count }.by(2)
|
87
|
+
|
88
|
+
model = model.reload
|
89
|
+
# The IDs are now the keys of ActiveStorage objects
|
90
|
+
expect(model.another_place).to eq(
|
91
|
+
model.multiple_field.map(&:key))
|
92
|
+
# preserves the existing attachment
|
93
|
+
expect(model.multiple_field_attachments).to include(attachment)
|
94
|
+
expect(model.another_place).to include(attachment.key)
|
95
|
+
|
96
|
+
# all assets are in activestorage
|
97
|
+
model.multiple_field.each do |blob|
|
98
|
+
expect(blob.public_url).to start_with(
|
99
|
+
'https://www.example.com/rails/active_storage/blobs/')
|
100
|
+
end
|
101
|
+
|
102
|
+
# DB column is reset to nil
|
103
|
+
expect(MultipleAssetModel.where.not('another_place' => nil).count)
|
104
|
+
.to eq(0)
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'Stops after hitting the batch size' do
|
108
|
+
1.upto(11).each do |i|
|
109
|
+
asset = Tessa::Asset.new(id: i,
|
110
|
+
meta: { name: 'README.md' },
|
111
|
+
private_download_url: 'https://test.com/README.md')
|
112
|
+
allow(Tessa::Asset).to receive(:find)
|
113
|
+
.with(i)
|
114
|
+
.and_return(asset)
|
115
|
+
allow(Tessa::Asset).to receive(:find)
|
116
|
+
.with([i])
|
117
|
+
.and_return([asset])
|
118
|
+
end
|
119
|
+
stub_request(:get, 'https://test.com/README.md')
|
120
|
+
.to_return(body: File.new('README.md'))
|
121
|
+
|
122
|
+
# Mix of the two models...
|
123
|
+
models =
|
124
|
+
1.upto(11).map do |i|
|
125
|
+
if i % 2 == 0
|
126
|
+
MultipleAssetModel.create!(another_place: [i])
|
127
|
+
else
|
128
|
+
SingleAssetModel.create!(avatar_id: i)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
final_state = nil
|
133
|
+
final_options = nil
|
134
|
+
dbl = double('set')
|
135
|
+
expect(dbl).to receive(:perform_later) do |state, options|
|
136
|
+
final_state = Marshal.load(state)
|
137
|
+
final_options = options
|
138
|
+
end
|
139
|
+
expect(Tessa::MigrateAssetsJob).to receive(:set)
|
140
|
+
.with(wait: 10.minutes)
|
141
|
+
.and_return(dbl)
|
142
|
+
|
143
|
+
expect {
|
144
|
+
subject.perform
|
145
|
+
}.to change { ActiveStorage::Attachment.count }.by(10)
|
146
|
+
|
147
|
+
expect(final_state.fully_processed?).to be false
|
148
|
+
expect(final_options).to eq({
|
149
|
+
batch_size: 10,
|
150
|
+
interval: 10.minutes.to_i
|
151
|
+
})
|
152
|
+
# One of the two models was fully processed
|
153
|
+
expect(final_state.model_queue.count(&:fully_processed?))
|
154
|
+
.to eq(1)
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'Skips over Tessa errors' do
|
158
|
+
1.upto(11).each do |i|
|
159
|
+
if i % 2 == 0
|
160
|
+
allow(Tessa::Asset).to receive(:find)
|
161
|
+
.with(i)
|
162
|
+
.and_raise(Tessa::RequestFailed)
|
163
|
+
next
|
164
|
+
end
|
165
|
+
|
166
|
+
allow(Tessa::Asset).to receive(:find)
|
167
|
+
.with(i)
|
168
|
+
.and_return(
|
169
|
+
Tessa::Asset.new(id: i,
|
170
|
+
meta: { name: 'README.md' },
|
171
|
+
private_download_url: 'https://test.com/README.md')
|
172
|
+
)
|
173
|
+
end
|
174
|
+
stub_request(:get, 'https://test.com/README.md')
|
175
|
+
.to_return(body: File.new('README.md'))
|
176
|
+
|
177
|
+
# Mix of the two models...
|
178
|
+
models =
|
179
|
+
1.upto(11).map do |i|
|
180
|
+
SingleAssetModel.create!(avatar_id: i)
|
181
|
+
end
|
182
|
+
|
183
|
+
final_state = nil
|
184
|
+
final_options = nil
|
185
|
+
dbl = double('set')
|
186
|
+
expect(dbl).to receive(:perform_later) do |state, options|
|
187
|
+
final_state = Marshal.load(state)
|
188
|
+
final_options = options
|
189
|
+
end
|
190
|
+
expect(Tessa::MigrateAssetsJob).to receive(:set)
|
191
|
+
.with(wait: 10.minutes)
|
192
|
+
.and_return(dbl)
|
193
|
+
|
194
|
+
expect {
|
195
|
+
subject.perform
|
196
|
+
}.to change { ActiveStorage::Attachment.count }.by(5)
|
197
|
+
|
198
|
+
expect(final_state.fully_processed?).to be false
|
199
|
+
field_state = final_state.next_model.next_field
|
200
|
+
expect(field_state.offset).to eq(5)
|
201
|
+
end
|
202
|
+
|
203
|
+
it 'Resumes from marshalled state' do
|
204
|
+
|
205
|
+
file = Rack::Test::UploadedFile.new('README.md')
|
206
|
+
|
207
|
+
state = Tessa::MigrateAssetsJob::ProcessingState.initialize_from_models(
|
208
|
+
[SingleAssetModel])
|
209
|
+
field_state = state.model_queue
|
210
|
+
.detect { |m| m.class_name == 'SingleAssetModel' }
|
211
|
+
.field_queue
|
212
|
+
.detect { |m| m.field_name == :avatar }
|
213
|
+
|
214
|
+
1.upto(10).each do |i|
|
215
|
+
if i % 2 == 0
|
216
|
+
# This one failed
|
217
|
+
SingleAssetModel.create!(avatar_id: i).tap do |r|
|
218
|
+
field_state.offset += 1
|
219
|
+
end
|
220
|
+
else
|
221
|
+
# This one succeeded and is in ActiveStorage
|
222
|
+
SingleAssetModel.create!(avatar: file).tap do |r|
|
223
|
+
field_state.success_count += 1
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
# This one still needs to transition
|
229
|
+
model = SingleAssetModel.create!(avatar_id: 11)
|
230
|
+
|
231
|
+
asset = Tessa::Asset.new(id: 11,
|
232
|
+
meta: { name: 'README.md' },
|
233
|
+
private_download_url: 'https://test.com/README.md')
|
234
|
+
allow(Tessa::Asset).to receive(:find)
|
235
|
+
.with(11)
|
236
|
+
.and_return(asset)
|
237
|
+
stub_request(:get, 'https://test.com/README.md')
|
238
|
+
.to_return(body: File.new('README.md'))
|
239
|
+
|
240
|
+
# Doesn't reenqueue since we finished processing
|
241
|
+
expect(Tessa::MigrateAssetsJob).to_not receive(:set)
|
242
|
+
|
243
|
+
expect {
|
244
|
+
subject.perform(Marshal.dump(state), { batch_size: 2, interval: 3 })
|
245
|
+
}.to change { ActiveStorage::Attachment.count }.by(1)
|
246
|
+
end
|
247
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tessa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Justin Powell
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-06-
|
12
|
+
date: 2022-06-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: faraday
|
@@ -151,6 +151,7 @@ files:
|
|
151
151
|
- app/javascript/tessa/index.js.coffee
|
152
152
|
- bin/rspec
|
153
153
|
- config/routes.rb
|
154
|
+
- lib/tasks/tessa.rake
|
154
155
|
- lib/tessa.rb
|
155
156
|
- lib/tessa/active_storage/asset_wrapper.rb
|
156
157
|
- lib/tessa/asset.rb
|
@@ -160,6 +161,7 @@ files:
|
|
160
161
|
- lib/tessa/config.rb
|
161
162
|
- lib/tessa/controller_helpers.rb
|
162
163
|
- lib/tessa/engine.rb
|
164
|
+
- lib/tessa/jobs/migrate_assets_job.rb
|
163
165
|
- lib/tessa/model.rb
|
164
166
|
- lib/tessa/model/dynamic_extensions.rb
|
165
167
|
- lib/tessa/model/field.rb
|
@@ -240,6 +242,7 @@ files:
|
|
240
242
|
- spec/tessa/asset_spec.rb
|
241
243
|
- spec/tessa/config_spec.rb
|
242
244
|
- spec/tessa/controller_helpers_spec.rb
|
245
|
+
- spec/tessa/jobs/migrate_assets_job_spec.rb
|
243
246
|
- spec/tessa/model_field_spec.rb
|
244
247
|
- spec/tessa/model_spec.rb
|
245
248
|
- spec/tessa/rack_upload_proxy_spec.rb
|
@@ -340,6 +343,7 @@ test_files:
|
|
340
343
|
- spec/tessa/asset_spec.rb
|
341
344
|
- spec/tessa/config_spec.rb
|
342
345
|
- spec/tessa/controller_helpers_spec.rb
|
346
|
+
- spec/tessa/jobs/migrate_assets_job_spec.rb
|
343
347
|
- spec/tessa/model_field_spec.rb
|
344
348
|
- spec/tessa/model_spec.rb
|
345
349
|
- spec/tessa/rack_upload_proxy_spec.rb
|