tessa 1.0.2 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9f9b764a91daef07d23dcc4adf14060ec1546e3f6cdff0b548b10c4caeb56598
4
- data.tar.gz: e40a45f82d40fb0e3051dc67ffd263541cc2afec0a836912b8dfadbbd9bddf16
3
+ metadata.gz: d31c66f27d8c731d0df1e58f196bd6279f4f2603ab39e33f75f8cf46630745fe
4
+ data.tar.gz: fe6050f938245f6502771adff53f13bd3af40a0339a1098d96bbd2416d6e0a2b
5
5
  SHA512:
6
- metadata.gz: 7de9f6c50a229e8cd628fc09d6fcb4bdbb83b0bd7366c12b547870ce245b936026bcabc28412ffe1bcbca929300e3c0693012b439621063da3f755db1716f71d
7
- data.tar.gz: ec6c2da67ac86e579541bdc485351032718c475077e98fa046cc5cd8959eb728c6a5da915522b73fe7031360c9eb57a0612a27ee39ec467dca619be4e3ea0188
6
+ metadata.gz: 76047a38e882dd59fe104b8be03a2970d996e70eb5f469549d8529e384f46732f5f76dad54e0d7c3d158c60de032b01c467ed13202209d0c4fdad4faa0328b84
7
+ data.tar.gz: '085ac99b9dcce059741887f7852d4a2fd08412b35fa5e520ac61c79f49d5511f43485fe6836863910d07f9336d0c295b31b1bb691b51abc3ea00cefefac35a0b'
data/Gemfile CHANGED
@@ -7,3 +7,8 @@ group :development do
7
7
  gem 'pry'
8
8
  gem 'dotenv'
9
9
  end
10
+
11
+ group :test do
12
+ gem 'rspec-rails'
13
+ gem 'webmock'
14
+ end
@@ -0,0 +1,7 @@
1
+
2
+ namespace :tessa do
3
+ desc "Begins the migration of all Tessa assets to ActiveStorage."
4
+ task :migrate => :environment do
5
+ Tessa::MigrateAssetsJob.perform_later
6
+ end
7
+ end
@@ -20,7 +20,8 @@ module Tessa::ActiveStorage
20
20
  def meta
21
21
  {
22
22
  mime_type: content_type,
23
- size: byte_size
23
+ size: byte_size,
24
+ name: filename
24
25
  }
25
26
  end
26
27
 
@@ -0,0 +1,216 @@
1
+ require 'open-uri'
2
+
3
+ class Tessa::MigrateAssetsJob < ActiveJob::Base
4
+
5
+ queue_as :low
6
+
7
+ def perform(*args)
8
+ options = args&.extract_options!
9
+ options = {
10
+ batch_size: 10,
11
+ interval: 10.minutes.to_i
12
+ }.merge!(options&.symbolize_keys || {})
13
+
14
+ interval = options[:interval].seconds
15
+ processing_state = args.first ? Marshal.load(args.first) : load_models_from_registry
16
+
17
+ if processing_state.fully_processed?
18
+ Rails.logger.info("Nothing to do - all models have transitioned to ActiveStorage")
19
+ return
20
+ end
21
+
22
+ processing_state.batch_count = 0
23
+ while processing_state.batch_count < options[:batch_size]
24
+ model_state = processing_state.next_model
25
+
26
+ process(processing_state, model_state, options)
27
+
28
+ break if processing_state.fully_processed?
29
+ end
30
+
31
+ if processing_state.fully_processed?
32
+ Rails.logger.info("Finished processing all Tessa assets")
33
+ else
34
+ remaining_batches = (processing_state.count / options[:batch_size].to_f).ceil
35
+
36
+ Rails.logger.info("Continuing processing in #{interval}, "\
37
+ "ETA #{(remaining_batches * interval).from_now}. "\
38
+ "Working on #{processing_state.next_model.next_field}")
39
+
40
+ processing_state.batch_count = 0
41
+ self.class.set(wait: interval)
42
+ .perform_later(Marshal.dump(processing_state), options)
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def process(processing_state, model_state, options)
49
+ while processing_state.batch_count < options[:batch_size]
50
+ field_state = model_state.next_field
51
+
52
+ process_field(processing_state, field_state, options)
53
+
54
+ return if model_state.fully_processed?
55
+ end
56
+ end
57
+
58
+ def process_field(processing_state, field_state, options)
59
+ while processing_state.batch_count < options[:batch_size]
60
+ remaining = options[:batch_size] - processing_state.batch_count
61
+
62
+ next_batch = field_state.query
63
+ .offset(field_state.offset)
64
+ .limit(remaining)
65
+
66
+ next_batch.each_with_index do |record, idx|
67
+ begin
68
+ # Wait 1 second in between records to slow things down and let the
69
+ # system process for a bit
70
+ sleep 1 if idx > 0
71
+
72
+ reupload(record, field_state)
73
+ Rails.logger.info("#{record.class}#{record.id}##{field_state.field_name}: success")
74
+ field_state.success_count += 1
75
+ rescue StandardError => ex
76
+ Rails.logger.error("#{record.class}#{record.id}##{field_state.field_name}: error - #{ex}")
77
+ field_state.offset += 1
78
+ ensure
79
+ processing_state.batch_count += 1
80
+ end
81
+ end
82
+
83
+ return if field_state.fully_processed?
84
+ end
85
+ end
86
+
87
+ def reupload(record, field_state)
88
+ if field_state.tessa_field.multiple?
89
+ reupload_multiple(record, field_state)
90
+ else
91
+ reupload_single(record, field_state)
92
+ end
93
+ end
94
+
95
+ def reupload_single(record, field_state)
96
+ # models with ActiveStorage uploads have nil in the column, but if you call
97
+ # the method it hits the dynamic extensions and gives you the blob key
98
+ database_id = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
99
+ return unless database_id
100
+
101
+ asset = Tessa::Asset.find(database_id)
102
+
103
+ attachable = {
104
+ io: URI.open(asset.private_download_url),
105
+ filename: asset.meta[:name]
106
+ }
107
+
108
+ record.public_send("#{field_state.tessa_field.name}=", attachable)
109
+ record.save!
110
+ end
111
+
112
+ def reupload_multiple(record, field_state)
113
+ database_ids = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
114
+ return unless database_ids
115
+
116
+ assets = Tessa::Asset.find(database_ids)
117
+
118
+ attachables = assets.map do |asset|
119
+ {
120
+ io: URI.open(asset.private_download_url),
121
+ filename: asset.meta[:name]
122
+ }
123
+ end
124
+
125
+ record.public_send("#{field_state.tessa_field.name}=", attachables)
126
+ record.save!
127
+ end
128
+
129
+ def load_models_from_registry
130
+ Rails.application.eager_load!
131
+
132
+ # Load all Tessa models that can have attachments (not form objects)
133
+ models = Tessa.model_registry
134
+ .select { |m| m.respond_to?(:has_one_attached) }
135
+
136
+ # Initialize our Record Keeping object
137
+ ProcessingState.initialize_from_models(models)
138
+ end
139
+
140
+ ProcessingState = Struct.new(:model_queue, :batch_count) do
141
+ def self.initialize_from_models(models)
142
+ new(
143
+ models.map do |model|
144
+ ModelProcessingState.initialize_from_model(model)
145
+ end,
146
+ 0
147
+ )
148
+ end
149
+
150
+ def next_model
151
+ model_queue.detect { |i| !i.fully_processed? }
152
+ end
153
+
154
+ def fully_processed?
155
+ model_queue.all?(&:fully_processed?)
156
+ end
157
+
158
+ def count
159
+ model_queue.sum { |m| m.field_queue.sum { |f| f.count - f.offset } }
160
+ end
161
+ end
162
+
163
+ ModelProcessingState = Struct.new(:class_name, :field_queue) do
164
+ def self.initialize_from_model(model)
165
+ new(
166
+ model.name,
167
+ model.tessa_fields.map do |name, _|
168
+ FieldProcessingState.initialize_from_model(model, name)
169
+ end
170
+ )
171
+ end
172
+
173
+ def next_field
174
+ field_queue.detect { |i| !i.fully_processed? }
175
+ end
176
+
177
+ def model
178
+ @model ||= class_name.constantize
179
+ end
180
+
181
+ def fully_processed?
182
+ field_queue.all?(&:fully_processed?)
183
+ end
184
+ end
185
+
186
+ FieldProcessingState = Struct.new(:class_name, :field_name, :offset, :success_count) do
187
+ def self.initialize_from_model(model, field_name)
188
+ new(
189
+ model.name,
190
+ field_name,
191
+ 0,
192
+ 0
193
+ )
194
+ end
195
+
196
+ def model
197
+ @model ||= class_name.constantize
198
+ end
199
+
200
+ def tessa_field
201
+ model.tessa_fields[field_name]
202
+ end
203
+
204
+ def query
205
+ model.where.not(Hash[tessa_field.id_field, nil])
206
+ end
207
+
208
+ def count
209
+ query.count
210
+ end
211
+
212
+ def fully_processed?
213
+ offset >= count
214
+ end
215
+ end
216
+ end
@@ -62,7 +62,8 @@ class Tessa::DynamicExtensions
62
62
 
63
63
  def attributes
64
64
  super.merge({
65
- '#{field.id_field}' => #{field.id_field}
65
+ '#{field.id_field}' => #{field.id_field},
66
+ '_tessa_#{field.id_field}' => super['#{field.id_field}']
66
67
  })
67
68
  end
68
69
  CODE
@@ -105,7 +106,7 @@ class Tessa::DynamicExtensions
105
106
  else
106
107
  ids = self.#{field.id_field}
107
108
  ids.delete(change.id.to_i)
108
- self.#{field.id_field} = ids
109
+ self.#{field.id_field} = ids.any? ? ids : nil
109
110
  end
110
111
  end
111
112
  attachables.changes.select(&:add?).each do |change|
@@ -115,14 +116,17 @@ class Tessa::DynamicExtensions
115
116
  end
116
117
  when nil
117
118
  a.detach
119
+ self.#{field.id_field} = nil
118
120
  else
119
121
  a.attach(*attachables)
122
+ self.#{field.id_field} = nil
120
123
  end
121
124
  end
122
125
 
123
126
  def attributes
124
127
  super.merge({
125
- '#{field.id_field}' => #{field.id_field}
128
+ '#{field.id_field}' => #{field.id_field},
129
+ '_tessa_#{field.id_field}' => super['#{field.id_field}']
126
130
  })
127
131
  end
128
132
  CODE
data/lib/tessa/model.rb CHANGED
@@ -10,6 +10,8 @@ module Tessa
10
10
  base.extend ClassMethods
11
11
  base.after_commit :apply_tessa_change_sets if base.respond_to?(:after_commit)
12
12
  base.before_destroy :remove_all_tessa_assets if base.respond_to?(:before_destroy)
13
+
14
+ Tessa.model_registry << base
13
15
  end
14
16
 
15
17
  module InstanceMethods
data/lib/tessa/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Tessa
2
- VERSION = "1.0.2"
2
+ VERSION = "1.1.0"
3
3
  end
data/lib/tessa.rb CHANGED
@@ -15,6 +15,10 @@ require "tessa/rack_upload_proxy"
15
15
  require "tessa/upload"
16
16
  require "tessa/view_helpers"
17
17
 
18
+ if defined?(ActiveJob)
19
+ require "tessa/jobs/migrate_assets_job"
20
+ end
21
+
18
22
  module Tessa
19
23
  class << self
20
24
  def config
@@ -31,6 +35,10 @@ module Tessa
31
35
  return find_asset(ids)
32
36
  end
33
37
 
38
+ def model_registry
39
+ @model_registry ||= []
40
+ end
41
+
34
42
  private
35
43
 
36
44
  def find_asset(id)
data/spec/rails_helper.rb CHANGED
@@ -5,10 +5,14 @@ require 'spec_helper'
5
5
 
6
6
  require File.expand_path('dummy/config/environment.rb', __dir__)
7
7
 
8
+ require 'rspec/rails'
9
+
8
10
  RSpec.configure do |config|
9
11
  ActiveStorage::Current.host = 'https://www.example.com'
10
12
  Rails.application.routes.default_url_options = {
11
13
  protocol: 'https',
12
14
  host: "www.example.com"
13
15
  }
16
+
17
+ config.use_transactional_fixtures = true
14
18
  end
data/spec/spec_helper.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require 'webmock/rspec'
1
2
  require 'tessa'
2
3
  require 'tempfile'
3
4
 
@@ -11,6 +12,8 @@ if ENV['SIMPLE_COV'] || ENV['CC_TEST_REPORTER_ID']
11
12
  end
12
13
 
13
14
  RSpec.configure do |config|
15
+ WebMock.disable_net_connect!
16
+
14
17
  config.expect_with :rspec do |expectations|
15
18
  expectations.include_chain_clauses_in_custom_matcher_descriptions = true
16
19
  end
@@ -1,7 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  RSpec.describe Tessa::Config do
4
- subject(:config) { Tessa::Config.new }
4
+ let(:cfg) { Tessa::Config.new }
5
5
 
6
6
  shared_examples_for "defaults to environment variable" do
7
7
  around { |ex| swap_environment_var(variable_name, 'from-env') { ex.run } }
@@ -19,63 +19,63 @@ RSpec.describe Tessa::Config do
19
19
  describe "#username" do
20
20
  it_behaves_like "defaults to environment variable" do
21
21
  let(:variable_name) { 'TESSA_USERNAME' }
22
- subject(:username) { config.username }
22
+ subject { cfg.username }
23
23
  end
24
24
 
25
25
  it "behaves like a normal accessor" do
26
- config.username = "my-new-value"
27
- expect(config.username).to eq("my-new-value")
26
+ cfg.username = "my-new-value"
27
+ expect(cfg.username).to eq("my-new-value")
28
28
  end
29
29
  end
30
30
 
31
31
  describe "#password" do
32
32
  it_behaves_like "defaults to environment variable" do
33
33
  let(:variable_name) { 'TESSA_PASSWORD' }
34
- subject(:password) { config.password }
34
+ subject { cfg.password }
35
35
  end
36
36
 
37
37
  it "behaves like a normal accessor" do
38
- config.password = "my-new-value"
39
- expect(config.password).to eq("my-new-value")
38
+ cfg.password = "my-new-value"
39
+ expect(cfg.password).to eq("my-new-value")
40
40
  end
41
41
  end
42
42
 
43
43
  describe "#url" do
44
44
  it_behaves_like "defaults to environment variable" do
45
45
  let(:variable_name) { 'TESSA_URL' }
46
- subject(:url) { config.url }
46
+ subject { cfg.url }
47
47
  end
48
48
 
49
49
  it "behaves like a normal accessor" do
50
- config.url = "my-new-value"
51
- expect(config.url).to eq("my-new-value")
50
+ cfg.url = "my-new-value"
51
+ expect(cfg.url).to eq("my-new-value")
52
52
  end
53
53
  end
54
54
 
55
55
  describe "#strategy" do
56
56
  it_behaves_like "defaults to environment variable" do
57
57
  let(:variable_name) { 'TESSA_STRATEGY' }
58
- subject(:strategy) { config.strategy }
58
+ subject { cfg.strategy }
59
59
  end
60
60
 
61
61
  it "uses the string 'default' when no envvar passed" do
62
- expect(config.strategy).to eq("default")
62
+ expect(cfg.strategy).to eq("default")
63
63
  end
64
64
 
65
65
  it "behaves like a normal accessor" do
66
- config.strategy = "my-new-value"
67
- expect(config.strategy).to eq("my-new-value")
66
+ cfg.strategy = "my-new-value"
67
+ expect(cfg.strategy).to eq("my-new-value")
68
68
  end
69
69
  end
70
70
 
71
71
  describe "#connection" do
72
72
  it "is a Faraday::Connection" do
73
- expect(config.connection).to be_a(Faraday::Connection)
73
+ expect(cfg.connection).to be_a(Faraday::Connection)
74
74
  end
75
75
 
76
- context "with values configured" do
77
- subject(:connection) { config.connection }
78
- before { args.each { |k, v| config.send("#{k}=", v) } }
76
+ context "with values cfgured" do
77
+ subject { cfg.connection }
78
+ before { args.each { |k, v| cfg.send("#{k}=", v) } }
79
79
  let(:args) {
80
80
  {
81
81
  url: "http://tessa.test",
@@ -85,28 +85,28 @@ RSpec.describe Tessa::Config do
85
85
  }
86
86
 
87
87
  it "sets faraday's url prefix to our url" do
88
- expect(connection.url_prefix.to_s).to match(config.url)
88
+ expect(subject.url_prefix.to_s).to match(cfg.url)
89
89
  end
90
90
 
91
91
  context "with faraday spy" do
92
92
  let(:spy) { instance_spy(Faraday::Connection) }
93
93
  before do
94
94
  expect(Faraday).to receive(:new).and_yield(spy)
95
- connection
95
+ subject
96
96
  end
97
97
 
98
98
  it "sets up url_encoded request handler" do
99
99
  expect(spy).to have_received(:request).with(:url_encoded)
100
100
  end
101
101
 
102
- it "configures the default adapter" do
102
+ it "cfgures the default adapter" do
103
103
  expect(spy).to have_received(:adapter).with(:net_http)
104
104
  end
105
105
  end
106
106
  end
107
107
 
108
108
  it "caches the result" do
109
- expect(config.connection.object_id).to eq(config.connection.object_id)
109
+ expect(cfg.connection.object_id).to eq(cfg.connection.object_id)
110
110
  end
111
111
  end
112
112
  end
@@ -0,0 +1,247 @@
1
+ require 'rails_helper'
2
+
3
+ require 'tessa/jobs/migrate_assets_job'
4
+
5
+ RSpec.describe Tessa::MigrateAssetsJob do
6
+ it 'does nothing if no db rows' do
7
+
8
+ expect {
9
+ subject.perform
10
+ }.to_not change { ActiveStorage::Attachment.count }
11
+ end
12
+
13
+ it 'creates attachments for old tessa assets' do
14
+ allow(Tessa::Asset).to receive(:find)
15
+ .with(1)
16
+ .and_return(
17
+ Tessa::Asset.new(id: 1,
18
+ meta: { name: 'README.md' },
19
+ private_download_url: 'https://test.com/README.md')
20
+ )
21
+
22
+ allow(Tessa::Asset).to receive(:find)
23
+ .with(2)
24
+ .and_return(
25
+ Tessa::Asset.new(id: 2,
26
+ meta: { name: 'LICENSE.txt' },
27
+ private_download_url: 'https://test.com/LICENSE.txt'),
28
+ )
29
+
30
+ stub_request(:get, 'https://test.com/README.md')
31
+ .to_return(body: File.new('README.md'))
32
+ stub_request(:get, 'https://test.com/LICENSE.txt')
33
+ .to_return(body: File.new('LICENSE.txt'))
34
+
35
+ # DB models with those
36
+ models = [
37
+ SingleAssetModel.create!(avatar_id: 1),
38
+ SingleAssetModel.create!(avatar_id: 2)
39
+ ]
40
+
41
+ expect {
42
+ subject.perform
43
+ }.to change { ActiveStorage::Attachment.count }.by(2)
44
+
45
+ models.each do |m|
46
+ expect(m.reload.avatar_id).to eq(m.avatar.key)
47
+ # Now it's in activestorage
48
+ expect(m.avatar.public_url).to start_with(
49
+ 'https://www.example.com/rails/active_storage/blobs/')
50
+ end
51
+
52
+ expect(SingleAssetModel.where.not('avatar_id' => nil).count)
53
+ .to eq(0)
54
+ end
55
+
56
+ it 'preserves ActiveStorage blobs' do
57
+ allow(Tessa::Asset).to receive(:find)
58
+ .with([1, 2])
59
+ .and_return([
60
+ Tessa::Asset.new(id: 1,
61
+ meta: { name: 'README.md' },
62
+ private_download_url: 'https://test.com/README.md'),
63
+ Tessa::Asset.new(id: 2,
64
+ meta: { name: 'LICENSE.txt' },
65
+ private_download_url: 'https://test.com/LICENSE.txt'),
66
+ ])
67
+ stub_request(:get, 'https://test.com/README.md')
68
+ .to_return(body: File.new('README.md'))
69
+ stub_request(:get, 'https://test.com/LICENSE.txt')
70
+ .to_return(body: File.new('LICENSE.txt'))
71
+
72
+ file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
73
+
74
+ model = MultipleAssetModel.create!(
75
+ # The Tessa DB column has the one asset
76
+ another_place: [1, 2]
77
+ )
78
+ # But has already attached a second ActiveStorage blob
79
+ ::ActiveStorage::Attached::Many.new("multiple_field", model, dependent: :purge_later)
80
+ .attach(file2)
81
+ model.save!
82
+ attachment = model.multiple_field_attachments.first
83
+
84
+ expect {
85
+ subject.perform
86
+ }.to change { ActiveStorage::Attachment.count }.by(2)
87
+
88
+ model = model.reload
89
+ # The IDs are now the keys of ActiveStorage objects
90
+ expect(model.another_place).to eq(
91
+ model.multiple_field.map(&:key))
92
+ # preserves the existing attachment
93
+ expect(model.multiple_field_attachments).to include(attachment)
94
+ expect(model.another_place).to include(attachment.key)
95
+
96
+ # all assets are in activestorage
97
+ model.multiple_field.each do |blob|
98
+ expect(blob.public_url).to start_with(
99
+ 'https://www.example.com/rails/active_storage/blobs/')
100
+ end
101
+
102
+ # DB column is reset to nil
103
+ expect(MultipleAssetModel.where.not('another_place' => nil).count)
104
+ .to eq(0)
105
+ end
106
+
107
+ it 'Stops after hitting the batch size' do
108
+ 1.upto(11).each do |i|
109
+ asset = Tessa::Asset.new(id: i,
110
+ meta: { name: 'README.md' },
111
+ private_download_url: 'https://test.com/README.md')
112
+ allow(Tessa::Asset).to receive(:find)
113
+ .with(i)
114
+ .and_return(asset)
115
+ allow(Tessa::Asset).to receive(:find)
116
+ .with([i])
117
+ .and_return([asset])
118
+ end
119
+ stub_request(:get, 'https://test.com/README.md')
120
+ .to_return(body: File.new('README.md'))
121
+
122
+ # Mix of the two models...
123
+ models =
124
+ 1.upto(11).map do |i|
125
+ if i % 2 == 0
126
+ MultipleAssetModel.create!(another_place: [i])
127
+ else
128
+ SingleAssetModel.create!(avatar_id: i)
129
+ end
130
+ end
131
+
132
+ final_state = nil
133
+ final_options = nil
134
+ dbl = double('set')
135
+ expect(dbl).to receive(:perform_later) do |state, options|
136
+ final_state = Marshal.load(state)
137
+ final_options = options
138
+ end
139
+ expect(Tessa::MigrateAssetsJob).to receive(:set)
140
+ .with(wait: 10.minutes)
141
+ .and_return(dbl)
142
+
143
+ expect {
144
+ subject.perform
145
+ }.to change { ActiveStorage::Attachment.count }.by(10)
146
+
147
+ expect(final_state.fully_processed?).to be false
148
+ expect(final_options).to eq({
149
+ batch_size: 10,
150
+ interval: 10.minutes.to_i
151
+ })
152
+ # One of the two models was fully processed
153
+ expect(final_state.model_queue.count(&:fully_processed?))
154
+ .to eq(1)
155
+ end
156
+
157
+ it 'Skips over Tessa errors' do
158
+ 1.upto(11).each do |i|
159
+ if i % 2 == 0
160
+ allow(Tessa::Asset).to receive(:find)
161
+ .with(i)
162
+ .and_raise(Tessa::RequestFailed)
163
+ next
164
+ end
165
+
166
+ allow(Tessa::Asset).to receive(:find)
167
+ .with(i)
168
+ .and_return(
169
+ Tessa::Asset.new(id: i,
170
+ meta: { name: 'README.md' },
171
+ private_download_url: 'https://test.com/README.md')
172
+ )
173
+ end
174
+ stub_request(:get, 'https://test.com/README.md')
175
+ .to_return(body: File.new('README.md'))
176
+
177
+ # Mix of the two models...
178
+ models =
179
+ 1.upto(11).map do |i|
180
+ SingleAssetModel.create!(avatar_id: i)
181
+ end
182
+
183
+ final_state = nil
184
+ final_options = nil
185
+ dbl = double('set')
186
+ expect(dbl).to receive(:perform_later) do |state, options|
187
+ final_state = Marshal.load(state)
188
+ final_options = options
189
+ end
190
+ expect(Tessa::MigrateAssetsJob).to receive(:set)
191
+ .with(wait: 10.minutes)
192
+ .and_return(dbl)
193
+
194
+ expect {
195
+ subject.perform
196
+ }.to change { ActiveStorage::Attachment.count }.by(5)
197
+
198
+ expect(final_state.fully_processed?).to be false
199
+ field_state = final_state.next_model.next_field
200
+ expect(field_state.offset).to eq(5)
201
+ end
202
+
203
+ it 'Resumes from marshalled state' do
204
+
205
+ file = Rack::Test::UploadedFile.new('README.md')
206
+
207
+ state = Tessa::MigrateAssetsJob::ProcessingState.initialize_from_models(
208
+ [SingleAssetModel])
209
+ field_state = state.model_queue
210
+ .detect { |m| m.class_name == 'SingleAssetModel' }
211
+ .field_queue
212
+ .detect { |m| m.field_name == :avatar }
213
+
214
+ 1.upto(10).each do |i|
215
+ if i % 2 == 0
216
+ # This one failed
217
+ SingleAssetModel.create!(avatar_id: i).tap do |r|
218
+ field_state.offset += 1
219
+ end
220
+ else
221
+ # This one succeeded and is in ActiveStorage
222
+ SingleAssetModel.create!(avatar: file).tap do |r|
223
+ field_state.success_count += 1
224
+ end
225
+ end
226
+ end
227
+
228
+ # This one still needs to transition
229
+ model = SingleAssetModel.create!(avatar_id: 11)
230
+
231
+ asset = Tessa::Asset.new(id: 11,
232
+ meta: { name: 'README.md' },
233
+ private_download_url: 'https://test.com/README.md')
234
+ allow(Tessa::Asset).to receive(:find)
235
+ .with(11)
236
+ .and_return(asset)
237
+ stub_request(:get, 'https://test.com/README.md')
238
+ .to_return(body: File.new('README.md'))
239
+
240
+ # Doesn't reenqueue since we finished processing
241
+ expect(Tessa::MigrateAssetsJob).to_not receive(:set)
242
+
243
+ expect {
244
+ subject.perform(Marshal.dump(state), { batch_size: 2, interval: 3 })
245
+ }.to change { ActiveStorage::Attachment.count }.by(1)
246
+ end
247
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tessa
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Powell
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-06-22 00:00:00.000000000 Z
12
+ date: 2022-06-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: faraday
@@ -151,6 +151,7 @@ files:
151
151
  - app/javascript/tessa/index.js.coffee
152
152
  - bin/rspec
153
153
  - config/routes.rb
154
+ - lib/tasks/tessa.rake
154
155
  - lib/tessa.rb
155
156
  - lib/tessa/active_storage/asset_wrapper.rb
156
157
  - lib/tessa/asset.rb
@@ -160,6 +161,7 @@ files:
160
161
  - lib/tessa/config.rb
161
162
  - lib/tessa/controller_helpers.rb
162
163
  - lib/tessa/engine.rb
164
+ - lib/tessa/jobs/migrate_assets_job.rb
163
165
  - lib/tessa/model.rb
164
166
  - lib/tessa/model/dynamic_extensions.rb
165
167
  - lib/tessa/model/field.rb
@@ -240,6 +242,7 @@ files:
240
242
  - spec/tessa/asset_spec.rb
241
243
  - spec/tessa/config_spec.rb
242
244
  - spec/tessa/controller_helpers_spec.rb
245
+ - spec/tessa/jobs/migrate_assets_job_spec.rb
243
246
  - spec/tessa/model_field_spec.rb
244
247
  - spec/tessa/model_spec.rb
245
248
  - spec/tessa/rack_upload_proxy_spec.rb
@@ -340,6 +343,7 @@ test_files:
340
343
  - spec/tessa/asset_spec.rb
341
344
  - spec/tessa/config_spec.rb
342
345
  - spec/tessa/controller_helpers_spec.rb
346
+ - spec/tessa/jobs/migrate_assets_job_spec.rb
343
347
  - spec/tessa/model_field_spec.rb
344
348
  - spec/tessa/model_spec.rb
345
349
  - spec/tessa/rack_upload_proxy_spec.rb