tessa 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9f9b764a91daef07d23dcc4adf14060ec1546e3f6cdff0b548b10c4caeb56598
4
- data.tar.gz: e40a45f82d40fb0e3051dc67ffd263541cc2afec0a836912b8dfadbbd9bddf16
3
+ metadata.gz: d31c66f27d8c731d0df1e58f196bd6279f4f2603ab39e33f75f8cf46630745fe
4
+ data.tar.gz: fe6050f938245f6502771adff53f13bd3af40a0339a1098d96bbd2416d6e0a2b
5
5
  SHA512:
6
- metadata.gz: 7de9f6c50a229e8cd628fc09d6fcb4bdbb83b0bd7366c12b547870ce245b936026bcabc28412ffe1bcbca929300e3c0693012b439621063da3f755db1716f71d
7
- data.tar.gz: ec6c2da67ac86e579541bdc485351032718c475077e98fa046cc5cd8959eb728c6a5da915522b73fe7031360c9eb57a0612a27ee39ec467dca619be4e3ea0188
6
+ metadata.gz: 76047a38e882dd59fe104b8be03a2970d996e70eb5f469549d8529e384f46732f5f76dad54e0d7c3d158c60de032b01c467ed13202209d0c4fdad4faa0328b84
7
+ data.tar.gz: '085ac99b9dcce059741887f7852d4a2fd08412b35fa5e520ac61c79f49d5511f43485fe6836863910d07f9336d0c295b31b1bb691b51abc3ea00cefefac35a0b'
data/Gemfile CHANGED
@@ -7,3 +7,8 @@ group :development do
7
7
  gem 'pry'
8
8
  gem 'dotenv'
9
9
  end
10
+
11
+ group :test do
12
+ gem 'rspec-rails'
13
+ gem 'webmock'
14
+ end
@@ -0,0 +1,7 @@
1
+
2
+ namespace :tessa do
3
+ desc "Begins the migration of all Tessa assets to ActiveStorage."
4
+ task :migrate => :environment do
5
+ Tessa::MigrateAssetsJob.perform_later
6
+ end
7
+ end
@@ -20,7 +20,8 @@ module Tessa::ActiveStorage
20
20
  def meta
21
21
  {
22
22
  mime_type: content_type,
23
- size: byte_size
23
+ size: byte_size,
24
+ name: filename
24
25
  }
25
26
  end
26
27
 
@@ -0,0 +1,216 @@
1
+ require 'open-uri'
2
+
3
+ class Tessa::MigrateAssetsJob < ActiveJob::Base
4
+
5
+ queue_as :low
6
+
7
+ def perform(*args)
8
+ options = args&.extract_options!
9
+ options = {
10
+ batch_size: 10,
11
+ interval: 10.minutes.to_i
12
+ }.merge!(options&.symbolize_keys || {})
13
+
14
+ interval = options[:interval].seconds
15
+ processing_state = args.first ? Marshal.load(args.first) : load_models_from_registry
16
+
17
+ if processing_state.fully_processed?
18
+ Rails.logger.info("Nothing to do - all models have transitioned to ActiveStorage")
19
+ return
20
+ end
21
+
22
+ processing_state.batch_count = 0
23
+ while processing_state.batch_count < options[:batch_size]
24
+ model_state = processing_state.next_model
25
+
26
+ process(processing_state, model_state, options)
27
+
28
+ break if processing_state.fully_processed?
29
+ end
30
+
31
+ if processing_state.fully_processed?
32
+ Rails.logger.info("Finished processing all Tessa assets")
33
+ else
34
+ remaining_batches = (processing_state.count / options[:batch_size].to_f).ceil
35
+
36
+ Rails.logger.info("Continuing processing in #{interval}, "\
37
+ "ETA #{(remaining_batches * interval).from_now}. "\
38
+ "Working on #{processing_state.next_model.next_field}")
39
+
40
+ processing_state.batch_count = 0
41
+ self.class.set(wait: interval)
42
+ .perform_later(Marshal.dump(processing_state), options)
43
+ end
44
+ end
45
+
46
+ private
47
+
48
+ def process(processing_state, model_state, options)
49
+ while processing_state.batch_count < options[:batch_size]
50
+ field_state = model_state.next_field
51
+
52
+ process_field(processing_state, field_state, options)
53
+
54
+ return if model_state.fully_processed?
55
+ end
56
+ end
57
+
58
+ def process_field(processing_state, field_state, options)
59
+ while processing_state.batch_count < options[:batch_size]
60
+ remaining = options[:batch_size] - processing_state.batch_count
61
+
62
+ next_batch = field_state.query
63
+ .offset(field_state.offset)
64
+ .limit(remaining)
65
+
66
+ next_batch.each_with_index do |record, idx|
67
+ begin
68
+ # Wait 1 second in between records to slow things down and let the
69
+ # system process for a bit
70
+ sleep 1 if idx > 0
71
+
72
+ reupload(record, field_state)
73
+ Rails.logger.info("#{record.class}#{record.id}##{field_state.field_name}: success")
74
+ field_state.success_count += 1
75
+ rescue StandardError => ex
76
+ Rails.logger.error("#{record.class}#{record.id}##{field_state.field_name}: error - #{ex}")
77
+ field_state.offset += 1
78
+ ensure
79
+ processing_state.batch_count += 1
80
+ end
81
+ end
82
+
83
+ return if field_state.fully_processed?
84
+ end
85
+ end
86
+
87
+ def reupload(record, field_state)
88
+ if field_state.tessa_field.multiple?
89
+ reupload_multiple(record, field_state)
90
+ else
91
+ reupload_single(record, field_state)
92
+ end
93
+ end
94
+
95
+ def reupload_single(record, field_state)
96
+ # models with ActiveStorage uploads have nil in the column, but if you call
97
+ # the method it hits the dynamic extensions and gives you the blob key
98
+ database_id = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
99
+ return unless database_id
100
+
101
+ asset = Tessa::Asset.find(database_id)
102
+
103
+ attachable = {
104
+ io: URI.open(asset.private_download_url),
105
+ filename: asset.meta[:name]
106
+ }
107
+
108
+ record.public_send("#{field_state.tessa_field.name}=", attachable)
109
+ record.save!
110
+ end
111
+
112
+ def reupload_multiple(record, field_state)
113
+ database_ids = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
114
+ return unless database_ids
115
+
116
+ assets = Tessa::Asset.find(database_ids)
117
+
118
+ attachables = assets.map do |asset|
119
+ {
120
+ io: URI.open(asset.private_download_url),
121
+ filename: asset.meta[:name]
122
+ }
123
+ end
124
+
125
+ record.public_send("#{field_state.tessa_field.name}=", attachables)
126
+ record.save!
127
+ end
128
+
129
+ def load_models_from_registry
130
+ Rails.application.eager_load!
131
+
132
+ # Load all Tessa models that can have attachments (not form objects)
133
+ models = Tessa.model_registry
134
+ .select { |m| m.respond_to?(:has_one_attached) }
135
+
136
+ # Initialize our Record Keeping object
137
+ ProcessingState.initialize_from_models(models)
138
+ end
139
+
140
+ ProcessingState = Struct.new(:model_queue, :batch_count) do
141
+ def self.initialize_from_models(models)
142
+ new(
143
+ models.map do |model|
144
+ ModelProcessingState.initialize_from_model(model)
145
+ end,
146
+ 0
147
+ )
148
+ end
149
+
150
+ def next_model
151
+ model_queue.detect { |i| !i.fully_processed? }
152
+ end
153
+
154
+ def fully_processed?
155
+ model_queue.all?(&:fully_processed?)
156
+ end
157
+
158
+ def count
159
+ model_queue.sum { |m| m.field_queue.sum { |f| f.count - f.offset } }
160
+ end
161
+ end
162
+
163
+ ModelProcessingState = Struct.new(:class_name, :field_queue) do
164
+ def self.initialize_from_model(model)
165
+ new(
166
+ model.name,
167
+ model.tessa_fields.map do |name, _|
168
+ FieldProcessingState.initialize_from_model(model, name)
169
+ end
170
+ )
171
+ end
172
+
173
+ def next_field
174
+ field_queue.detect { |i| !i.fully_processed? }
175
+ end
176
+
177
+ def model
178
+ @model ||= class_name.constantize
179
+ end
180
+
181
+ def fully_processed?
182
+ field_queue.all?(&:fully_processed?)
183
+ end
184
+ end
185
+
186
+ FieldProcessingState = Struct.new(:class_name, :field_name, :offset, :success_count) do
187
+ def self.initialize_from_model(model, field_name)
188
+ new(
189
+ model.name,
190
+ field_name,
191
+ 0,
192
+ 0
193
+ )
194
+ end
195
+
196
+ def model
197
+ @model ||= class_name.constantize
198
+ end
199
+
200
+ def tessa_field
201
+ model.tessa_fields[field_name]
202
+ end
203
+
204
+ def query
205
+ model.where.not(Hash[tessa_field.id_field, nil])
206
+ end
207
+
208
+ def count
209
+ query.count
210
+ end
211
+
212
+ def fully_processed?
213
+ offset >= count
214
+ end
215
+ end
216
+ end
@@ -62,7 +62,8 @@ class Tessa::DynamicExtensions
62
62
 
63
63
  def attributes
64
64
  super.merge({
65
- '#{field.id_field}' => #{field.id_field}
65
+ '#{field.id_field}' => #{field.id_field},
66
+ '_tessa_#{field.id_field}' => super['#{field.id_field}']
66
67
  })
67
68
  end
68
69
  CODE
@@ -105,7 +106,7 @@ class Tessa::DynamicExtensions
105
106
  else
106
107
  ids = self.#{field.id_field}
107
108
  ids.delete(change.id.to_i)
108
- self.#{field.id_field} = ids
109
+ self.#{field.id_field} = ids.any? ? ids : nil
109
110
  end
110
111
  end
111
112
  attachables.changes.select(&:add?).each do |change|
@@ -115,14 +116,17 @@ class Tessa::DynamicExtensions
115
116
  end
116
117
  when nil
117
118
  a.detach
119
+ self.#{field.id_field} = nil
118
120
  else
119
121
  a.attach(*attachables)
122
+ self.#{field.id_field} = nil
120
123
  end
121
124
  end
122
125
 
123
126
  def attributes
124
127
  super.merge({
125
- '#{field.id_field}' => #{field.id_field}
128
+ '#{field.id_field}' => #{field.id_field},
129
+ '_tessa_#{field.id_field}' => super['#{field.id_field}']
126
130
  })
127
131
  end
128
132
  CODE
data/lib/tessa/model.rb CHANGED
@@ -10,6 +10,8 @@ module Tessa
10
10
  base.extend ClassMethods
11
11
  base.after_commit :apply_tessa_change_sets if base.respond_to?(:after_commit)
12
12
  base.before_destroy :remove_all_tessa_assets if base.respond_to?(:before_destroy)
13
+
14
+ Tessa.model_registry << base
13
15
  end
14
16
 
15
17
  module InstanceMethods
data/lib/tessa/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Tessa
2
- VERSION = "1.0.2"
2
+ VERSION = "1.1.0"
3
3
  end
data/lib/tessa.rb CHANGED
@@ -15,6 +15,10 @@ require "tessa/rack_upload_proxy"
15
15
  require "tessa/upload"
16
16
  require "tessa/view_helpers"
17
17
 
18
+ if defined?(ActiveJob)
19
+ require "tessa/jobs/migrate_assets_job"
20
+ end
21
+
18
22
  module Tessa
19
23
  class << self
20
24
  def config
@@ -31,6 +35,10 @@ module Tessa
31
35
  return find_asset(ids)
32
36
  end
33
37
 
38
+ def model_registry
39
+ @model_registry ||= []
40
+ end
41
+
34
42
  private
35
43
 
36
44
  def find_asset(id)
data/spec/rails_helper.rb CHANGED
@@ -5,10 +5,14 @@ require 'spec_helper'
5
5
 
6
6
  require File.expand_path('dummy/config/environment.rb', __dir__)
7
7
 
8
+ require 'rspec/rails'
9
+
8
10
  RSpec.configure do |config|
9
11
  ActiveStorage::Current.host = 'https://www.example.com'
10
12
  Rails.application.routes.default_url_options = {
11
13
  protocol: 'https',
12
14
  host: "www.example.com"
13
15
  }
16
+
17
+ config.use_transactional_fixtures = true
14
18
  end
data/spec/spec_helper.rb CHANGED
@@ -1,3 +1,4 @@
1
+ require 'webmock/rspec'
1
2
  require 'tessa'
2
3
  require 'tempfile'
3
4
 
@@ -11,6 +12,8 @@ if ENV['SIMPLE_COV'] || ENV['CC_TEST_REPORTER_ID']
11
12
  end
12
13
 
13
14
  RSpec.configure do |config|
15
+ WebMock.disable_net_connect!
16
+
14
17
  config.expect_with :rspec do |expectations|
15
18
  expectations.include_chain_clauses_in_custom_matcher_descriptions = true
16
19
  end
@@ -1,7 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  RSpec.describe Tessa::Config do
4
- subject(:config) { Tessa::Config.new }
4
+ let(:cfg) { Tessa::Config.new }
5
5
 
6
6
  shared_examples_for "defaults to environment variable" do
7
7
  around { |ex| swap_environment_var(variable_name, 'from-env') { ex.run } }
@@ -19,63 +19,63 @@ RSpec.describe Tessa::Config do
19
19
  describe "#username" do
20
20
  it_behaves_like "defaults to environment variable" do
21
21
  let(:variable_name) { 'TESSA_USERNAME' }
22
- subject(:username) { config.username }
22
+ subject { cfg.username }
23
23
  end
24
24
 
25
25
  it "behaves like a normal accessor" do
26
- config.username = "my-new-value"
27
- expect(config.username).to eq("my-new-value")
26
+ cfg.username = "my-new-value"
27
+ expect(cfg.username).to eq("my-new-value")
28
28
  end
29
29
  end
30
30
 
31
31
  describe "#password" do
32
32
  it_behaves_like "defaults to environment variable" do
33
33
  let(:variable_name) { 'TESSA_PASSWORD' }
34
- subject(:password) { config.password }
34
+ subject { cfg.password }
35
35
  end
36
36
 
37
37
  it "behaves like a normal accessor" do
38
- config.password = "my-new-value"
39
- expect(config.password).to eq("my-new-value")
38
+ cfg.password = "my-new-value"
39
+ expect(cfg.password).to eq("my-new-value")
40
40
  end
41
41
  end
42
42
 
43
43
  describe "#url" do
44
44
  it_behaves_like "defaults to environment variable" do
45
45
  let(:variable_name) { 'TESSA_URL' }
46
- subject(:url) { config.url }
46
+ subject { cfg.url }
47
47
  end
48
48
 
49
49
  it "behaves like a normal accessor" do
50
- config.url = "my-new-value"
51
- expect(config.url).to eq("my-new-value")
50
+ cfg.url = "my-new-value"
51
+ expect(cfg.url).to eq("my-new-value")
52
52
  end
53
53
  end
54
54
 
55
55
  describe "#strategy" do
56
56
  it_behaves_like "defaults to environment variable" do
57
57
  let(:variable_name) { 'TESSA_STRATEGY' }
58
- subject(:strategy) { config.strategy }
58
+ subject { cfg.strategy }
59
59
  end
60
60
 
61
61
  it "uses the string 'default' when no envvar passed" do
62
- expect(config.strategy).to eq("default")
62
+ expect(cfg.strategy).to eq("default")
63
63
  end
64
64
 
65
65
  it "behaves like a normal accessor" do
66
- config.strategy = "my-new-value"
67
- expect(config.strategy).to eq("my-new-value")
66
+ cfg.strategy = "my-new-value"
67
+ expect(cfg.strategy).to eq("my-new-value")
68
68
  end
69
69
  end
70
70
 
71
71
  describe "#connection" do
72
72
  it "is a Faraday::Connection" do
73
- expect(config.connection).to be_a(Faraday::Connection)
73
+ expect(cfg.connection).to be_a(Faraday::Connection)
74
74
  end
75
75
 
76
- context "with values configured" do
77
- subject(:connection) { config.connection }
78
- before { args.each { |k, v| config.send("#{k}=", v) } }
76
+ context "with values cfgured" do
77
+ subject { cfg.connection }
78
+ before { args.each { |k, v| cfg.send("#{k}=", v) } }
79
79
  let(:args) {
80
80
  {
81
81
  url: "http://tessa.test",
@@ -85,28 +85,28 @@ RSpec.describe Tessa::Config do
85
85
  }
86
86
 
87
87
  it "sets faraday's url prefix to our url" do
88
- expect(connection.url_prefix.to_s).to match(config.url)
88
+ expect(subject.url_prefix.to_s).to match(cfg.url)
89
89
  end
90
90
 
91
91
  context "with faraday spy" do
92
92
  let(:spy) { instance_spy(Faraday::Connection) }
93
93
  before do
94
94
  expect(Faraday).to receive(:new).and_yield(spy)
95
- connection
95
+ subject
96
96
  end
97
97
 
98
98
  it "sets up url_encoded request handler" do
99
99
  expect(spy).to have_received(:request).with(:url_encoded)
100
100
  end
101
101
 
102
- it "configures the default adapter" do
102
+ it "cfgures the default adapter" do
103
103
  expect(spy).to have_received(:adapter).with(:net_http)
104
104
  end
105
105
  end
106
106
  end
107
107
 
108
108
  it "caches the result" do
109
- expect(config.connection.object_id).to eq(config.connection.object_id)
109
+ expect(cfg.connection.object_id).to eq(cfg.connection.object_id)
110
110
  end
111
111
  end
112
112
  end
@@ -0,0 +1,247 @@
1
+ require 'rails_helper'
2
+
3
+ require 'tessa/jobs/migrate_assets_job'
4
+
5
+ RSpec.describe Tessa::MigrateAssetsJob do
6
+ it 'does nothing if no db rows' do
7
+
8
+ expect {
9
+ subject.perform
10
+ }.to_not change { ActiveStorage::Attachment.count }
11
+ end
12
+
13
+ it 'creates attachments for old tessa assets' do
14
+ allow(Tessa::Asset).to receive(:find)
15
+ .with(1)
16
+ .and_return(
17
+ Tessa::Asset.new(id: 1,
18
+ meta: { name: 'README.md' },
19
+ private_download_url: 'https://test.com/README.md')
20
+ )
21
+
22
+ allow(Tessa::Asset).to receive(:find)
23
+ .with(2)
24
+ .and_return(
25
+ Tessa::Asset.new(id: 2,
26
+ meta: { name: 'LICENSE.txt' },
27
+ private_download_url: 'https://test.com/LICENSE.txt'),
28
+ )
29
+
30
+ stub_request(:get, 'https://test.com/README.md')
31
+ .to_return(body: File.new('README.md'))
32
+ stub_request(:get, 'https://test.com/LICENSE.txt')
33
+ .to_return(body: File.new('LICENSE.txt'))
34
+
35
+ # DB models with those
36
+ models = [
37
+ SingleAssetModel.create!(avatar_id: 1),
38
+ SingleAssetModel.create!(avatar_id: 2)
39
+ ]
40
+
41
+ expect {
42
+ subject.perform
43
+ }.to change { ActiveStorage::Attachment.count }.by(2)
44
+
45
+ models.each do |m|
46
+ expect(m.reload.avatar_id).to eq(m.avatar.key)
47
+ # Now it's in activestorage
48
+ expect(m.avatar.public_url).to start_with(
49
+ 'https://www.example.com/rails/active_storage/blobs/')
50
+ end
51
+
52
+ expect(SingleAssetModel.where.not('avatar_id' => nil).count)
53
+ .to eq(0)
54
+ end
55
+
56
+ it 'preserves ActiveStorage blobs' do
57
+ allow(Tessa::Asset).to receive(:find)
58
+ .with([1, 2])
59
+ .and_return([
60
+ Tessa::Asset.new(id: 1,
61
+ meta: { name: 'README.md' },
62
+ private_download_url: 'https://test.com/README.md'),
63
+ Tessa::Asset.new(id: 2,
64
+ meta: { name: 'LICENSE.txt' },
65
+ private_download_url: 'https://test.com/LICENSE.txt'),
66
+ ])
67
+ stub_request(:get, 'https://test.com/README.md')
68
+ .to_return(body: File.new('README.md'))
69
+ stub_request(:get, 'https://test.com/LICENSE.txt')
70
+ .to_return(body: File.new('LICENSE.txt'))
71
+
72
+ file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
73
+
74
+ model = MultipleAssetModel.create!(
75
+ # The Tessa DB column has the one asset
76
+ another_place: [1, 2]
77
+ )
78
+ # But has already attached a second ActiveStorage blob
79
+ ::ActiveStorage::Attached::Many.new("multiple_field", model, dependent: :purge_later)
80
+ .attach(file2)
81
+ model.save!
82
+ attachment = model.multiple_field_attachments.first
83
+
84
+ expect {
85
+ subject.perform
86
+ }.to change { ActiveStorage::Attachment.count }.by(2)
87
+
88
+ model = model.reload
89
+ # The IDs are now the keys of ActiveStorage objects
90
+ expect(model.another_place).to eq(
91
+ model.multiple_field.map(&:key))
92
+ # preserves the existing attachment
93
+ expect(model.multiple_field_attachments).to include(attachment)
94
+ expect(model.another_place).to include(attachment.key)
95
+
96
+ # all assets are in activestorage
97
+ model.multiple_field.each do |blob|
98
+ expect(blob.public_url).to start_with(
99
+ 'https://www.example.com/rails/active_storage/blobs/')
100
+ end
101
+
102
+ # DB column is reset to nil
103
+ expect(MultipleAssetModel.where.not('another_place' => nil).count)
104
+ .to eq(0)
105
+ end
106
+
107
+ it 'Stops after hitting the batch size' do
108
+ 1.upto(11).each do |i|
109
+ asset = Tessa::Asset.new(id: i,
110
+ meta: { name: 'README.md' },
111
+ private_download_url: 'https://test.com/README.md')
112
+ allow(Tessa::Asset).to receive(:find)
113
+ .with(i)
114
+ .and_return(asset)
115
+ allow(Tessa::Asset).to receive(:find)
116
+ .with([i])
117
+ .and_return([asset])
118
+ end
119
+ stub_request(:get, 'https://test.com/README.md')
120
+ .to_return(body: File.new('README.md'))
121
+
122
+ # Mix of the two models...
123
+ models =
124
+ 1.upto(11).map do |i|
125
+ if i % 2 == 0
126
+ MultipleAssetModel.create!(another_place: [i])
127
+ else
128
+ SingleAssetModel.create!(avatar_id: i)
129
+ end
130
+ end
131
+
132
+ final_state = nil
133
+ final_options = nil
134
+ dbl = double('set')
135
+ expect(dbl).to receive(:perform_later) do |state, options|
136
+ final_state = Marshal.load(state)
137
+ final_options = options
138
+ end
139
+ expect(Tessa::MigrateAssetsJob).to receive(:set)
140
+ .with(wait: 10.minutes)
141
+ .and_return(dbl)
142
+
143
+ expect {
144
+ subject.perform
145
+ }.to change { ActiveStorage::Attachment.count }.by(10)
146
+
147
+ expect(final_state.fully_processed?).to be false
148
+ expect(final_options).to eq({
149
+ batch_size: 10,
150
+ interval: 10.minutes.to_i
151
+ })
152
+ # One of the two models was fully processed
153
+ expect(final_state.model_queue.count(&:fully_processed?))
154
+ .to eq(1)
155
+ end
156
+
157
+ it 'Skips over Tessa errors' do
158
+ 1.upto(11).each do |i|
159
+ if i % 2 == 0
160
+ allow(Tessa::Asset).to receive(:find)
161
+ .with(i)
162
+ .and_raise(Tessa::RequestFailed)
163
+ next
164
+ end
165
+
166
+ allow(Tessa::Asset).to receive(:find)
167
+ .with(i)
168
+ .and_return(
169
+ Tessa::Asset.new(id: i,
170
+ meta: { name: 'README.md' },
171
+ private_download_url: 'https://test.com/README.md')
172
+ )
173
+ end
174
+ stub_request(:get, 'https://test.com/README.md')
175
+ .to_return(body: File.new('README.md'))
176
+
177
+ # Mix of the two models...
178
+ models =
179
+ 1.upto(11).map do |i|
180
+ SingleAssetModel.create!(avatar_id: i)
181
+ end
182
+
183
+ final_state = nil
184
+ final_options = nil
185
+ dbl = double('set')
186
+ expect(dbl).to receive(:perform_later) do |state, options|
187
+ final_state = Marshal.load(state)
188
+ final_options = options
189
+ end
190
+ expect(Tessa::MigrateAssetsJob).to receive(:set)
191
+ .with(wait: 10.minutes)
192
+ .and_return(dbl)
193
+
194
+ expect {
195
+ subject.perform
196
+ }.to change { ActiveStorage::Attachment.count }.by(5)
197
+
198
+ expect(final_state.fully_processed?).to be false
199
+ field_state = final_state.next_model.next_field
200
+ expect(field_state.offset).to eq(5)
201
+ end
202
+
203
+ it 'Resumes from marshalled state' do
204
+
205
+ file = Rack::Test::UploadedFile.new('README.md')
206
+
207
+ state = Tessa::MigrateAssetsJob::ProcessingState.initialize_from_models(
208
+ [SingleAssetModel])
209
+ field_state = state.model_queue
210
+ .detect { |m| m.class_name == 'SingleAssetModel' }
211
+ .field_queue
212
+ .detect { |m| m.field_name == :avatar }
213
+
214
+ 1.upto(10).each do |i|
215
+ if i % 2 == 0
216
+ # This one failed
217
+ SingleAssetModel.create!(avatar_id: i).tap do |r|
218
+ field_state.offset += 1
219
+ end
220
+ else
221
+ # This one succeeded and is in ActiveStorage
222
+ SingleAssetModel.create!(avatar: file).tap do |r|
223
+ field_state.success_count += 1
224
+ end
225
+ end
226
+ end
227
+
228
+ # This one still needs to transition
229
+ model = SingleAssetModel.create!(avatar_id: 11)
230
+
231
+ asset = Tessa::Asset.new(id: 11,
232
+ meta: { name: 'README.md' },
233
+ private_download_url: 'https://test.com/README.md')
234
+ allow(Tessa::Asset).to receive(:find)
235
+ .with(11)
236
+ .and_return(asset)
237
+ stub_request(:get, 'https://test.com/README.md')
238
+ .to_return(body: File.new('README.md'))
239
+
240
+ # Doesn't reenqueue since we finished processing
241
+ expect(Tessa::MigrateAssetsJob).to_not receive(:set)
242
+
243
+ expect {
244
+ subject.perform(Marshal.dump(state), { batch_size: 2, interval: 3 })
245
+ }.to change { ActiveStorage::Attachment.count }.by(1)
246
+ end
247
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tessa
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Justin Powell
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-06-22 00:00:00.000000000 Z
12
+ date: 2022-06-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: faraday
@@ -151,6 +151,7 @@ files:
151
151
  - app/javascript/tessa/index.js.coffee
152
152
  - bin/rspec
153
153
  - config/routes.rb
154
+ - lib/tasks/tessa.rake
154
155
  - lib/tessa.rb
155
156
  - lib/tessa/active_storage/asset_wrapper.rb
156
157
  - lib/tessa/asset.rb
@@ -160,6 +161,7 @@ files:
160
161
  - lib/tessa/config.rb
161
162
  - lib/tessa/controller_helpers.rb
162
163
  - lib/tessa/engine.rb
164
+ - lib/tessa/jobs/migrate_assets_job.rb
163
165
  - lib/tessa/model.rb
164
166
  - lib/tessa/model/dynamic_extensions.rb
165
167
  - lib/tessa/model/field.rb
@@ -240,6 +242,7 @@ files:
240
242
  - spec/tessa/asset_spec.rb
241
243
  - spec/tessa/config_spec.rb
242
244
  - spec/tessa/controller_helpers_spec.rb
245
+ - spec/tessa/jobs/migrate_assets_job_spec.rb
243
246
  - spec/tessa/model_field_spec.rb
244
247
  - spec/tessa/model_spec.rb
245
248
  - spec/tessa/rack_upload_proxy_spec.rb
@@ -340,6 +343,7 @@ test_files:
340
343
  - spec/tessa/asset_spec.rb
341
344
  - spec/tessa/config_spec.rb
342
345
  - spec/tessa/controller_helpers_spec.rb
346
+ - spec/tessa/jobs/migrate_assets_job_spec.rb
343
347
  - spec/tessa/model_field_spec.rb
344
348
  - spec/tessa/model_spec.rb
345
349
  - spec/tessa/rack_upload_proxy_spec.rb