tessa 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3af9279c5dbfc648d1044110181aeac67f621cd2d60dbe4846f959855db06c12
-  data.tar.gz: afdab873b2e139c7d0e317b758f15e7c810a083a29e792fbdb2747e240ea590b
+  metadata.gz: d31c66f27d8c731d0df1e58f196bd6279f4f2603ab39e33f75f8cf46630745fe
+  data.tar.gz: fe6050f938245f6502771adff53f13bd3af40a0339a1098d96bbd2416d6e0a2b
 SHA512:
-  metadata.gz: d3ec7026403a90d19eb97b92ad5ae9ffce2bf7f473f879a27f19f863f3d776b834aa40f1e437a8a15a81e499a9aa5872f98a4de1ff6630e9cb33fffed673e27e
-  data.tar.gz: 4f1b7591de7519f6936234e69796d358c13b0d238c987fb956ebd60289d254838028f05f1c934177264f83494529636ba6e958887cd78006898f542c74d3e1f9
+  metadata.gz: 76047a38e882dd59fe104b8be03a2970d996e70eb5f469549d8529e384f46732f5f76dad54e0d7c3d158c60de032b01c467ed13202209d0c4fdad4faa0328b84
+  data.tar.gz: '085ac99b9dcce059741887f7852d4a2fd08412b35fa5e520ac61c79f49d5511f43485fe6836863910d07f9336d0c295b31b1bb691b51abc3ea00cefefac35a0b'

data/Gemfile

@@ -7,3 +7,8 @@ group :development do
   gem 'pry'
   gem 'dotenv'
 end
+
+group :test do
+  gem 'rspec-rails'
+  gem 'webmock'
+end

data/lib/tasks/tessa.rake

@@ -0,0 +1,7 @@
+
+namespace :tessa do
+  desc "Begins the migration of all Tessa assets to ActiveStorage."
+  task :migrate => :environment do
+    Tessa::MigrateAssetsJob.perform_later
+  end
+end

data/lib/tessa/active_storage/asset_wrapper.rb

@@ -1,5 +1,9 @@
 module Tessa::ActiveStorage
   class AssetWrapper < SimpleDelegator
+    def id
+      key
+    end
+
     def public_url
       Rails.application.routes.url_helpers.
         rails_blob_url(__getobj__, disposition: :inline)
@@ -14,7 +18,11 @@ module Tessa::ActiveStorage
     end
 
     def meta
-      {}
+      {
+        mime_type: content_type,
+        size: byte_size,
+        name: filename
+      }
     end
 
     def failure?

data/lib/tessa/jobs/migrate_assets_job.rb

@@ -0,0 +1,216 @@
+require 'open-uri'
+
+class Tessa::MigrateAssetsJob < ActiveJob::Base
+
+  queue_as :low
+
+  def perform(*args)
+    options = args&.extract_options!
+    options = {
+      batch_size: 10,
+      interval: 10.minutes.to_i
+    }.merge!(options&.symbolize_keys || {})
+
+    interval = options[:interval].seconds
+    processing_state = args.first ? Marshal.load(args.first) : load_models_from_registry
+
+    if processing_state.fully_processed?
+      Rails.logger.info("Nothing to do - all models have transitioned to ActiveStorage")
+      return
+    end
+
+    processing_state.batch_count = 0
+    while processing_state.batch_count < options[:batch_size]
+      model_state = processing_state.next_model
+
+      process(processing_state, model_state, options)
+
+      break if processing_state.fully_processed?
+    end
+
+    if processing_state.fully_processed?
+      Rails.logger.info("Finished processing all Tessa assets")
+    else
+      remaining_batches = (processing_state.count / options[:batch_size].to_f).ceil
+
+      Rails.logger.info("Continuing processing in #{interval}, "\
+        "ETA #{(remaining_batches * interval).from_now}. "\
+        "Working on #{processing_state.next_model.next_field}")
+
+      processing_state.batch_count = 0
+      self.class.set(wait: interval)
+        .perform_later(Marshal.dump(processing_state), options)
+    end
+  end
+
+  private
+
+  def process(processing_state, model_state, options)
+    while processing_state.batch_count < options[:batch_size]
+      field_state = model_state.next_field
+
+      process_field(processing_state, field_state, options)
+
+      return if model_state.fully_processed?
+    end
+  end
+
+  def process_field(processing_state, field_state, options)
+    while processing_state.batch_count < options[:batch_size]
+      remaining = options[:batch_size] - processing_state.batch_count
+
+      next_batch = field_state.query
+        .offset(field_state.offset)
+        .limit(remaining)
+
+      next_batch.each_with_index do |record, idx|
+        begin
+          # Wait 1 second in between records to slow things down and let the
+          # system process for a bit
+          sleep 1 if idx > 0
+
+          reupload(record, field_state)
+          Rails.logger.info("#{record.class}#{record.id}##{field_state.field_name}: success")
+          field_state.success_count += 1
+        rescue StandardError => ex
+          Rails.logger.error("#{record.class}#{record.id}##{field_state.field_name}: error - #{ex}")
+          field_state.offset += 1
+        ensure
+          processing_state.batch_count += 1
+        end
+      end
+
+      return if field_state.fully_processed?
+    end
+  end
+
+  def reupload(record, field_state)
+    if field_state.tessa_field.multiple?
+      reupload_multiple(record, field_state)
+    else
+      reupload_single(record, field_state)
+    end    
+  end
+
+  def reupload_single(record, field_state)
+    # models with ActiveStorage uploads have nil in the column, but if you call
+    # the method it hits the dynamic extensions and gives you the blob key
+    database_id = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
+    return unless database_id
+
+    asset = Tessa::Asset.find(database_id)
+
+    attachable = {
+      io: URI.open(asset.private_download_url),
+      filename: asset.meta[:name]
+    }
+
+    record.public_send("#{field_state.tessa_field.name}=", attachable)
+    record.save!
+  end
+
+  def reupload_multiple(record, field_state)
+    database_ids = record.attributes["_tessa_#{field_state.tessa_field.id_field}"]
+    return unless database_ids
+    
+    assets = Tessa::Asset.find(database_ids)
+
+    attachables = assets.map do |asset|
+      {
+        io: URI.open(asset.private_download_url),
+        filename: asset.meta[:name]
+      }
+    end
+
+    record.public_send("#{field_state.tessa_field.name}=", attachables)
+    record.save!
+  end
+
+  def load_models_from_registry
+    Rails.application.eager_load!
+
+    # Load all Tessa models that can have attachments (not form objects)
+    models = Tessa.model_registry
+      .select { |m| m.respond_to?(:has_one_attached) }
+
+    # Initialize our Record Keeping object
+    ProcessingState.initialize_from_models(models)
+  end
+
+  ProcessingState = Struct.new(:model_queue, :batch_count) do
+    def self.initialize_from_models(models)
+      new(
+        models.map do |model|
+          ModelProcessingState.initialize_from_model(model)
+        end,
+        0
+      )
+    end
+
+    def next_model
+      model_queue.detect { |i| !i.fully_processed? }
+    end
+
+    def fully_processed?
+      model_queue.all?(&:fully_processed?)
+    end
+
+    def count
+      model_queue.sum { |m| m.field_queue.sum { |f| f.count - f.offset } }
+    end
+  end
+
+  ModelProcessingState = Struct.new(:class_name, :field_queue) do
+    def self.initialize_from_model(model)
+      new(
+        model.name,
+        model.tessa_fields.map do |name, _|
+          FieldProcessingState.initialize_from_model(model, name)
+        end
+      )
+    end
+
+    def next_field
+      field_queue.detect { |i| !i.fully_processed? }
+    end
+
+    def model
+      @model ||= class_name.constantize
+    end
+
+    def fully_processed?
+      field_queue.all?(&:fully_processed?)
+    end
+  end
+
+  FieldProcessingState = Struct.new(:class_name, :field_name, :offset, :success_count) do
+    def self.initialize_from_model(model, field_name)
+      new(
+        model.name,
+        field_name,
+        0,
+        0
+      )
+    end
+
+    def model
+      @model ||= class_name.constantize
+    end
+
+    def tessa_field
+      model.tessa_fields[field_name]
+    end
+
+    def query
+      model.where.not(Hash[tessa_field.id_field, nil])
+    end
+
+    def count
+      query.count
+    end
+
+    def fully_processed?
+      offset >= count
+    end
+  end
+end

data/lib/tessa/model/dynamic_extensions.rb

@@ -44,9 +44,11 @@ class Tessa::DynamicExtensions
             
             case attachable
             when Tessa::AssetChangeSet
-              attachable.changes.each do |change|
-                a.attach(change.id) if change.add?
-                a.detach if change.remove?
+              attachable.changes.select(&:remove?).each { a.detatch }
+              attachable.changes.select(&:add?).each do |change|
+                next if #{field.id_field} == change.id
+
+                a.attach(change.id)
               end
             when nil
               a.detach
@@ -60,7 +62,8 @@ class Tessa::DynamicExtensions
 
           def attributes
             super.merge({
-              '#{field.id_field}' => #{field.id_field}
+              '#{field.id_field}' => #{field.id_field},
+              '_tessa_#{field.id_field}' => super['#{field.id_field}']
             })
           end
         CODE
@@ -72,24 +75,22 @@ class Tessa::DynamicExtensions
     def build(mod)
       mod.class_eval <<~CODE, __FILE__, __LINE__ + 1
           def #{name}
-            if #{name}_attachments.present?
-              return #{name}_attachments.map do |a|
-                Tessa::ActiveStorage::AssetWrapper.new(a)
-              end
-            end
-
-            # fall back to old Tessa fetch if not present
-            if field = self.class.tessa_fields["#{name}".to_sym]
-              @#{name} ||= fetch_tessa_remote_assets(field.id(on: self))
-            end
+            field = self.class.tessa_fields["#{name}".to_sym]
+            tessa_ids = field.id(on: self) - #{name}_attachments.map(&:key)
+            
+            @#{name} ||= [
+              *#{name}_attachments.map { |a| Tessa::ActiveStorage::AssetWrapper.new(a) },
+              *fetch_tessa_remote_assets(tessa_ids)
+            ]
           end
 
           def #{field.id_field}
-            # Use the attachment's key
-            return #{name}_attachments.map(&:key) if #{name}_attachments.present?
-
-            # fallback to Tessa's database column
-            super
+            [
+              # Use the attachment's key
+              *#{name}_attachments.map(&:key),
+              # include from Tessa's database column
+              *super
+            ]
           end
 
           def #{name}=(attachables)
@@ -99,23 +100,33 @@ class Tessa::DynamicExtensions
 
             case attachables
             when Tessa::AssetChangeSet
-              attachables.changes.each do |change|
-                a.attach(change.id) if change.add?
-                raise 'TODO' if change.remove?
+              attachables.changes.select(&:remove?).each do |change|
+                if existing = #{name}_attachments.find { |a| a.key == change.id }
+                  existing.destroy
+                else
+                  ids = self.#{field.id_field}
+                  ids.delete(change.id.to_i)
+                  self.#{field.id_field} = ids.any? ? ids : nil
+                end
+              end
+              attachables.changes.select(&:add?).each do |change|
+                next if #{field.id_field}.include? change.id
+
+                a.attach(change.id)
               end
             when nil
               a.detach
+              self.#{field.id_field} = nil
             else
               a.attach(*attachables)
+              self.#{field.id_field} = nil
             end
-
-            # overwrite the tessa ID in the database
-            self.#{field.id_field} = nil
           end
 
           def attributes
             super.merge({
-              '#{field.id_field}' => #{field.id_field}
+              '#{field.id_field}' => #{field.id_field},
+              '_tessa_#{field.id_field}' => super['#{field.id_field}']
             })
           end
         CODE

data/lib/tessa/model.rb

@@ -10,6 +10,8 @@ module Tessa
       base.extend ClassMethods
       base.after_commit :apply_tessa_change_sets if base.respond_to?(:after_commit)
       base.before_destroy :remove_all_tessa_assets if base.respond_to?(:before_destroy)
+
+      Tessa.model_registry << base
     end
 
     module InstanceMethods

data/lib/tessa/version.rb

@@ -1,3 +1,3 @@
 module Tessa
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/tessa.rb

@@ -15,38 +15,60 @@ require "tessa/rack_upload_proxy"
 require "tessa/upload"
 require "tessa/view_helpers"
 
+if defined?(ActiveJob)
+  require "tessa/jobs/migrate_assets_job"
+end
+
 module Tessa
-  def self.config
-    @config ||= Config.new
-  end
+  class << self
+    def config
+      @config ||= Config.new
+    end
 
-  def self.setup
-    yield config
-  end
+    def setup
+      yield config
+    end
 
-  def self.find_assets(ids)
-    if [*ids].empty?
-      if ids.is_a?(Array)
-        []
-      else
-        nil
-      end
-    elsif (blobs = ::ActiveStorage::Blob.where(key: ids).to_a).present?
-      if ids.is_a?(Array)
-        blobs.map { |a| Tessa::ActiveStorage::AssetWrapper.new(a) }
-      else
-        Tessa::ActiveStorage::AssetWrapper.new(blobs.first)
-      end
-    else
-      Tessa::Asset.find(ids)
+    def find_assets(ids)
+      return find_all_assets(ids) if ids.is_a?(Array)
+
+      return find_asset(ids)
     end
-  rescue Tessa::RequestFailed => err
-    if ids.is_a?(Array)
-      ids.map do |id|
-        Tessa::Asset::Failure.factory(id: id, response: err.response)
+
+    def model_registry
+      @model_registry ||= []
+    end
+
+    private
+
+    def find_asset(id)
+      return nil unless id
+
+      if blob = ::ActiveStorage::Blob.find_by(key: id)
+        return Tessa::ActiveStorage::AssetWrapper.new(blob)
       end
-    else
-      Tessa::Asset::Failure.factory(id: ids, response: err.response)
+
+      Tessa::Asset.find(id)
+    rescue Tessa::RequestFailed => err
+      Tessa::Asset::Failure.factory(id: id, response: err.response)
+    end
+
+    def find_all_assets(ids)
+      return [] if ids.empty?
+
+      blobs = ::ActiveStorage::Blob.where(key: ids).to_a
+        .map { |a| Tessa::ActiveStorage::AssetWrapper.new(a) }
+      ids = ids - blobs.map(&:key)
+      assets = 
+        begin
+          Tessa::Asset.find(ids) if ids.any?
+        rescue Tessa::RequestFailed => err
+          ids.map do |id|
+            Tessa::Asset::Failure.factory(id: id, response: err.response)
+          end
+        end
+
+      [*blobs, *assets]
     end
   end
 

data/spec/rails_helper.rb

@@ -5,10 +5,14 @@ require 'spec_helper'
 
 require File.expand_path('dummy/config/environment.rb', __dir__)
 
+require 'rspec/rails'
+
 RSpec.configure do |config|
   ActiveStorage::Current.host = 'https://www.example.com'
   Rails.application.routes.default_url_options = {
     protocol: 'https',
     host: "www.example.com"
   }
+
+  config.use_transactional_fixtures = true
 end

data/spec/spec_helper.rb

@@ -1,3 +1,4 @@
+require 'webmock/rspec'
 require 'tessa'
 require 'tempfile'
 
@@ -11,6 +12,8 @@ if ENV['SIMPLE_COV'] || ENV['CC_TEST_REPORTER_ID']
 end
 
 RSpec.configure do |config|
+  WebMock.disable_net_connect!
+
   config.expect_with :rspec do |expectations|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end

data/spec/tessa/config_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 RSpec.describe Tessa::Config do
-  subject(:config) { Tessa::Config.new }
+  let(:cfg) { Tessa::Config.new }
 
   shared_examples_for "defaults to environment variable" do
     around { |ex| swap_environment_var(variable_name, 'from-env') { ex.run } }
@@ -19,63 +19,63 @@ RSpec.describe Tessa::Config do
   describe "#username" do
     it_behaves_like "defaults to environment variable" do
       let(:variable_name) { 'TESSA_USERNAME' }
-      subject(:username) { config.username }
+      subject { cfg.username }
     end
 
     it "behaves like a normal accessor" do
-      config.username = "my-new-value"
-      expect(config.username).to eq("my-new-value")
+      cfg.username = "my-new-value"
+      expect(cfg.username).to eq("my-new-value")
     end
   end
 
   describe "#password" do
     it_behaves_like "defaults to environment variable" do
       let(:variable_name) { 'TESSA_PASSWORD' }
-      subject(:password) { config.password }
+      subject { cfg.password }
     end
 
     it "behaves like a normal accessor" do
-      config.password = "my-new-value"
-      expect(config.password).to eq("my-new-value")
+      cfg.password = "my-new-value"
+      expect(cfg.password).to eq("my-new-value")
     end
   end
 
   describe "#url" do
     it_behaves_like "defaults to environment variable" do
       let(:variable_name) { 'TESSA_URL' }
-      subject(:url) { config.url }
+      subject { cfg.url }
     end
 
     it "behaves like a normal accessor" do
-      config.url = "my-new-value"
-      expect(config.url).to eq("my-new-value")
+      cfg.url = "my-new-value"
+      expect(cfg.url).to eq("my-new-value")
     end
   end
 
   describe "#strategy" do
     it_behaves_like "defaults to environment variable" do
       let(:variable_name) { 'TESSA_STRATEGY' }
-      subject(:strategy) { config.strategy }
+      subject { cfg.strategy }
     end
 
     it "uses the string 'default' when no envvar passed" do
-      expect(config.strategy).to eq("default")
+      expect(cfg.strategy).to eq("default")
     end
 
     it "behaves like a normal accessor" do
-      config.strategy = "my-new-value"
-      expect(config.strategy).to eq("my-new-value")
+      cfg.strategy = "my-new-value"
+      expect(cfg.strategy).to eq("my-new-value")
     end
   end
 
   describe "#connection" do
     it "is a Faraday::Connection" do
-      expect(config.connection).to be_a(Faraday::Connection)
+      expect(cfg.connection).to be_a(Faraday::Connection)
     end
 
-    context "with values configured" do
-      subject(:connection) { config.connection }
-      before { args.each { |k, v| config.send("#{k}=", v) } }
+    context "with values cfgured" do
+      subject { cfg.connection }
+      before { args.each { |k, v| cfg.send("#{k}=", v) } }
       let(:args) {
         {
           url: "http://tessa.test",
@@ -85,28 +85,28 @@ RSpec.describe Tessa::Config do
       }
 
       it "sets faraday's url prefix to our url" do
-        expect(connection.url_prefix.to_s).to match(config.url)
+        expect(subject.url_prefix.to_s).to match(cfg.url)
       end
 
       context "with faraday spy" do
         let(:spy) { instance_spy(Faraday::Connection) }
         before do
           expect(Faraday).to receive(:new).and_yield(spy)
-          connection
+          subject
         end
 
         it "sets up url_encoded request handler" do
           expect(spy).to have_received(:request).with(:url_encoded)
         end
 
-        it "configures the default adapter" do
+        it "cfgures the default adapter" do
           expect(spy).to have_received(:adapter).with(:net_http)
         end
       end
     end
 
     it "caches the result" do
-      expect(config.connection.object_id).to eq(config.connection.object_id)
+      expect(cfg.connection.object_id).to eq(cfg.connection.object_id)
     end
   end
 end

data/spec/tessa/jobs/migrate_assets_job_spec.rb

@@ -0,0 +1,247 @@
+require 'rails_helper'
+
+require 'tessa/jobs/migrate_assets_job'
+
+RSpec.describe Tessa::MigrateAssetsJob do
+  it 'does nothing if no db rows' do
+    
+    expect {
+      subject.perform
+    }.to_not change { ActiveStorage::Attachment.count }
+  end
+
+  it 'creates attachments for old tessa assets' do
+    allow(Tessa::Asset).to receive(:find)
+      .with(1)
+      .and_return(
+        Tessa::Asset.new(id: 1,
+          meta: { name: 'README.md' },
+          private_download_url: 'https://test.com/README.md')
+      )
+    
+    allow(Tessa::Asset).to receive(:find)
+      .with(2)
+      .and_return(
+        Tessa::Asset.new(id: 2,
+          meta: { name: 'LICENSE.txt' },
+          private_download_url: 'https://test.com/LICENSE.txt'),
+      )
+
+    stub_request(:get, 'https://test.com/README.md')
+      .to_return(body: File.new('README.md'))
+    stub_request(:get, 'https://test.com/LICENSE.txt')
+      .to_return(body: File.new('LICENSE.txt'))
+
+      # DB models with those
+    models = [
+      SingleAssetModel.create!(avatar_id: 1),
+      SingleAssetModel.create!(avatar_id: 2)
+    ]
+
+    expect {
+      subject.perform
+    }.to change { ActiveStorage::Attachment.count }.by(2)
+
+    models.each do |m|
+      expect(m.reload.avatar_id).to eq(m.avatar.key)
+      # Now it's in activestorage
+      expect(m.avatar.public_url).to start_with(
+        'https://www.example.com/rails/active_storage/blobs/')
+    end
+
+    expect(SingleAssetModel.where.not('avatar_id' => nil).count)
+      .to eq(0)
+  end
+
+  it 'preserves ActiveStorage blobs' do
+    allow(Tessa::Asset).to receive(:find)
+      .with([1, 2])
+      .and_return([
+        Tessa::Asset.new(id: 1,
+          meta: { name: 'README.md' },
+          private_download_url: 'https://test.com/README.md'),
+        Tessa::Asset.new(id: 2,
+          meta: { name: 'LICENSE.txt' },
+          private_download_url: 'https://test.com/LICENSE.txt'),
+      ])
+    stub_request(:get, 'https://test.com/README.md')
+      .to_return(body: File.new('README.md'))
+    stub_request(:get, 'https://test.com/LICENSE.txt')
+      .to_return(body: File.new('LICENSE.txt'))
+
+    file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
+
+    model = MultipleAssetModel.create!(
+      # The Tessa DB column has the one asset
+      another_place: [1, 2]
+    )
+    # But has already attached a second ActiveStorage blob
+    ::ActiveStorage::Attached::Many.new("multiple_field", model, dependent: :purge_later)
+      .attach(file2)
+    model.save!
+    attachment = model.multiple_field_attachments.first
+
+    expect {
+      subject.perform
+    }.to change { ActiveStorage::Attachment.count }.by(2)
+
+    model = model.reload
+    # The IDs are now the keys of ActiveStorage objects
+    expect(model.another_place).to eq(
+      model.multiple_field.map(&:key))
+    # preserves the existing attachment
+    expect(model.multiple_field_attachments).to include(attachment)
+    expect(model.another_place).to include(attachment.key)
+
+    # all assets are in activestorage
+    model.multiple_field.each do |blob|
+      expect(blob.public_url).to start_with(
+        'https://www.example.com/rails/active_storage/blobs/')
+    end
+
+    # DB column is reset to nil
+    expect(MultipleAssetModel.where.not('another_place' => nil).count)
+      .to eq(0)
+  end
+
+  it 'Stops after hitting the batch size' do
+    1.upto(11).each do |i|
+      asset = Tessa::Asset.new(id: i,
+        meta: { name: 'README.md' },
+        private_download_url: 'https://test.com/README.md')
+      allow(Tessa::Asset).to receive(:find)
+        .with(i)
+        .and_return(asset)
+      allow(Tessa::Asset).to receive(:find)
+        .with([i])
+        .and_return([asset])
+    end
+    stub_request(:get, 'https://test.com/README.md')
+      .to_return(body: File.new('README.md'))
+
+    # Mix of the two models...
+    models =
+      1.upto(11).map do |i|
+        if i % 2 == 0
+          MultipleAssetModel.create!(another_place: [i])
+        else
+          SingleAssetModel.create!(avatar_id: i)
+        end
+      end
+
+    final_state = nil
+    final_options = nil
+    dbl = double('set')
+    expect(dbl).to receive(:perform_later) do |state, options|
+      final_state = Marshal.load(state)
+      final_options = options
+    end
+    expect(Tessa::MigrateAssetsJob).to receive(:set)
+      .with(wait: 10.minutes)
+      .and_return(dbl) 
+    
+    expect {
+      subject.perform
+    }.to change { ActiveStorage::Attachment.count }.by(10)
+
+    expect(final_state.fully_processed?).to be false
+    expect(final_options).to eq({
+      batch_size: 10,
+      interval: 10.minutes.to_i
+    })
+    # One of the two models was fully processed
+    expect(final_state.model_queue.count(&:fully_processed?))
+      .to eq(1)
+  end
+
+  it 'Skips over Tessa errors' do
+    1.upto(11).each do |i|
+      if i % 2 == 0
+        allow(Tessa::Asset).to receive(:find)
+          .with(i)
+          .and_raise(Tessa::RequestFailed)
+        next
+      end
+
+      allow(Tessa::Asset).to receive(:find)
+        .with(i)
+        .and_return(
+          Tessa::Asset.new(id: i,
+            meta: { name: 'README.md' },
+            private_download_url: 'https://test.com/README.md')
+        )
+    end
+    stub_request(:get, 'https://test.com/README.md')
+      .to_return(body: File.new('README.md'))
+
+    # Mix of the two models...
+    models =
+      1.upto(11).map do |i|
+        SingleAssetModel.create!(avatar_id: i)
+      end
+
+    final_state = nil
+    final_options = nil
+    dbl = double('set')
+    expect(dbl).to receive(:perform_later) do |state, options|
+      final_state = Marshal.load(state)
+      final_options = options
+    end
+    expect(Tessa::MigrateAssetsJob).to receive(:set)
+      .with(wait: 10.minutes)
+      .and_return(dbl) 
+    
+    expect {
+      subject.perform
+    }.to change { ActiveStorage::Attachment.count }.by(5)
+
+    expect(final_state.fully_processed?).to be false
+    field_state = final_state.next_model.next_field
+    expect(field_state.offset).to eq(5)
+  end
+
+  it 'Resumes from marshalled state' do
+
+    file = Rack::Test::UploadedFile.new('README.md')
+
+    state = Tessa::MigrateAssetsJob::ProcessingState.initialize_from_models(
+      [SingleAssetModel])
+    field_state = state.model_queue
+      .detect { |m| m.class_name == 'SingleAssetModel' }
+      .field_queue
+      .detect { |m| m.field_name == :avatar }
+    
+    1.upto(10).each do |i|
+      if i % 2 == 0
+        # This one failed
+        SingleAssetModel.create!(avatar_id: i).tap do |r|
+          field_state.offset += 1
+        end
+      else
+        # This one succeeded and is in ActiveStorage
+        SingleAssetModel.create!(avatar: file).tap do |r|
+          field_state.success_count += 1
+        end
+      end
+    end
+    
+    # This one still needs to transition
+    model = SingleAssetModel.create!(avatar_id: 11)
+
+    asset = Tessa::Asset.new(id: 11,
+      meta: { name: 'README.md' },
+      private_download_url: 'https://test.com/README.md')
+    allow(Tessa::Asset).to receive(:find)
+      .with(11)
+      .and_return(asset)
+    stub_request(:get, 'https://test.com/README.md')
+      .to_return(body: File.new('README.md'))
+
+    # Doesn't reenqueue since we finished processing
+    expect(Tessa::MigrateAssetsJob).to_not receive(:set)
+
+    expect {
+      subject.perform(Marshal.dump(state), { batch_size: 2, interval: 3 })
+    }.to change { ActiveStorage::Attachment.count }.by(1)
+  end
+end

data/spec/tessa/model_spec.rb

@@ -41,6 +41,7 @@ RSpec.describe Tessa::Model do
       let(:model) {
         MultipleAssetModel
       }
+      let(:instance) { model.new(another_place: []) }
 
       it "sets all attributes on ModelField properly" do
         field = model.tessa_fields[:multiple_field]
@@ -89,6 +90,7 @@ RSpec.describe Tessa::Model do
       let(:model) {
         MultipleAssetModel
       }
+      let(:instance) { model.new(another_place: []) }
       subject(:getter) { instance.multiple_field }
 
       it "calls find for each of the file_ids and returns result" do
@@ -181,9 +183,11 @@ RSpec.describe Tessa::Model do
         SingleAssetModel
       }
       subject(:getter) { instance.avatar }
+      let(:file) {
+        Rack::Test::UploadedFile.new("README.md")
+      }
 
       it 'attaches uploaded file' do
-        file = Rack::Test::UploadedFile.new("README.md")
         instance.avatar = file
 
         expect(getter.name).to eq('avatar')
@@ -194,28 +198,65 @@ RSpec.describe Tessa::Model do
       end
 
       it 'sets the ID to be the ActiveStorage key' do
-        file = Rack::Test::UploadedFile.new("README.md")
         instance.avatar = file
 
         expect(instance.avatar_id).to eq(instance.avatar_attachment.key)
       end
 
       it 'sets the ID in the attributes' do
-        file = Rack::Test::UploadedFile.new("README.md")
         instance.avatar = file
         
         expect(instance.attributes['avatar_id']).to eq(instance.avatar_attachment.key)
       end
+
+      it 'attaches signed ID from Tessa::AssetChangeSet' do
+        blob = ::ActiveStorage::Blob.create_before_direct_upload!({
+          filename: 'README.md',
+          byte_size: file.size,
+          content_type: file.content_type,
+          checksum: '1234'
+        })
+
+        changeset = Tessa::AssetChangeSet.new(
+          changes: [{ 'id' => blob.signed_id, 'action' => 'add' }]
+        )
+        instance.avatar = changeset
+
+        expect(instance.avatar_id).to eq(instance.avatar_attachment.key)
+      end
+
+      it 'does nothing when "add"ing an existing blob' do
+        # Before this HTTP POST, we've previously uploaded this file
+        instance.avatar = file
+
+        # In this HTTP POST, we re-upload the 'add' action with the same ID
+        changeset = Tessa::AssetChangeSet.new(
+          changes: [{ 'id' => instance.avatar_attachment.key, 'action' => 'add' }]
+        )
+
+        # We expect that we're not going to detatch the existing attachment
+        expect(instance.avatar_attachment).to_not receive(:destroy)
+
+        # act
+        instance.avatar = changeset
+
+        expect(instance.avatar_id).to eq(instance.avatar_attachment.key)
+      end
     end
 
     context "with a multiple typed field" do
       let(:model) {
         MultipleAssetModel
       }
+      let(:instance) { model.new(another_place: []) }
+      let(:file) {
+        Rack::Test::UploadedFile.new("README.md")
+      }
+      let(:file2) {
+        Rack::Test::UploadedFile.new("LICENSE.txt")
+      }
 
       it 'attaches uploaded files' do
-        file = Rack::Test::UploadedFile.new("README.md")
-        file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
         instance.multiple_field = [file, file2]
 
         expect(instance.multiple_field[0].name).to eq('multiple_field')
@@ -231,20 +272,119 @@ RSpec.describe Tessa::Model do
       end
 
       it 'sets the ID to be the ActiveStorage key' do
-        file = Rack::Test::UploadedFile.new("README.md")
-        file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
         instance.multiple_field = [file, file2]
 
         expect(instance.another_place).to eq(instance.multiple_field_attachments.map(&:key))
       end
 
       it 'sets the ID in the attributes' do
-        file = Rack::Test::UploadedFile.new("README.md")
-        file2 = Rack::Test::UploadedFile.new("LICENSE.txt")
         instance.multiple_field = [file, file2]
         
         expect(instance.attributes['another_place']).to eq(instance.multiple_field_attachments.map(&:key))
       end
+
+      it 'attaches signed ID from Tessa::AssetChangeSet' do
+        blob = ::ActiveStorage::Blob.create_before_direct_upload!({
+          filename: 'README.md',
+          byte_size: file.size,
+          content_type: file.content_type,
+          checksum: '1234'
+        })
+        blob2 = ::ActiveStorage::Blob.create_before_direct_upload!({
+          filename: "LICENSE.txt",
+          byte_size: file2.size,
+          content_type: file2.content_type,
+          checksum: '5678'
+        })
+
+        changeset = Tessa::AssetChangeSet.new(
+          changes: [
+            { 'id' => blob.signed_id, 'action' => 'add' },
+            { 'id' => blob2.signed_id, 'action' => 'add' },
+          ]
+        )
+        instance.multiple_field = changeset
+
+        expect(instance.another_place).to eq([
+          blob.key,
+          blob2.key
+        ])
+      end
+
+      it 'does nothing when "add"ing an existing blob' do
+        # Before this HTTP POST, we've previously uploaded these files
+        instance.multiple_field = [file, file2]
+        keys = instance.multiple_field_attachments.map(&:key)
+
+        # In this HTTP POST, we re-upload the 'add' action with the same ID
+        changeset = Tessa::AssetChangeSet.new(
+          changes: [
+            { 'id' => keys[0], 'action' => 'add' },
+            { 'id' => keys[1], 'action' => 'add' },
+          ]
+        )
+
+        # We expect that we're not going to detatch the existing attachment
+        instance.multiple_field_attachments.each do |a|
+          expect(a).to_not receive(:destroy)
+        end
+
+        # act
+        instance.multiple_field = changeset
+
+        expect(instance.another_place).to eq(keys)
+      end
+
+      it 'replaces Tessa assets with ActiveStorage assets' do
+        # Before deploying this code, we previously had DB records with Tessa IDs
+        instance.update!(another_place: [1, 2, 3])
+
+        # In this HTTP POST, we removed one of the tessa assets and uploaded a
+        # new ActiveStorage asset
+        blob = ::ActiveStorage::Blob.create_before_direct_upload!({
+          filename: 'README.md',
+          byte_size: file.size,
+          content_type: file.content_type,
+          checksum: '1234'
+        })
+        changeset = Tessa::AssetChangeSet.new(
+          changes: [
+            { 'id' => 1, 'action' => 'add' },
+            { 'id' => 2, 'action' => 'remove' },
+            { 'id' => 3, 'action' => 'add' },
+            { 'id' => blob.signed_id, 'action' => 'add' },
+          ]
+        )
+
+        # We'll download these assets when we access #multiple_field
+        allow(Tessa.config.connection).to receive(:get)
+          .with("/assets/1,3")
+          .and_return(double("response",
+            success?: true,
+            body: [
+              { 'id' => 1, 'public_url' => 'test1' },
+              { 'id' => 2, 'public_url' => 'test2' }
+            ].to_json))
+
+        blob.upload(file)
+
+        # act
+        instance.multiple_field = changeset
+
+        expect(instance.another_place).to eq([
+          blob.key, 1, 3
+        ])
+
+        assets = instance.multiple_field
+        expect(assets[0].key).to eq(blob.key)
+        expect(assets[0].service_url)
+          .to start_with('https://www.example.com/rails/active_storage/disk/')
+
+        expect(assets[1].id).to eq(1)
+        expect(assets[1].public_url).to eq('test1')
+        expect(assets[2].id).to eq(2)
+        expect(assets[2].public_url).to eq('test2')
+      end
     end
   end
 
@@ -385,6 +525,7 @@ RSpec.describe Tessa::Model do
       let(:model) {
         MultipleAssetModel
       }
+      let(:instance) { model.new(another_place: []) }
       
       before do
         instance.another_place = [2, 3]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tessa
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Justin Powell
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-15 00:00:00.000000000 Z
+date: 2022-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -151,6 +151,7 @@ files:
 - app/javascript/tessa/index.js.coffee
 - bin/rspec
 - config/routes.rb
+- lib/tasks/tessa.rake
 - lib/tessa.rb
 - lib/tessa/active_storage/asset_wrapper.rb
 - lib/tessa/asset.rb
@@ -160,6 +161,7 @@ files:
 - lib/tessa/config.rb
 - lib/tessa/controller_helpers.rb
 - lib/tessa/engine.rb
+- lib/tessa/jobs/migrate_assets_job.rb
 - lib/tessa/model.rb
 - lib/tessa/model/dynamic_extensions.rb
 - lib/tessa/model/field.rb
@@ -240,6 +242,7 @@ files:
 - spec/tessa/asset_spec.rb
 - spec/tessa/config_spec.rb
 - spec/tessa/controller_helpers_spec.rb
+- spec/tessa/jobs/migrate_assets_job_spec.rb
 - spec/tessa/model_field_spec.rb
 - spec/tessa/model_spec.rb
 - spec/tessa/rack_upload_proxy_spec.rb
@@ -340,6 +343,7 @@ test_files:
 - spec/tessa/asset_spec.rb
 - spec/tessa/config_spec.rb
 - spec/tessa/controller_helpers_spec.rb
+- spec/tessa/jobs/migrate_assets_job_spec.rb
 - spec/tessa/model_field_spec.rb
 - spec/tessa/model_spec.rb
 - spec/tessa/rack_upload_proxy_spec.rb