saviour 0.4.14 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1dae3217796105e752f538dcbb2b5eb38fdd55c15e3c428f8cc38e91e0636fec
-  data.tar.gz: 7a44ec336be6447a20b72855e90fa72fb8adfedb4c16e83187f7c38e8cc742e2
+  metadata.gz: '04095eaf0410d74e00e023a53cc473e8b5f2898f49673a329127d446a87f0080'
+  data.tar.gz: a3cba3ccb7d59f0f0e219475f003f21f102205132d3bc2cfb9ccc2ca6b0b5c54
 SHA512:
-  metadata.gz: 0017f5ba9358177d91dfb6b4509bbe68a25fc9fb021d8fce7d8f349eaf5cc53c06754e3f0d78a0a96d2f0c77952b7a14d8cf913cf971a43b8ce9a86228a411c1
-  data.tar.gz: a01dc0d1e49bc3aca7b468e51b3916edd81025268183faca3744a19ce55a0a0ba4b3a78c753b9c3c89cec20b384376c281f9942457c884071d99050bc8a786aa
+  metadata.gz: 83428891e963c4a96bffe2865380bbd586af945029e23d50c86c8b6aacac99f2ee7b928d89383bba0e1845175396e0b6f0c972d18e2df03390579cbf197518f0
+  data.tar.gz: 3ba8709c6d680f74eb933c3336b3954cd284b8bd52445db2cab72e2c58e1534152667a482b5cab2b5c8cf49a95a63e22f2a0a33e4cebf1becd03f3e1dddc65a6

data/lib/saviour/base_uploader.rb

@@ -5,6 +5,7 @@ module Saviour
   class BaseUploader
     def initialize(opts = {})
       @data = opts.fetch(:data, {})
+      @stash = {}
     end
 
     def method_missing(name, *args, &block)
@@ -51,6 +52,14 @@ module Saviour
       throw(:halt_process)
     end
 
+    def stash(hash)
+      @stash.merge!(hash)
+    end
+
+    def stashed
+      @stash
+    end
+
     def store_dir
       @store_dir ||= Uploader::StoreDirExtractor.new(self).store_dir
     end
@@ -76,10 +85,17 @@ module Saviour
         process(name, opts, :file, &block)
       end
 
-
       def store_dir(name = nil, &block)
         store_dirs.push(name || block)
       end
+
+      def after_upload(&block)
+        after_upload_hooks.push(block)
+      end
+
+      def after_upload_hooks
+        @after_upload ||= []
+      end
     end
   end
 end

data/lib/saviour/config.rb

@@ -39,6 +39,14 @@ module Saviour
           end
         end
       end
+
+      def concurrent_workers
+        @concurrent_workers || 4
+      end
+
+      def concurrent_workers=(x)
+        @concurrent_workers = x
+      end
     end
   end
 end

data/lib/saviour/db_helpers.rb

@@ -47,22 +47,22 @@ module Saviour
 
     class << self
 
-      def run_after_commit(&block)
-        unless ActiveRecord::Base.connection.current_transaction.open?
+      def run_after_commit(connection = ActiveRecord::Base.connection, &block)
+        unless connection.current_transaction.open?
           raise NotInTransaction, 'Trying to use `run_after_commit` but no transaction is currently open.'
         end
 
         dummy = CommitDummy.new(block)
-        ActiveRecord::Base.connection.add_transaction_record(dummy)
+        connection.add_transaction_record(dummy)
       end
 
-      def run_after_rollback(&block)
-        unless ActiveRecord::Base.connection.current_transaction.open?
+      def run_after_rollback(connection = ActiveRecord::Base.connection, &block)
+        unless connection.current_transaction.open?
           raise NotInTransaction, 'Trying to use `run_after_commit` but no transaction is currently open.'
         end
 
         dummy = RollbackDummy.new(block)
-        ActiveRecord::Base.connection.add_transaction_record(dummy)
+        connection.add_transaction_record(dummy)
       end
     end
   end

data/lib/saviour/file.rb

@@ -176,9 +176,6 @@ module Saviour
       !@source && !persisted?
     end
 
-
-    private
-
     def uploader
       @uploader ||= @uploader_klass.new(data: { model: @model, attached_as: @attached_as })
     end

data/lib/saviour/life_cycle.rb

@@ -1,5 +1,80 @@
 module Saviour
   class LifeCycle
+    class FileCreator
+      def initialize(current_path, file, column, connection)
+        @file = file
+        @column = column
+        @current_path = current_path
+        @connection = connection
+      end
+
+      def upload
+        @new_path = @file.write
+
+        return unless @new_path
+
+        DbHelpers.run_after_rollback(@connection) do
+          Config.storage.delete(@new_path)
+        end
+
+        [@column, @new_path]
+      end
+
+      def uploader
+        @file.uploader
+      end
+    end
+
+    class FileUpdater
+      def initialize(current_path, file, column, connection)
+        @file = file
+        @column = column
+        @current_path = current_path
+        @connection = connection
+      end
+
+      def upload
+        dup_temp_path = SecureRandom.hex
+
+        dup_file = proc do
+          Config.storage.cp @current_path, dup_temp_path
+
+          DbHelpers.run_after_commit(@connection) do
+            Config.storage.delete dup_temp_path
+          end
+
+          DbHelpers.run_after_rollback(@connection) do
+            Config.storage.mv dup_temp_path, @current_path
+          end
+        end
+
+        @new_path = @file.write(
+          before_write: ->(path) { dup_file.call if @current_path == path }
+        )
+
+        return unless @new_path
+
+        if @current_path && @current_path != @new_path
+          DbHelpers.run_after_commit(@connection) do
+            Config.storage.delete(@current_path)
+          end
+        end
+
+        # Delete the newly uploaded file only if it's an update in a different path
+        if @current_path.nil? || @current_path != @new_path
+          DbHelpers.run_after_rollback(@connection) do
+            Config.storage.delete(@new_path)
+          end
+        end
+
+        [@column, @new_path]
+      end
+
+      def uploader
+        @file.uploader
+      end
+    end
+
     def initialize(model, persistence_klass)
       raise ConfigurationError, "Please provide an object compatible with Saviour." unless model.class.respond_to?(:attached_files)
 
@@ -16,71 +91,50 @@ module Saviour
     end
 
     def create!
-      attached_files.each do |column|
-        next unless @model.send(column).changed?
-
-        persistence_layer = @persistence_klass.new(@model)
-        new_path = @model.send(column).write
-
-        if new_path
-          persistence_layer.write(column, new_path)
-
-          DbHelpers.run_after_rollback do
-            Config.storage.delete(new_path)
-          end
-        end
-      end
+      process_upload(FileCreator)
     end
 
     def update!
-      attached_files.each do |column|
-        next unless @model.send(column).changed?
-
-        update_file(column)
-      end
+      process_upload(FileUpdater)
     end
 
-
     private
 
-
-    def update_file(column)
+    def process_upload(klass)
       persistence_layer = @persistence_klass.new(@model)
-      current_path = persistence_layer.read(column)
-      dup_temp_path = SecureRandom.hex
 
-      dup_file = proc do
-        Config.storage.cp current_path, dup_temp_path
-
-        DbHelpers.run_after_commit do
-          Config.storage.delete dup_temp_path
-        end
+      uploaders = attached_files.map do |column|
+        next unless @model.send(column).changed?
 
-        DbHelpers.run_after_rollback do
-          Config.storage.mv dup_temp_path, current_path
-        end
-      end
+        klass.new(
+          persistence_layer.read(column),
+          @model.send(column),
+          column,
+          ActiveRecord::Base.connection
+        )
+      end.compact
 
-      new_path = @model.send(column).write(
-        before_write: ->(path) { dup_file.call if current_path == path }
-      )
+      pool = Concurrent::FixedThreadPool.new(Saviour::Config.concurrent_workers)
+      futures = uploaders.map { |uploader| Concurrent::Future.execute(executor: pool) { uploader.upload } }
 
-      if new_path
-        persistence_layer.write(column, new_path)
+      pool.shutdown
+      pool.wait_for_termination
 
-        if current_path && current_path != new_path
-          DbHelpers.run_after_commit do
-            Config.storage.delete(current_path)
-          end
+      result = futures.map do |x|
+        x.value.tap do
+          raise(x.reason) if x.rejected?
         end
+      end.compact
 
-        # Delete the newly uploaded file only if it's an update in a different path
-        if current_path.nil? || current_path != new_path
-          DbHelpers.run_after_rollback do
-            Config.storage.delete(new_path)
-          end
+      attrs = result.to_h
+
+      uploaders.map(&:uploader).select { |x| x.class.after_upload_hooks.any? }.each do |uploader|
+        uploader.class.after_upload_hooks.each do |hook|
+          uploader.instance_exec(uploader.stashed, &hook)
         end
       end
+
+      persistence_layer.write_attrs(attrs) if attrs.length > 0
     end
 
     def attached_files

data/lib/saviour/persistence_layer.rb

@@ -12,6 +12,10 @@ module Saviour
       @model.update_columns(attr => value)
     end
 
+    def write_attrs(attributes)
+      @model.update_columns(attributes)
+    end
+
     def persisted?
       @model.persisted? || @model.destroyed?
     end

data/lib/saviour/version.rb

@@ -1,3 +1,3 @@
 module Saviour
-  VERSION = "0.4.14"
+  VERSION = "0.5.0"
 end

data/lib/saviour.rb

@@ -16,6 +16,7 @@ require 'saviour/db_helpers'
 
 require 'tempfile'
 require 'fileutils'
+require 'concurrent/future'
 
 module Saviour
   NoActiveRecordDetected = Class.new(StandardError)

data/saviour.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "activerecord", ">= 5.0"
   spec.add_dependency "activesupport", ">= 5.0"
+  spec.add_dependency "concurrent-ruby", ">= 1.0.5"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rspec"

data/spec/feature/concurrent_processors_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+describe "concurrent processors" do
+  before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
+
+  WAIT_TIME = 0.5
+
+  let(:uploader) {
+    Class.new(Saviour::BaseUploader) {
+      store_dir { "/store/dir" }
+
+      process_with_file do |file, filename|
+        sleep WAIT_TIME
+
+        [file, filename]
+      end
+    }
+  }
+
+  let(:klass) {
+    klass = Class.new(Test) { include Saviour::Model }
+    klass.attach_file :file, uploader
+    klass.attach_file :file_thumb, uploader
+    klass.attach_file :file_thumb_2, uploader
+    klass.attach_file :file_thumb_3, uploader
+    klass
+  }
+
+  context 'on update' do
+    it 'works concurrently with 4 workers' do
+      a = klass.create!
+
+      Saviour::Config.concurrent_workers = 4
+
+      t0 = Time.now
+      a.update_attributes! file: Saviour::StringSource.new("contents", "file.txt"),
+                           file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                           file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                           file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      diff = Time.now - t0
+      expect(diff).to be_within(0.05).of(WAIT_TIME)
+      expect(diff).to be < WAIT_TIME * 4
+    end
+
+    it 'works in serial with 1 worker' do
+      a = klass.create!
+
+      Saviour::Config.concurrent_workers = 1
+
+      t0 = Time.now
+      a.update_attributes! file: Saviour::StringSource.new("contents", "file.txt"),
+                           file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                           file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                           file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      expect(Time.now - t0).to be >= WAIT_TIME * 4
+    end
+
+    it 'concurrency can be adjusted' do
+      a = klass.create!
+
+      Saviour::Config.concurrent_workers = 2
+
+      t0 = Time.now
+      a.update_attributes! file: Saviour::StringSource.new("contents", "file.txt"),
+                           file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                           file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                           file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      diff = Time.now - t0
+      expect(diff).to be_within(0.05).of(WAIT_TIME * 2)
+      expect(diff).to be < WAIT_TIME * 4
+    end
+  end
+
+  context 'on create' do
+    it 'works concurrently with 4 workers' do
+      Saviour::Config.concurrent_workers = 4
+
+      t0 = Time.now
+      klass.create! file: Saviour::StringSource.new("contents", "file.txt"),
+                    file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                    file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                    file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      diff = Time.now - t0
+      expect(diff).to be_within(0.05).of(WAIT_TIME)
+      expect(diff).to be < WAIT_TIME * 4
+    end
+
+    it 'works in serial with 1 worker' do
+      Saviour::Config.concurrent_workers = 1
+
+      t0 = Time.now
+      klass.create! file: Saviour::StringSource.new("contents", "file.txt"),
+                    file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                    file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                    file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      expect(Time.now - t0).to be >= WAIT_TIME * 4
+    end
+
+    it 'concurrency can be adjusted' do
+      Saviour::Config.concurrent_workers = 2
+
+      t0 = Time.now
+      klass.create! file: Saviour::StringSource.new("contents", "file.txt"),
+                    file_thumb: Saviour::StringSource.new("contents", "file_2.txt"),
+                    file_thumb_2: Saviour::StringSource.new("contents", "file_3.txt"),
+                    file_thumb_3: Saviour::StringSource.new("contents", "file_4.txt")
+
+      diff = Time.now - t0
+      expect(diff).to be_within(0.05).of(WAIT_TIME * 2)
+      expect(diff).to be < WAIT_TIME * 4
+    end
+  end
+end

data/spec/feature/crud_workflows_spec.rb

@@ -103,6 +103,26 @@ describe "saving a new file" do
         a.save
       end
     end
+
+    context do
+      let(:klass) {
+        a = Class.new(Test) { include Saviour::Model }
+        a.attach_file :file, uploader
+        a.attach_file :file_thumb, uploader
+        a
+      }
+
+      it "saves to db only once with multiple file attachments" do
+        # 1 create + 1 update with two attributes
+        expected_query = %Q{UPDATE "tests" SET "file" = '/store/dir/file.txt', "file_thumb" = '/store/dir/file.txt'}
+        expect_to_yield_queries(count: 2, including: [expected_query]) do
+          klass.create!(
+            file: Saviour::StringSource.new("foo", "file.txt"),
+            file_thumb: Saviour::StringSource.new("foo", "file.txt")
+          )
+        end
+      end
+    end
   end
 
   describe "deletion" do
@@ -142,6 +162,28 @@ describe "saving a new file" do
       a.update_attributes! file: Saviour::StringSource.new("foo", "file.txt")
       expect(Saviour::Config.storage.read(a[:file])).to eq "foo"
     end
+
+    context do
+      let(:klass) {
+        a = Class.new(Test) { include Saviour::Model }
+        a.attach_file :file, uploader
+        a.attach_file :file_thumb, uploader
+        a
+      }
+
+      it "saves to db only once with multiple file attachments" do
+        a = klass.create!
+
+        # 2 update's, first empty and then 1 with the two attributes
+        expected_query = %Q{UPDATE "tests" SET "file" = '/store/dir/file.txt', "file_thumb" = '/store/dir/file.txt'}
+        expect_to_yield_queries(count: 2, including: [expected_query]) do
+          a.update_attributes!(
+            file: Saviour::StringSource.new("foo", "file.txt"),
+            file_thumb: Saviour::StringSource.new("foo", "file.txt")
+          )
+        end
+      end
+    end
   end
 
   describe "dupping" do

data/spec/feature/processors_api_spec.rb

@@ -72,4 +72,30 @@ describe "processor's API" do
       end
     end
   end
+
+  describe "errors raised from processors are propagated" do
+    let(:uploader) {
+      Class.new(Saviour::BaseUploader) do
+        store_dir { "/store/dir/#{model.id}" }
+
+        process do |contents, filename|
+          raise "custom problem!"
+        end
+      end
+    }
+
+    it "on update" do
+      a = klass.create!
+
+      expect {
+        a.update_attributes! file: Saviour::StringSource.new("contents", "filename.txt")
+      }.to raise_error.with_message("custom problem!")
+    end
+
+    it "on create" do
+      expect {
+        klass.create! file: Saviour::StringSource.new("contents", "filename.txt")
+      }.to raise_error.with_message("custom problem!")
+    end
+  end
 end

data/spec/feature/stash_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+describe "stash data on process" do
+  before { allow(Saviour::Config).to receive(:storage).and_return(Saviour::LocalStorage.new(local_prefix: @tmpdir, public_url_prefix: "http://domain.com")) }
+
+  it 'stores data on after upload on update' do
+    uploader = Class.new(Saviour::BaseUploader) {
+      store_dir { "/store/dir" }
+
+      process_with_file do |file, filename|
+        stash(file_size: File.size(file.path))
+
+        [file, filename]
+      end
+
+      after_upload do |stash|
+        model.update_attributes!(file_size: stash[:file_size])
+      end
+    }
+
+    klass = Class.new(Test) { include Saviour::Model }
+    klass.attach_file :file, uploader
+
+    a = klass.create!
+
+    a.update_attributes! file: Saviour::StringSource.new("a" * 74, "file.txt")
+    expect(a.file_size).to eq 74
+  end
+
+  it 'stores data on after upload on create' do
+    uploader = Class.new(Saviour::BaseUploader) {
+      store_dir { "/store/dir" }
+
+      process_with_file do |file, filename|
+        stash(file_size: File.size(file.path))
+
+        [file, filename]
+      end
+
+      after_upload do |stash|
+        model.update_attributes!(file_size: stash[:file_size])
+      end
+    }
+
+    klass = Class.new(Test) { include Saviour::Model }
+    klass.attach_file :file, uploader
+
+    a = klass.create! file: Saviour::StringSource.new("a" * 74, "file.txt")
+
+    expect(a.file_size).to eq 74
+  end
+
+  it 'stashes are independent per uploader' do
+    uploader = Class.new(Saviour::BaseUploader) {
+      store_dir { "/store/dir" }
+
+      process_with_file do |file, filename|
+        # same ':size' key in the stash
+        stash(size: File.size(file.path))
+
+        [file, filename]
+      end
+
+      after_upload do |stash|
+        model.update_attributes!("size_#{attached_as}" => stash[:size])
+      end
+    }
+
+    klass = Class.new(Test) { include Saviour::Model }
+    klass.attach_file :file, uploader
+    klass.attach_file :file_thumb, uploader
+
+    a = klass.create!
+
+    # - 1 initial empty query
+    # - 2 queries to update size
+    # - 1 query to assign stored paths
+    expect_to_yield_queries(count: 4) do
+      a.update_attributes! file: Saviour::StringSource.new("a" * 74, "file.txt"),
+                           file_thumb: Saviour::StringSource.new("a" * 31, "file_2.txt")
+    end
+
+    expect(a.size_file).to eq 74
+    expect(a.size_file_thumb).to eq 31
+  end
+end

data/spec/spec_helper.rb

@@ -7,15 +7,17 @@ require 'active_record'
 require 'sqlite3'
 require 'get_process_mem'
 
+require 'support/active_record_asserts'
+
 require File.expand_path("../../lib/saviour", __FILE__)
 
 connection_opts = case ENV.fetch('DB', "sqlite")
                     when "sqlite"
-                      {adapter: "sqlite3", database: ":memory:"}
+                      { adapter: "sqlite3", database: ":memory:" }
                     when "mysql"
-                      {adapter: "mysql2", database: "saviour", username: "root", encoding: "utf8"}
+                      { adapter: "mysql2", database: "saviour", username: "root", encoding: "utf8" }
                     when "postgres"
-                      {adapter: "postgresql", database: "saviour", username: "postgres"}
+                      { adapter: "postgresql", database: "saviour", username: "postgres" }
                   end
 
 ActiveRecord::Base.establish_connection(connection_opts)
@@ -44,6 +46,7 @@ RSpec.configure do |config|
 
   config.before do
     Test.delete_all
+    Saviour::Config.concurrent_workers = 4
   end
 end
 

data/spec/support/active_record_asserts.rb

@@ -0,0 +1,35 @@
+module ActiveRecordAssertions
+  def expect_to_yield_queries(count: nil, including: [])
+    AssertionsTracker.clear!
+    yield
+
+    expect(AssertionsTracker.data.size).to eq(count) if count
+
+    including.each do |query|
+      expect(AssertionsTracker.data).to include a_string_matching(Regexp.new(query))
+    end
+  end
+end
+
+module AssertionsTracker
+  def self.data
+    @data ||= []
+  end
+
+  def self.clear!
+    @data = []
+  end
+end
+
+RSpec.configure do |config|
+  config.before do
+    AssertionsTracker.clear!
+  end
+
+  config.include ActiveRecordAssertions
+end
+
+ActiveSupport::Notifications.subscribe "sql.active_record" do |name, started, finished, unique_id, data|
+  AssertionsTracker.data.push(data[:sql]) if data[:name] == "SQL"
+end
+

data/spec/support/schema.rb

@@ -3,7 +3,11 @@ ActiveRecord::Schema.define do
     t.string :file
     t.string :file_thumb
     t.string :file_thumb_2
+    t.string :file_thumb_3
     t.string :name
+    t.integer :file_size
+    t.integer :size_file
+    t.integer :size_file_thumb
     t.timestamps null: false
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: saviour
 version: !ruby/object:Gem::Version
-  version: 0.4.14
+  version: 0.5.0
 platform: ruby
 authors:
 - Roger Campos
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-06 00:00:00.000000000 Z
+date: 2018-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.5
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -186,6 +200,7 @@ files:
 - lib/saviour/version.rb
 - saviour.gemspec
 - spec/feature/allow_overriding_attached_as_method_spec.rb
+- spec/feature/concurrent_processors_spec.rb
 - spec/feature/crud_workflows_spec.rb
 - spec/feature/dirty_spec.rb
 - spec/feature/follow_file_spec.rb
@@ -199,6 +214,7 @@ files:
 - spec/feature/remove_attachment_spec.rb
 - spec/feature/reopens_file_at_every_process_spec.rb
 - spec/feature/rewind_source_before_read_spec.rb
+- spec/feature/stash_spec.rb
 - spec/feature/transactional_behavior_spec.rb
 - spec/feature/uploader_declaration_spec.rb
 - spec/feature/validations_spec.rb
@@ -211,6 +227,7 @@ files:
 - spec/models/s3_storage_spec.rb
 - spec/models/url_source_spec.rb
 - spec/spec_helper.rb
+- spec/support/active_record_asserts.rb
 - spec/support/data/camaloon.jpg
 - spec/support/data/example.xml
 - spec/support/data/text.txt