kithe 2.0.0.pre.alpha2 → 2.0.0.pre.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '087115597d14a7b5ac33827a00f62fb37a37d7186deea82f3d6204e09a77d0b4'
-  data.tar.gz: a5f87fdf80a1f521999fcb5a039bcd28831e1427f71e2590729327bac356f148
+  metadata.gz: cfaa6ce3c41abdc0779014cb028d3ae7acbbfee6124eb24abca06f52e6f60c30
+  data.tar.gz: 4f3951e52b4175c86933870933fd38c0c84dfcf11a5ad8cd18f1eb8c479305b0
 SHA512:
-  metadata.gz: 57c51b54e947477b1f69426caa6adc9beb37ea72c25f6210874e2e1837c4ad0a98ad4d43322ee5fddfa2c6e7fb2f311b01c05f9d58435785a3774180225bf23c
-  data.tar.gz: 1ed33f2a15c07ce47959e21345694815820ed8daa11cf04c8cfdfa9fb274f95ecc56cf326a87c07112adc765f04bd55970ab54efea7f8e44b8a14ca2f8508393
+  metadata.gz: 6fe1836bde7b33a29d95324afe745ceeecb59ca95c8952775cce39dd8233c8daf171da971cc4a7bb2691e8dad2b91d50766c9232e3dce5b333da6f14d2f8d6bd
+  data.tar.gz: de9bcd5fc0c22c18d9dde7ea9ec1bdc708e08035ee0f5d208131628d6d37bbc61574ae213b0ef69e6161c1fbdf334aad545c734614e10cee8f63f96666ed86dc

data/app/jobs/kithe/create_derivatives_job.rb

@@ -1,7 +1,7 @@
 module Kithe
   class CreateDerivativesJob < Job
-    def perform(asset, mark_created: nil)
-        asset.create_derivatives(mark_created: mark_created)
+    def perform(asset)
+        asset.create_derivatives
     end
     # This error typically occurs when several large assets, whose derivatives
     # take a long time to generate, are deleted immediately after ingest.

data/app/models/kithe/asset.rb

@@ -1,5 +1,5 @@
 class Kithe::Asset < Kithe::Model
-  has_many :derivatives, foreign_key: "asset_id", inverse_of: "asset", dependent: :destroy # dependent destroy to get shrine destroy logic for assets
+  include Kithe::Asset::SetShrineUploader
 
   # These associations exist for hetereogenous eager-loading, but hide em.
   # They are defined as self-pointing below.
@@ -26,15 +26,12 @@ class Kithe::Asset < Kithe::Model
     :md5, :sha1, :sha512,
     to: :file, allow_nil: true
   delegate :stored?, to: :file_attacher
-  delegate :set_promotion_directives, to: :file_attacher
+  delegate :set_promotion_directives, :promotion_directives, to: :file_attacher
 
-  after_save :remove_invalid_derivatives
 
-  # will be sent to file_attacher.promotion_directives=, provided by our
+  # will be sent to file_attacher.set_promotion_directives, provided by our
   # kithe_promotion_hooks shrine plugin.
-  class_attribute :promotion_directives, instance_writer: false, default: {}
-
-  class_attribute :derivative_definitions, instance_writer: false, default: []
+  class_attribute :promotion_directives, instance_accessor: false, default: {}
 
   # Callbacks are called by our kiteh_promotion_callbacks shrine plugin, around
   # shrine promotion. A before callback can cancel promotion with the usual
@@ -45,112 +42,94 @@ class Kithe::Asset < Kithe::Model
   before_promotion :refresh_metadata_before_promotion
   after_promotion :schedule_derivatives
 
-
-
-  # Establish a derivative definition that will be used to create a derivative
-  # when #create_derivatives is called, for instance automatically after promotion.
+  # A convenience to call file_attacher.create_persisted_derivatives (from :kithe_derivatives)
+  # to create derivatives with conncurrent access safety, with the :kithe_derivatives
+  # processor argument, to create derivatives defined using kithe_derivative_definitions.
   #
-  # The most basic definition consists of a derivative key, and a ruby block that
-  # takes the original file, transforms it, and returns a ruby File or other
-  # (shrine-compatible) IO-like object. It will usually be done inside a custom Asset
-  # class definition.
+  # This is designed for use with kithe_derivatives processor, and has options only relevant
+  # to it, although could be expanded to take a processor argument in the future if needed.
   #
-  #     class Asset < Kithe::Asset
-  #       define_derivative :thumbnail do |original_file|
-  #       end
-  #     end
+  # Create derivatives for every definition added to uploader/attacher with kithe_derivatives
+  # `define_derivative`. Ordinarily will create a definition for every definition
+  # that has not been marked `default_create: false`.
   #
-  # The original_file passed in will be a ruby File object that is already open for reading. If
-  # you need a local file path for your transformation, just use `original_file.path`.
+  # But you can also pass `only` and/or `except` to customize the list of definitions to be created,
+  # possibly including some that are `default_create: false`.
   #
-  # The return value can be any IO-like object. If it is a ruby File or Tempfile,
-  # that temporary file will be deleted for you after the derivative has been created. If you
-  # have to make any intermediate files, you are responsible for cleaning them up. Ruby stdlib
-  # Tempfile and Dir.mktmpdir may be useful.
+  # create_derivatives should be idempotent. If it has failed having only created some derivatives,
+  # you can always just run it again.
   #
-  # If in order to do your transformation you need additional information about the original,
-  # just add a `record:` keyword argument to your block, and the Asset object will be passed in:
+  # Will normally re-create derivatives (per existing definitions) even if they already exist,
+  # but pass `lazy: false` to skip creating if a derivative with a given key already exists.
+  # This will use the asset `derivatives` association, so if you are doing this in bulk for several
+  # assets, you should eager-load the derivatives association for efficiency.
   #
-  #     define_derivative :thumbnail do |original_file, record:|
-  #        record.width, record.height, record.content_type # etc
-  #     end
+  # ## Avoiding eager-download
   #
-  # Derivatives are normally uploaded to the Shrine storage labeled :kithe_derivatives,
-  # but a definition can specify an alternate Shrine storage id. (specified shrine storage key
-  # is applied on derivative creation; if you change it with existing derivatives, they should
-  # remain, and be accessible, where they were created; there is no built-in solution at present
-  # for moving them).
+  # Additionally, this has a feature to 'trick' shrine into not eager downloading
+  # the original before our kithe_derivatives processor has a chance to decide if it
+  # needs to create any derivatives (based on `lazy` arg), making no-op
+  # create_derivatives so much faster and more efficient, which matters when doing
+  # bulk operations say with our rake task.
   #
-  #     define_derivative :thumbnail, storage_key: :my_thumb_storage do |original| # ...
+  # kithe_derivatives processor must then do a Shrine.with_file to make sure
+  # it's a local file, when needed.
   #
-  # You can also set `default_create: false` if you want a particular definition not to be
-  # included in a no-arg `asset.create_derivatives` that is normally triggered on asset creation.
+  # See https://github.com/shrinerb/shrine/issues/470
   #
-  # And you can set content_type to either a specific type like `image/jpeg` (or array of such) or a general type
-  # like `image`, if you want to define a derivative generation routine for only certain types.
-  # If multiple blocks for the same key are defined, with different content_type restrictions,
-  # the most specific one will be used.  That is, for a JPG, `image/jpeg` beats `image` beats no restriction.
-  def self.define_derivative(key, storage_key: :kithe_derivatives, content_type: nil, default_create: true, &block)
-    # Make sure we dup the array to handle sub-classes on class_attribute
-    self.derivative_definitions = self.derivative_definitions.dup.push(
-      DerivativeDefinition.new(
-        key: key,
-        storage_key: storage_key,
-        content_type: content_type,
-        default_create: default_create,
-        proc: block
-      )
-    )
-  end
+  def create_derivatives(only: nil, except: nil, lazy: false)
+    source = file
+    return false unless source
 
-  # Returns all derivative keys with a definition, as array of strings
-  def self.defined_derivative_keys
-    self.derivative_definitions.collect(&:key).uniq.collect(&:to_s)
-  end
+    #local_files = file_attacher.process_derivatives(:kithe_derivatives, only: only, except: except, lazy: lazy)
+    local_files = _process_kithe_derivatives_without_download(source, only: only, except: except, lazy: lazy)
 
-  # If you have a subclass that has inherited derivative definitions, you can
-  # remove them -- only by key, will remove any definitions with that key regardless
-  # of content_type restrictions.
-  #
-  # This could be considered rather bad OO design, you might want to consider
-  # a different class hieararchy where you don't have to do this. But it's here.
-  def self.remove_derivative_definition!(*keys)
-    keys = keys.collect(&:to_sym)
-    self.derivative_definitions = self.derivative_definitions.reject do |defn|
-      keys.include?(defn.key.to_sym)
-    end
+    file_attacher.add_persisted_derivatives(local_files)
   end
 
-  # Create derivatives for every definition added with `define_derivative. Ordinarily
-  # will create a definition for every definition that has not been marked `default_create: false`.
+  # Working around Shrine's insistence on pre-downloading original before calling derivative processor.
+  # We want to avoid that, so when our `lazy` argument is in use, original does not get eagerly downloaded,
+  # but only gets downloaded if needed to make derivatives.
   #
-  # But you can also pass `only` and/or `except` to customize the list of definitions to be created,
-  # possibly including some that are `default_create: false`.
+  # This is a somewhat hacky way to do that, loking at the internals of shrine `process_derivatives`,
+  # and pulling them out to skip the parts we don't want. We also lose shrine instrumentation
+  # around this action.
   #
-  # create_derivatives should be idempotent. If it has failed having only created some derivatives,
-  # you can always just run it again.
+  # See: https://github.com/shrinerb/shrine/issues/470
   #
-  # Will normally re-create derivatives (per existing definitions) even if they already exist,
-  # but pass `lazy: false` to skip creating if a derivative with a given key already exists.
-  # This will use the asset `derivatives` association, so if you are doing this in bulk for several
-  # assets, you should eager-load the derivatives association for efficiency.
-  def create_derivatives(only: nil, except: nil, lazy: false, mark_created: nil)
-    DerivativeCreator.new(derivative_definitions, self, only: only, except: except, lazy: lazy, mark_created: mark_created).call
+  # If that were resolved, the 'ordinary' shrine thing would be to replace calls
+  # to this local private method with:
+  #
+  #     file_attacher.process_derivatives(:kithe_derivatives, only: only, except: except, lazy: lazy)
+  #
+  private def _process_kithe_derivatives_without_download(source, **options)
+    processor = file_attacher.class.derivatives_processor(:kithe_derivatives)
+    local_files = file_attacher.instance_exec(source, **options, &processor)
   end
 
-  # Adds an associated derivative with key and io bytestream specified.
+  # Just a convennience for file_attacher.add_persisted_derivatives (from :kithe_derivatives),
+  # feel free to use that if you want to add more than one etc.  By default stores to
+  # :kithe_derivatives, just as `add_persisted_derivatives` does.
+  #
+  # Note that just like shrine's own usual `add_derivative(s)`, it assumes any files
+  # you pass it are meant to be temporary and will delete them, unless you pass
+  # `delete: false`.
+  #
+  # Adds an associated derivative with key and io bytestream specified,
+  # doing so in a way that is safe from race conditions under multi-process
+  # concurrency.
+  #
   # Normally you don't use this with derivatives defined with `define_derivative`,
   # this is used by higher-level API. But if you'd like to add a derivative not defined
   # with `define_derivative`, or for any other reason would like to manually add
-  # a derivative, this is public API meant for that.
+  # a derivative.
   #
-  # Ensures safe from race conditions under multi-thread/process concurrency, to
-  # make sure any existing derivative with same key is atomically replaced,
-  # and if the asset#file is changed to a different bytestream (compared to what's in memory
-  # for this asset), we don't end up saving a derivative based on old one.
+  # Can specify any options normally allowed for kithe `add_persisted_derivatives`,
+  # which are also generally any allowed for shrine `add_derivative`.
   #
-  # Can specify any metadata values to be force set on the Derivative#file, and
-  # a specific Shrine storage key (defaults to :kithe_derivatives shrine storage)
+  #     asset.update_derivative("big_thumb", File.open(something))
+  #     asset.update_derivative("something", File.open(something), delete: false)
+  #     asset.update_derivative("something", File.open(something), storage_key: :registered_storage, metadata: { "foo": "bar" })
   #
   # @param key derivative-type identifying key
   # @param io An IO-like object (according to Shrine), bytestream for the derivative
@@ -158,30 +137,20 @@ class Kithe::Asset < Kithe::Model
   # @param metadata an optional hash of key/values to set as default metadata for the Derivative#file
   #   shrine object.
   #
-  # @return [Derivative] the Derivative created, or nil if it was not created because no longer
+  # @return [Shrine::UploadedFile] the Derivative created, or false if it was not created because no longer
   #   applicable (underlying Asset#file has changed in db)
-  def update_derivative(key, io, storage_key: :kithe_derivatives, metadata: {})
-    DerivativeUpdater.new(self, key, io, storage_key: storage_key, metadata: metadata).update.tap do |result|
-      self.derivatives.reset if result
-    end
+  def update_derivative(key, io, **options)
+    result = update_derivatives({ key => io }, **options)
+    result && result.values.first
   end
 
-  def remove_derivative(key)
-    if association(:derivatives).loaded?
-      derivatives.find_all { |d| d.key == key.to_s }.each do |deriv|
-        derivatives.delete(deriv)
-      end
-    else
-      Kithe::Derivative.where(key: key.to_s, asset: self).each do |deriv|
-        deriv.destroy!
-      end
-    end
+  def update_derivatives(deriv_hash, **options)
+    file_attacher.add_persisted_derivatives(deriv_hash, **options)
   end
 
-  # Just finds the Derivative object matching supplied key. if you're going to be calling
-  # this on a list of Asset objects, make sure to preload :derivatives association.
-  def derivative_for(key)
-    derivatives.find {|d| d.key == key.to_s }
+  # just a convenience for kithe remove_persisted_derivatives
+  def remove_derivatives(*keys)
+    file_attacher.remove_persisted_derivatives(*keys)
   end
 
   # Runs the shrine promotion step, that we normally have in backgrounding, manually
@@ -205,41 +174,6 @@ class Kithe::Asset < Kithe::Model
     file_attacher.atomic_promote(**context)
   end
 
-  # The derivative creator sets metadata when it's created all derivatives
-  # defined as `default_create`. So we can tell you if it's done or not.
-  def derivatives_created?
-    if file
-      !!file.metadata["derivatives_created"]
-    end
-  end
-
-  # Take out a DB lock on the asset with unchanged sha512 saved in metadata. If a lock
-  # can't be acquired -- which would be expected to be because the asset has already changed
-  # and has a new sha for some reason -- returns nil.
-  #
-  # Useful for making a change to an asset making sure it applies to a certain original file.
-  #
-  # Needs to be done in a transaction, and you should keep the transaction SMALL AS POSSIBLE.
-  # We can't check to make sure you're in a transaction reliably because of Rails transactional
-  # tests, you gotta do it!
-  #
-  # This method is mostly intended for internal Kithe use, cause it's a bit tricky.
-  def acquire_lock_on_sha
-    raise ArgumentError.new("Can't acquire lock without sha512 in metadata") if self.sha512.blank?
-
-    Kithe::Asset.where(id: self.id).where("file_data -> 'metadata' ->> 'sha512' = ?", self.sha512).lock.first
-  end
-
-  # Intentionally only true if there WAS a sha512 before AND it's changed.
-  # Allowing false on nil previous sha512 allows certain conditions, mostly only
-  # in testing, where you want to assign a derivative to not-yet-promoted file.
-  def saved_change_to_file_sha?
-    saved_change_to_file_data? &&
-      saved_change_to_file_data.first.try(:dig, "metadata", "sha512") != nil &&
-      saved_change_to_file_data.first.try(:dig, "metadata", "sha512") !=
-        saved_change_to_file_data.second.try(:dig, "metadata", "sha512")
-  end
-
   # An Asset is it's own representative
   def representative
     self
@@ -252,8 +186,10 @@ class Kithe::Asset < Kithe::Model
 
   def initialize(*args)
     super
-    if promotion_directives.present?
-      file_attacher.set_promotion_directives(promotion_directives)
+
+    # copy class-level global promotion directives as initial instance value
+    if self.class.promotion_directives.present?
+      self.set_promotion_directives(self.class.promotion_directives)
     end
   end
 
@@ -261,7 +197,7 @@ class Kithe::Asset < Kithe::Model
 
   # called by after_promotion hook
   def schedule_derivatives
-    return unless self.derivative_definitions.present? # no need to schedule if we don't have any
+    return unless self.file_attacher.kithe_derivative_definitions.present? # no need to schedule if we don't have any
 
     Kithe::TimingPromotionDirective.new(key: :create_derivatives, directives: file_attacher.promotion_directives) do |directive|
       if directive.inline?
@@ -276,12 +212,4 @@ class Kithe::Asset < Kithe::Model
   def refresh_metadata_before_promotion
     file.refresh_metadata!(promoting: true)
   end
-
-  # Meant to be called in after_save hook, looks at activerecord dirty tracking in order
-  # to removes all derivatives if the asset sha512 has changed
-  def remove_invalid_derivatives
-    if saved_change_to_file_sha?
-      derivatives.destroy_all
-    end
-  end
 end

data/app/models/kithe/asset/derivative_creator.rb

@@ -1,6 +1,6 @@
 # Creates derivatives from definitions stored on an Asset class
 class Kithe::Asset::DerivativeCreator
-  attr_reader :definitions, :asset, :only, :except, :lazy, :mark_created
+  attr_reader :definitions, :shrine_attacher, :only, :except, :lazy, :source_io
 
   # A helper class that provides the implementation for Kithe::Asset#create_derivatives,
   # normally only expected to be called from there.
@@ -13,57 +13,62 @@ class Kithe::Asset::DerivativeCreator
   # (deleted) if it is a File or Tempfile object.
   #
   # @param definitions an array of DerivativeDefinition
-  # @param asset an Asset instance
+  # @param shrine_attacher a shrine attacher instance holding attachment state from an individual
+  #   model, from eg `asset.file_attacher`
   # @param only array of definition keys, only execute these (doesn't matter if they are `default_create` or not)
   # @param except array of definition keys, exclude these from definitions of derivs to be created
   # @param lazy (default false), Normally we will create derivatives for all applicable definitions,
   #   overwriting any that already exist for a given key. If the definition has changed, a new
   #   derivative created with new definition will overwrite existing. However, if you pass lazy false,
   #   it'll skip derivative creation if the derivative already exists, which can save time
-  #   if you are only intending to create missing derivatives.  With lazy:false, the asset
-  #   derivatives association will be consulted, so should be eager-loaded if you are going
-  #   to be calling on multiple assets.
-  # @param mark_created [Boolean] if true will set shrine metadata indicating we've done
-  #   derivative creation phase, so Asset#derivatives_created? will return true. Defaults to nil,
-  #   meaning true if and only if `only` is nil -- mark created if creating default derivatives.
-  def initialize(definitions, asset, only:nil, except:nil, lazy: false, mark_created: :not_set)
+  #   if you are only intending to create missing derivatives.
+  def initialize(definitions, source_io:, shrine_attacher:, only:nil, except:nil, lazy: false)
     @definitions = definitions
-    @asset = asset
-    @only = only && Array(only)
-    @except = except && Array(except)
+    @source_io = source_io
+    @shrine_attacher = shrine_attacher
+    @only = only && Array(only).collect(&:to_sym)
+    @except = except && Array(except).collect(&:to_sym)
     @lazy = !!lazy
-    @mark_created = mark_created.nil? ? (only.nil? && except.nil?) : !! mark_created
+
+    unless shrine_attacher.kind_of?(Shrine::Attacher)
+      raise ArgumentError.new("expect second arg Shrine::Attacher not #{shrine_attacher.class}")
+    end
   end
 
   def call
-    return unless asset.file.present? # if no file, can't create derivatives
+    return unless shrine_attacher.file.present? # if no file, can't create derivatives
 
     definitions_to_create = applicable_definitions
+
     if lazy
-      existing_derivative_keys = asset.derivatives.collect(&:key).collect(&:to_s)
+      existing_derivative_keys = shrine_attacher.derivatives.keys
       definitions_to_create.reject! do |defn|
-        existing_derivative_keys.include?(defn.key.to_s)
+        existing_derivative_keys.include?(defn.key)
       end
     end
 
-    return unless definitions_to_create.present?
+    return {} unless definitions_to_create.present?
+
+    derivatives = {}
 
-    # Note, MAY make a superfluous copy and/or download of original file, ongoing
-    # discussion https://github.com/shrinerb/shrine/pull/329#issuecomment-443615868
-    # https://github.com/shrinerb/shrine/pull/332
-    Shrine.with_file(asset.file) do |original_file|
+    # Make sure we have this as a local file, because many processors
+    # require it, for instance for shelling out to command line.
+    # This might trigger a download, but note we are only doing it if we have
+    # `definitions_to_create`, otherwise we already returned.
+    shrine_attacher.shrine_class.with_file(source_io) do |source_io_as_file|
       definitions_to_create.each do |defn|
-        deriv_bytestream = defn.call(original_file: original_file, record: asset)
+        deriv_bytestream = defn.call(original_file: source_io_as_file, attacher: shrine_attacher)
 
         if deriv_bytestream
-          asset.update_derivative(defn.key, deriv_bytestream, storage_key: defn.storage_key)
-          cleanup_returned_io(deriv_bytestream)
+          derivatives[defn.key] =  deriv_bytestream
         end
 
-        original_file.rewind
+        # may not need this but it doesn't hurt...
+        source_io.rewind
       end
-      mark_derivatives_created! if mark_created
     end
+
+    derivatives
   end
 
   private
@@ -83,7 +88,7 @@ class Kithe::Asset::DerivativeCreator
     candidates = definitions.find_all do |defn|
       (only.nil? ? defn.default_create : only.include?(defn.key)) &&
       (except.nil? || ! except.include?(defn.key)) &&
-      defn.applies_to?(asset)
+      defn.applies_to_content_type?(shrine_attacher.file.content_type)
     end
 
     # Now we gotta filter out any duplicate keys based on our priority rules, but keep
@@ -107,39 +112,4 @@ class Kithe::Asset::DerivativeCreator
     # Now we uniq keeping last defn
     candidates.reverse.uniq {|d| d.key }.reverse
   end
-
-  def cleanup_returned_io(io)
-    if io.respond_to?(:close!)
-      # it's a Tempfile, clean it up now
-      io.close!
-    elsif io.is_a?(File)
-      # It's a File, close it and delete it.
-      io.close
-      File.unlink(io.path)
-    end
-  end
-
-  # Sets kithe asset metadata "derivatives_created" to `true`, so
-  # code can know that we're finished creating all `default_create`
-  # derivatives.
-  #
-  # Uses a db-level atomic jsonb update and db-locking to make sure it can do this
-  # without overwriting any other metadata changes, safely.
-  def mark_derivatives_created!
-    asset.transaction do
-      unless asset.acquire_lock_on_sha
-        # asset bytestream has changed
-        return nil
-      end
-
-      sql = <<~SQL
-        UPDATE "#{Kithe::Asset.table_name}"
-        SET file_data = jsonb_set(file_data, '{metadata, derivatives_created}', 'true')
-        WHERE id = '#{asset.id}'
-      SQL
-
-      #ActiveRecord::Base.connection.exec_update("update table set f1=#{ActiveRecord::Base.sanitize(f1)}")
-      Kithe::Asset.connection.execute(sql)
-    end
-  end
 end