kithe 2.3.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f1252c0becbda9f754e23af2333aaae66cf40cbfd6bacdad0f97425cd317de6
-  data.tar.gz: 8460f704869d27d5a2011f746e88b9016fde465d1cbb023c883fe600b0510c1d
+  metadata.gz: f0ca2d6002aec0accb7264e06f3abcb71f33cf5673fa3ef27bfa035cbca67195
+  data.tar.gz: 9c5ffce36b1ff7a318059babe307627e23ccc00f6997f22a1fdc0d547cfd8a0f
 SHA512:
-  metadata.gz: '0890b072c3bfb9901f4c910fcaf0f288dd72dc8c8164202998dfabf2db4b3f7a09dfeaa8df6c4c9cd242b0c98ec084c48f5214d4a2c8f4991ee6d08643495857'
-  data.tar.gz: e7c2bbe6f3f32ce67c646289f038a06d51f371fdb87634657296546c84fe3cb52a216b0a3c6648669f73e7d61f51b56a3afb0d28584fc85950dbf0936d6f5eda
+  metadata.gz: 42129dea71f6944d2a70ca69cde8c5f1802c8915ce32cf1b237dbc6223edfcd9cefe776255cb2321c1978da8906a322e88ca443240915b6b2ee1eee53063cd42
+  data.tar.gz: 44478b610e5eb8abda0b385932fb8528c3f6b962f77be9168391d940dac26b739c4c9ec81f1b0912123f178c19b2e2837b1cc6e68483b7c8c59d67e40f1bb8dc

data/app/characterization/kithe/ffprobe_characterization.rb

@@ -0,0 +1,186 @@
+require 'tty/command'
+require 'json'
+
+module Kithe
+  # Characterizes Audio or Video files using `ffprobe`, a tool that comes with `ffmpeg`.
+  #
+  # You can pass in a local File object (with a pathname), a local String pathname, or
+  # a remote URL. (Remote URLs will be passed directy to ffprobe, which can efficiently
+  # fetch just the bytes it needs)
+  #
+  # You can get back normalized A/V metadata:
+  #
+  #     metadata = FfprobeCharacterization.new(url).normalized_metadata
+  #
+  # Normalized metadata is a *flat* hash of typed JSON-able values. It uses
+  # keys based on what the ActiveEncode gem seems to use, but adds some extras
+  # and makes a few tweaks. See the #normalized_metadata method source for
+  # keys supplied.
+  #
+  # Or the complete FFprobe response as JSON. (We try to use ffprobe options that
+  # are exhausitive as to what is returned, including ffprobe version(s))
+  #
+  #     ffprobe_results = FfprobeCharacterization.new(url).ffprobe_hash
+  #
+  # The class method .characterize_from_uploader can usefully extract a URL if possible
+  # or else execute with a file, such as from a shrine `add_metadata` block.
+  #
+  #     add_metadata do |source_io, **context|
+  #       Kithe::FfprobeCharacterization.characterize_from_uploader(source_io, context)
+  #     end
+  #
+  class FfprobeCharacterization
+    class_attribute :ffprobe_command, default: "ffprobe"
+    class_attribute :ffprobe_timeout, default: 10
+
+    attr_reader :input_arg
+
+    # @param input [String,File] local File OR local filepath as String, OR remote URL as string
+    #   If you have a remote url, just passing hte remote url is way more performant than
+    #   downloading it yourself locally -- ffprobe will just fetch the bytes it needs.
+    def initialize(input)
+      if input.respond_to?(:path)
+        input = input.path
+      end
+      @input_arg = input
+    end
+
+    # a helper for creating a block for shrine uploader, you can always use
+    # FFprobeCharecterization.new directly too!
+    #
+    # * Does not run on "cache" action, only on promotion (or manual execution).
+    #
+    # * Will run only on items with "audio/" or "video/" content-type.
+    #
+    # * By default only on main original, not derivatives, although
+    #   you can pass `run_on_derivatives: true` if desired.
+    #
+    # Will use ffprobe with direct URL if possible based on source_io (ffprobe
+    # can very efficiently access only bytes needed from URL), otherwise will
+    # download local temp copy if necessary.
+    #
+    #    class AssetUploader < Kithe::AssetUploader
+    #      add_metadata do |source_io, **context|
+    #        Kithe::FfprobeCharacterization.characterize_from_uploader(source_io, context)
+    #      end
+    #
+    #      #...
+    #    end
+    #
+    def self.characterize_from_uploader(source_io, add_metadata_context, run_on_derivatives: false)
+      # only for A/V please
+      return {} unless add_metadata_context.dig(:metadata, "mime_type")&.start_with?(%r{\A(audio|video)/})
+
+      # don't run on cache, only on promotion or manual trigger
+      return {} unless add_metadata_context[:action] != :cache
+
+      # don't run on derivatives unless option given
+      return {} unless add_metadata_context[:derivative].nil? || run_on_derivatives
+
+      # ffprobe can use a URL and very efficiently only retrieve what bytes it needs...
+      if source_io.respond_to?(:url) && source_io.url.start_with?(/\Ahttps?:/)
+        Kithe::FfprobeCharacterization.new(source_io.url).normalized_metadata
+      else
+        # if not already a file, will download, possibly slow, but gets us to go.
+        Shrine.with_file(source_io) do |file|
+          Kithe::FfprobeCharacterization.new(file.path).normalized_metadata
+        end
+      end
+    end
+
+    # ffprobe args come from this suggestion:
+    #
+    # https://gist.github.com/nrk/2286511?permalink_comment_id=2593200#gistcomment-2593200
+    #
+    # We also add in various current version tags! If we're going to record all ffprobe
+    # output, we'll want that too!
+    def ffprobe_options
+      [
+        "-hide_banner",
+        "-loglevel", "fatal",
+        "-show_error", "-show_format", "-show_streams", "-show_programs",
+        "-show_chapters", "-show_private_data", "-show_versions",
+        "-print_format", "json",
+      ]
+    end
+
+    # ffprobe output parsed as JSON...
+    def ffprobe_hash
+      @ffprobe_hash ||= JSON.parse(ffprobe_stdout).merge(
+        "ffprobe_options_used" => ffprobe_options.join(" ")
+      )
+    end
+
+    # Returns a FLAT JSON-able hash of normalized a/v metadata.
+    #
+    # Tries to standardize to what ActiveEncode uses, with some changes and additions.
+    # https://github.com/samvera-labs/active_encode/blob/42f5ed5427a39e56093a5e82123918c4b2619a47/lib/active_encode/technical_metadata.rb
+    #
+    # A video file or other container can have more than one audio or video stream in it, although
+    # this is somewhat unusual for our domain. For the stream-specific audio_ and video_ metadata
+    # returned, we just choose the *first* returned audio or video stream (which may be more or
+    # less arbitrary)
+    #
+    # See also #ffprobe_hash for complete ffprobe results
+    def normalized_metadata
+      # overall  audio_sample_rate are null, audio codec is wrong
+      @normalized_metadata ||= {
+        "width" => first_video_stream_json&.dig("width"),
+        "height" => first_video_stream_json&.dig("height"),
+        "frame_rate" => video_frame_rate_as_float, # frames per second
+        "duration_seconds" => ffprobe_hash&.dig("format", "duration")&.to_f&.round(3),
+        "audio_codec" => first_audio_stream_json&.dig("codec_name"),
+        "video_codec" => first_video_stream_json&.dig("codec_name"),
+        "audio_bitrate" => first_audio_stream_json&.dig("bit_rate")&.to_i, # in bps
+        "video_bitrate" => first_video_stream_json&.dig("bit_rate")&.to_i, # in bps
+        # extra ones not ActiveEncode
+        "bitrate" => ffprobe_hash.dig("format", "bit_rate")&.to_i, # overall bitrate of whole file in bps
+        "audio_sample_rate" => first_audio_stream_json&.dig("sample_rate")&.to_i, # in Hz
+        "audio_channels" => first_audio_stream_json&.dig("channels")&.to_i, # usually 1 or 2 (for stereo)
+        "audio_channel_layout" => first_audio_stream_json&.dig("channel_layout"), # stereo or mono or (dolby) 2.1, or something else.
+      }.compact
+    end
+
+    # just the ffprobe version please. This is also available
+    # in ffprobe_hash
+    def ffprobe_version
+      ffprobe_hash.dig("program_version", "version")
+    end
+
+    private
+
+    def ffprobe_stdout
+      @ffprobe_output ||= TTY::Command.new(printer: :null).run(
+                            ffprobe_command,
+                            *ffprobe_options,
+                            input_arg,
+                            timeout: ffprobe_timeout).out
+    end
+
+    def first_video_stream_json
+      @first_video_stream_json ||= ffprobe_hash["streams"].find { |stream| stream["codec_type"] == "video" }
+    end
+
+    def first_audio_stream_json
+      @first_audio_stream_json ||= ffprobe_hash["streams"].find { |stream| stream["codec_type"] == "audio" }
+    end
+
+    # There are a few different values we could choose here. We're going to choose
+    # `avg_frame_rate` == total duration / number of frames,
+    # vs (not chosen) `r_frame_rate ` "the lowest framerate with which all timestamps can be represented accurately (it is the least common multiple of all framerates in the stream)"
+    #
+    # (note this sometimes gets us not what we expected, like it gets us 29.78 fps instead of 29.97)
+    #
+    # Then we have to change it from numerator/denomominator to float truncated to two decimal places,
+    # which we let ruby rational do for us.
+    def video_frame_rate_as_float
+      avg_frame_rate = first_video_stream_json&.dig("avg_frame_rate")
+
+      return nil unless avg_frame_rate
+
+      return nil if avg_frame_rate.split("/")[1] == "0" # sometimes it returns '0/0', don't know why.
+
+      Rational(avg_frame_rate).to_f.round(2)
+    end
+  end
+end

data/app/derivative_transformers/kithe/ffmpeg_extract_jpg.rb

@@ -0,0 +1,106 @@
+require 'tty/command'
+
+module Kithe
+  # Creates a JPG screen capture using ffmpeg, by default with the `thumbnail`
+  # filter to choose a representative frame from the first minute or so.
+  #
+  # @example tempfile = FfmpegExtractJpg.new.call(shrine_uploaded_file)
+  # @example tempfile = FfmpegExtractJpg.new.call(url)
+  # @example tempfile = FfmpegExtractJpg.new(start_seconds: 60).call(shrine_uploaded_file)
+  # @example tempfile = FfmpegExtractJpg.new(start_seconds: 10, width_pixels: 420).call(shrine_uploaded_file)
+  class FfmpegExtractJpg
+    class_attribute :ffmpeg_command, default: "ffmpeg"
+    attr_reader :start_seconds, :frame_sample_size, :width_pixels
+
+    # @param start_seconds [Integer] seek to this point to find thumbnail. If it's
+    #   after the end of the video, you won't get a thumb back though! [Default 0]
+    #
+    # @param frame_sample_size [Integer,false,nil] argument passed to ffmpeg thumbnail filter,
+    #   how many frames to sample, starting at start_seconds, to choose representative
+    #   thumbnail. If set to false, thumbnail filter won't be used. If this one
+    #   goes past the end of the video, ffmpeg is fine with it. Set to `false` to
+    #   disable use of ffmpeg sample feature, and just use exact frame at start_seconds.
+    #
+    #   NOTE: This can consume significant RAM depending on value and video resolution.
+    #
+    #   [Default false, not operative]
+    #
+    # @width_pixels [Integer] output thumb at this width. aspect ratio will be
+    #   maintained. Warning, if it's larger than video original, ffmpeg will
+    #   upscale!  If set to nil, thumb will be output at original video
+    #   resolution. [Default nil]
+    def initialize(start_seconds: 0, frame_sample_size: false, width_pixels: nil)
+      @start_seconds = start_seconds
+      @frame_sample_size = frame_sample_size
+      @width_pixels = width_pixels
+    end
+
+
+
+    # @param input_arg [String,File,Shrine::UploadedFile] local File; String that
+    #   can be URL or local file path; or Shrine::UploadedFile. If Shrine::UploadedFile,
+    #   we'll try to get a URL from it if we can, otherwise use or make a local tempfile.
+    #   Most efficient is if we have a remote URL to give ffmpeg, one way or another!
+    #
+    # @returns [Tempfile] jpg extracted from movie
+    def call(input_arg)
+      if input_arg.kind_of?(Shrine::UploadedFile)
+        if input_arg.respond_to?(:url) && input_arg.url&.start_with?(/https?\:/)
+          _call(input_arg.url)
+        else
+          Shrine.with_file(input_arg) do |local_file|
+            _call(local_file.path)
+          end
+        end
+      elsif input_arg.respond_to?(:path)
+        _call(input_arg.path)
+      else
+        _call(input_arg.to_s)
+      end
+    end
+
+    private
+
+    # Internal implementation, after input has already been normalized to an
+    # string that can be an ffmpeg arg.
+    #
+    # @param ffmpeg_source_arg [String] filepath or URL. ffmpeg can take urls, which
+    # can be very efficient.
+    #
+    # @returns Tempfile pointing to a thumbnail
+    def _call(ffmpeg_source_arg)
+      tempfile = Tempfile.new(['temp_deriv', ".jpg"])
+
+      ffmpeg_args = produce_ffmpeg_args(input_arg: ffmpeg_source_arg, output_path: tempfile.path)
+
+      TTY::Command.new(printer: :null).run(*ffmpeg_args)
+
+      return tempfile
+    rescue StandardError => e
+      tempfile.unlink if tempfile
+      raise e
+    end
+
+    def produce_ffmpeg_args(input_arg:, output_path:)
+      ffmpeg_args = [ffmpeg_command, "-y"]
+
+      if start_seconds && start_seconds > 0
+        ffmpeg_args.concat ["-ss", start_seconds.to_i]
+      end
+
+      ffmpeg_args.concat ["-i", input_arg]
+
+      video_filter_parts = []
+      video_filter_parts << "thumbnail=#{frame_sample_size}" if (frame_sample_size || 0) > 1
+      video_filter_parts << "scale=#{width_pixels}:-1" if width_pixels
+
+      if video_filter_parts.present?
+        ffmpeg_args.concat ["-vf", video_filter_parts.join(',')]
+      end
+
+      ffmpeg_args.concat ["-frames:v",  "1"]
+
+      ffmpeg_args << output_path
+    end
+  end
+end

data/app/indexing/kithe/indexable/record_index_updater.rb

@@ -24,6 +24,9 @@ module Kithe
       #   it's nil, meaning we'll find the indexer to use from current thread settings,
       #   or global settings.
       #
+      #   (note: this design means we can't currently use traject processing_thread_pool for
+      #   concurrency, sorry.)
+      #
       # @param writer [Traject::Writer] Can pass i a custom Traject::Writer which the
       #   index representation will be sent to. By default it's nil, meaning we'll find
       #   the writer to use from current thread settings or global settings.

data/app/models/kithe/asset.rb

@@ -27,7 +27,8 @@ class Kithe::Asset < Kithe::Model
     to: :file, allow_nil: true
   delegate :stored?, to: :file_attacher
   delegate :set_promotion_directives, :promotion_directives, to: :file_attacher
-
+  # delegate "metadata" as #file_metadata
+  delegate :metadata, to: :file, prefix: true, allow_nil: true
 
   # will be sent to file_attacher.set_promotion_directives, provided by our
   # kithe_promotion_hooks shrine plugin.
@@ -42,48 +43,24 @@ class Kithe::Asset < Kithe::Model
   before_promotion :refresh_metadata_before_promotion
   after_promotion :schedule_derivatives
 
-  # A convenience to call file_attacher.create_persisted_derivatives (from :kithe_derivatives)
-  # to create derivatives with conncurrent access safety, with the :kithe_derivatives
-  # processor argument, to create derivatives defined using kithe_derivative_definitions.
-  #
-  # This is designed for use with kithe_derivatives processor, and has options only relevant
-  # to it, although could be expanded to take a processor argument in the future if needed.
-  #
-  # Create derivatives for every definition added to uploader/attacher with kithe_derivatives
-  # `define_derivative`. Ordinarily will create a definition for every definition
-  # that has not been marked `default_create: false`.
-  #
-  # But you can also pass `only` and/or `except` to customize the list of definitions to be created,
-  # possibly including some that are `default_create: false`.
-  #
-  # create_derivatives should be idempotent. If it has failed having only created some derivatives,
-  # you can always just run it again.
+  # Proxies to file_attacher.kithe_create_derivatives to create kithe managed derivatives.
   #
-  # Will normally re-create derivatives (per existing definitions) even if they already exist,
-  # but pass `lazy: false` to skip creating if a derivative with a given key already exists.
-  # This will use the asset `derivatives` association, so if you are doing this in bulk for several
-  # assets, you should eager-load the derivatives association for efficiency.
+  # This will include any derivatives you created with in the uploader with kithe
+  # Attacher.define_derivative, as well as any derivative processors you opted into
+  # kithe management with `kithe_include_derivatives_processors` in uploader.
   #
-  # ## Avoiding eager-download
+  # This is safe for concurrent updates to the underlying model, the derivatives will
+  # be consistent and not left orphaned.
   #
-  # Additionally, this has a feature to 'trick' shrine into not eager downloading
-  # the original before our kithe_derivatives processor has a chance to decide if it
-  # needs to create any derivatives (based on `lazy` arg), making no-op
-  # create_derivatives so much faster and more efficient, which matters when doing
-  # bulk operations say with our rake task.
-  #
-  # kithe_derivatives processor must then do a Shrine.with_file to make sure
-  # it's a local file, when needed.
-  #
-  # See https://github.com/shrinerb/shrine/issues/470
+  # By default it will create all derivatives configured for default generation,
+  # but you can customize by listing derivative keys with :only and :except options,
+  # and with :lazy option which, if true, only executes derivative creation if
+  # a given derivative doesn't already exist.
   #
+  # See more in docs for kithe at
+  # Shrine::Plugins::KitheDerivativeDefinitions::AttacherMethods#kithe_create_derivatives
   def create_derivatives(only: nil, except: nil, lazy: false)
-    source = file
-    return false unless source
-
-    local_files = file_attacher.process_derivatives(:kithe_derivatives, only: only, except: except, lazy: lazy)
-
-    file_attacher.add_persisted_derivatives(local_files)
+    file_attacher.kithe_create_derivatives(only: only, except: except, lazy: lazy)
   end
 
 
@@ -185,7 +162,8 @@ class Kithe::Asset < Kithe::Model
 
   # called by after_promotion hook
   def schedule_derivatives
-    return unless self.file_attacher.kithe_derivative_definitions.present? # no need to schedule if we don't have any
+    # no need to schedule if we don't have any
+    return unless self.file_attacher.kithe_derivative_definitions.present? || self.file_attacher.kithe_include_derivatives_processors.present?
 
     Kithe::TimingPromotionDirective.new(key: :create_derivatives, directives: file_attacher.promotion_directives) do |directive|
       if directive.inline?

data/app/models/kithe/model.rb

@@ -32,8 +32,11 @@ class Kithe::Model < ActiveRecord::Base
   # this should only apply to Works, but we define it here so we can preload it
   # when fetching all Kithe::Model. And it's to Kithe::Model so it can include
   # both Works and Assets. We do some app-level validation to try and make it used
-  # as intended.
-  has_many :members, class_name: "Kithe::Model", foreign_key: :parent_id, inverse_of: :parent, dependent: :destroy
+  # as intended. Members are by default ordered by position, then created_at.
+  has_many :members, -> { order(position: :asc, created_at: :asc) },
+    class_name: "Kithe::Model", foreign_key: :parent_id,
+    inverse_of: :parent, dependent: :destroy
+
   belongs_to :parent, class_name: "Kithe::Model", inverse_of: :members, optional: true
 
   # a self-referential many-to-many is a bit confusing, but our "contains" relation
@@ -130,11 +133,15 @@ class Kithe::Model < ActiveRecord::Base
         LIMIT 1;
     EOS
 
-    # trying to use a prepared statement, hoping it means performance advantage
+    # trying to use a prepared statement, hoping it means performance advantage,
+    # this is super undocumented
+
+    bind = ActiveRecord::Relation::QueryAttribute.new("m.id", self.representative_id, ActiveRecord::Type::Value.new)
+
     result = self.class.connection.select_all(
       recursive_cte,
       "set_leaf_representative",
-      [[nil, self.representative_id]],
+      [bind],
       preparable: true
     ).first.try(:dig, "id")
 

data/app/models/kithe/parameters.rb

@@ -63,11 +63,24 @@
 class Kithe::Parameters < ActionController::Parameters
   attr_reader :auto_allowed_keys
 
-  def initialize(hash = {})
-    if hash.respond_to?(:to_unsafe_h)
-      hash = hash.to_unsafe_h
+
+  # Rails 7 adds another initializer method, annoyingly
+  if Rails.version.split.first.to_i >= 7
+    def initialize(hash = {}, logging_context = {})
+      if hash.respond_to?(:to_unsafe_h)
+        hash = hash.to_unsafe_h
+      end
+
+      super(hash, logging_context)
+    end
+  else
+    def initialize(hash = {})
+      if hash.respond_to?(:to_unsafe_h)
+        hash = hash.to_unsafe_h
+      end
+
+      super(hash)
     end
-    super(hash)
   end
 
   def permit_attr_json(klass, except:nil)

data/lib/kithe/indexable_settings.rb

@@ -6,22 +6,21 @@ module Kithe
     def initialize(solr_url:, writer_class_name:, writer_settings:,
                    model_name_solr_field:, solr_id_value_attribute:, disable_callbacks: false,
                    batching_mode_batch_size: 100)
-      @solr_url = solr_url
       @writer_class_name = writer_class_name
       @writer_settings = writer_settings
       @model_name_solr_field = model_name_solr_field
       @solr_id_value_attribute = solr_id_value_attribute || 'id'
       @batching_mode_batch_size = batching_mode_batch_size
+
+      # use our local setter to set solr_url also in writer_settings
+      solr_url = solr_url
     end
 
-    # Use configured solr_url, and merge together with configured
-    # writer_settings
-    def writer_settings
-      if solr_url
-        { "solr.url" => solr_url }.merge(@writer_settings)
-      else
-        @writer_settings
-      end
+
+    # set solr_url also in writer_settings, cause it's expected there.
+    def solr_url=(v)
+      @solr_url = v
+      writer_settings["solr.url"] = v if writer_settings
     end
 
     # Turn writer_class_name into an actual Class object.

data/lib/kithe/version.rb

@@ -1,3 +1,3 @@
 module Kithe
-  VERSION = '2.3.0'
+  VERSION = '2.6.0'
 end

data/lib/shrine/plugins/kithe_derivative_definitions.rb

@@ -5,9 +5,25 @@ class Shrine
         # use Rails class_attribute to conveniently have a class-level place
         # to store our derivative definitions that are inheritable and overrideable.
         # We store it on the Attacher class, because that's where shrine
-        # puts derivative processor definitions, so seems appropriate.
+        # puts derivative processor definitions, so seems appropriate. Normally
+        # not touched directly by non-kithe code.
         uploader::Attacher.class_attribute :kithe_derivative_definitions, instance_writer: false, default: []
 
+        # Kithe exersizes lifecycle control over derivatives, normally just the
+        # shrine processor labelled :kithe_derivatives. But you can opt additional shrine
+        # derivative processors into kithe control by listing their labels in this attribute.
+        #
+        # @example
+        #
+        #     class AssetUploader < Kithe::AssetUploader
+        #       Attacher.kithe_include_derivatives_processors += [:my_processor]
+        #       Attacher.derivatives(:my_processor) do |original|
+        #         derivatives
+        #       end
+        #     end
+        #
+        uploader::Attacher.class_attribute :kithe_include_derivatives_processors, instance_writer: false, default: []
+
         # Register our derivative processor, that will create our registered derivatives,
         # with our custom options.
         #
@@ -31,11 +47,12 @@ class Shrine
         #
         # The most basic definition consists of a derivative key, and a ruby block that
         # takes the original file, transforms it, and returns a ruby File or other
-        # (shrine-compatible) IO-like object. It will usually be done inside a custom Asset
-        # class definition.
+        # (shrine-compatible) IO-like object. It will usually be done inside your custom
+        # AssetUploader class definition.
         #
-        #     class Asset < Kithe::Asset
-        #       define_derivative :thumbnail do |original_file|
+        #     class AssetUploader < Kithe::AssetUploader
+        #       Attacher.define_derivative :thumbnail do |original_file|
+        #         someTempFileOrOtherIo
         #       end
         #     end
         #
@@ -52,7 +69,7 @@ class Shrine
         # will be passed in. You can then get the model object from `attacher.record`, or the
         # original file as a `Shrine::UploadedFile` object with `attacher.file`.
         #
-        #     define_derivative :thumbnail do |original_file, attacher:|
+        #     Attacher.define_derivative :thumbnail do |original_file, attacher:|
         #        attacher.record.title, attacher.file.width, attacher.file.content_type # etc
         #     end
         #
@@ -62,7 +79,7 @@ class Shrine
         # remain, and be accessible, where they were created; there is no built-in solution at present
         # for moving them).
         #
-        #     define_derivative :thumbnail, storage_key: :my_thumb_storage do |original| # ...
+        #     Attacher.define_derivative :thumbnail, storage_key: :my_thumb_storage do |original| # ...
         #
         # You can also set `default_create: false` if you want a particular definition not to be
         # included in a no-arg `asset.create_derivatives` that is normally triggered on asset creation.
@@ -101,6 +118,86 @@ class Shrine
           end.freeze
         end
       end
+
+      module AttacherMethods
+
+
+        # Similar to shrine create_derivatives, but with kithe standards:
+        #
+        # * Will call the :kithe_derivatives processor (that handles any define_derivative definitions),
+        #   plus any processors you've configured with kithe_include_derivatives_processors
+        #
+        # * Uses the methods added by :kithe_persisted_derivatives to add derivatives completely
+        #   concurrency-safely, if the model had it's attachment changed concurrently, you
+        #   won't get derivatives attached that belong to old version of original attachment,
+        #   and won't get any leftover "orphaned" derivatives either.
+        #
+        # The :kithe_derivatives processor has additional logic and options for determining
+        # *which* derivative definitions -- created with `define_deriative` will be executed:
+        #
+        # * Ordinarily will create a definition for every definition that has not been marked
+        #  `default_create: false`.
+        #
+        # * But you can also pass `only` and/or `except` to customize the list of definitions to be created,
+        #   possibly including some that are `default_create: false`.
+        #
+        # * Will normally re-create derivatives (per existing definitions) even if they already exist,
+        #   but pass `lazy: false` to skip creating if a derivative with a given key already exists.
+        #   This will use the asset `derivatives` association, so if you are doing this in bulk for several
+        #   assets, you should eager-load the derivatives association for efficiency.
+        #
+        # If you've added any custom processors with `kithe_include_derivatives_processors`, it's
+        # your responsibility to make them respect those options. See #process_kithe_derivative?
+        # helper method.
+        #
+        # create_derivatives should be idempotent. If it has failed having only created some derivatives,
+        # you can always just run it again.
+        #
+        def kithe_create_derivatives(only: nil, except: nil, lazy: false)
+          return false unless file
+
+          local_files = self.process_derivatives(:kithe_derivatives, only: only, except: except, lazy: lazy)
+
+          # include any other configured processors
+          self.kithe_include_derivatives_processors.each do |processor|
+            local_files.merge!(
+              self.process_derivatives(processor.to_sym, only: only, except: except, lazy: lazy)
+            )
+          end
+
+          self.add_persisted_derivatives(local_files)
+        end
+
+        # a helper method that you can use in your own shrine processors to
+        # handle only/except/lazy guarding logic.
+        #
+        # @return [Boolean] should the `key` be processed based on only/except/lazy conditions?
+        #
+        # @param key [Symbol] derivative key to check for guarded processing
+        # @param only [Array<Symbol>] If present, method will only return true if `key` is included in `only`
+        # @param except [Array<Symbol] If present, method will only return true if `key` is NOT included in `except`
+        # @param lazy [Boolean] If true, method will only return true if derivative key is not already present
+        #   in attacher with a value.
+        #
+        def process_kithe_derivative?(key, **options)
+          key = key.to_sym
+          only = options[:only] && Array(options[:only]).map(&:to_sym)
+          except = options[:except] && Array(options[:except]).map(&:to_sym)
+          lazy = !! options[:lazy]
+
+          (only.nil? ? true : only.include?(key)) &&
+          (except.nil? || ! except.include?(key)) &&
+          (!lazy || !derivatives.keys.include?(key))
+        end
+
+        # Convenience to check #process_kithe_derivative? for multiple keys at once,
+        # @return true if any key returns true
+        #
+        # @example process_any_kithe_derivative?([:thumb_mini, :thumb_large], only: [:thumb_large, :thumb_mega], lazy: true)
+        def process_any_kithe_derivative?(keys, **options)
+          keys.any? { |k| process_kithe_derivative?(k, **options) }
+        end
+      end
     end
     register_plugin(:kithe_derivative_definitions, KitheDerivativeDefinitions)
   end

data/lib/shrine/plugins/kithe_persisted_derivatives.rb

@@ -134,6 +134,10 @@ class Shrine
         def remove_persisted_derivatives(*paths, **options)
           return if paths.empty?
 
+          # Shrine does weird things if we pass in Strings, let's save ourselves
+          # the terrible debugging on that mistake, and noramlize to symbols
+          paths = paths.collect(&:to_sym)
+
           other_changes_allowed = !!options.delete(:allow_other_changes)
           if record && !other_changes_allowed && record.changed?
             raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kithe
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.6.0
 platform: ruby
 authors:
 - Jonathan Rochkind
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-02 00:00:00.000000000 Z
+date: 2022-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -19,7 +19,7 @@ dependencies:
         version: 5.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 5.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.2'
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: attr_json
   requirement: !ruby/object:Gem::Requirement
@@ -345,6 +345,8 @@ files:
 - README.md
 - Rakefile
 - app/assets/config/kithe_manifest.js
+- app/characterization/kithe/ffprobe_characterization.rb
+- app/derivative_transformers/kithe/ffmpeg_extract_jpg.rb
 - app/derivative_transformers/kithe/ffmpeg_transformer.rb
 - app/derivative_transformers/kithe/vips_cli_image_to_jpeg.rb
 - app/helpers/kithe/form_helper.rb
@@ -424,7 +426,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Shareable tools/components for building a digital collections app in Rails.