kithe 2.0.0.pre.alpha2 → 2.0.0.pre.beta1

data/lib/shrine/plugins/kithe_derivatives.rb

@@ -0,0 +1,54 @@
+require 'mini_mime'
+
+class Shrine
+  module Plugins
+    # Includes the Shrine `derivatives` plugin with some configuration, and
+    # extra features. The metadata for shrine derivatives is stored in the same
+    # JSON as the main file.
+    #
+    # * default kithe storage location of :kithe_derivatives
+    #
+    # * nice metadata["filename"] for derivatives, instead of default shrine fairly
+    #   random (filename ends up used by default in content-disposition headers when delivered)
+    #
+    # * Includes kithe_persisted_derivatives with #add_persisted_derivatives
+    #   and #create_persisted_derivatives methods for concurrency-safe
+    #   derivative persisting.
+    #
+    # ## Shrine derivatives references
+    #
+    # https://shrinerb.com/docs/plugins/derivatives
+    # https://shrinerb.com/docs/processing
+    class KitheDerivatives
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :derivatives, storage: -> (derivative) do
+          # default derivatives storage to
+          :kithe_derivatives
+        end
+
+        uploader.plugin :kithe_persisted_derivatives
+        uploader.plugin :kithe_derivative_definitions
+      end
+
+      module InstanceMethods
+
+        # Override to fix "filename" metadata to be something reasonable, regardless
+        # of what if anything was the filename of the IO being attached. shrine S3 will
+        # insist on setting a default content-disposition with this filename.
+        def extract_metadata(io, derivative:nil, **context)
+          result = super
+
+          if derivative && context[:record] && result["mime_type"]
+            extension = MiniMime.lookup_by_content_type(result["mime_type"] || "")&.extension || "bin"
+            result["filename"] = "#{context[:record].friendlier_id}_#{derivative}.#{extension}"
+          end
+
+          result
+        end
+      end
+
+    end
+    register_plugin(:kithe_derivatives, KitheDerivatives)
+  end
+end
+

data/lib/shrine/plugins/kithe_determine_mime_type.rb

@@ -0,0 +1,39 @@
+class Shrine
+  module Plugins
+    # Custom kithe logic for determining mime type, using the shrine mime_type plugin.
+    #
+    # We start out using the `marcel` analyzer.
+    # Marcel analyzer is pure-ruby and fast. It's from Basecamp and is what
+    # ActiveStorage uses. It is very similar to :mimemagic (and uses mimemagic
+    # under the hood), but mimemagic seems not to be maintained with up to date
+    # magic db? https://github.com/minad/mimemagic/pull/66
+    #
+    # But marcel is not able to catch some of our MP3s as audio/mpeg. The
+    # `mediainfo` CLI is, and is one of the tools Harvard FITS uses.
+    # If marcel came up blank, AND we are configured to use mediainfo CLI
+    # (which by default we will be if it's available), we will try
+    # shelling out to mediainfo command line.
+    #
+    # https://github.com/MediaArea/MediaInfo
+    #
+    # Ensure that if mime-type can't be otherwise determined, it is assigned
+    # "application/octet-stream", basically the type for generic binary.
+    class KitheDetermineMimeType
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :determine_mime_type, analyzer: -> (io, analyzers) do
+          mime_type = analyzers[:marcel].call(io)
+
+
+          if Kithe.use_mediainfo && mime_type == "application/octet-stream" || mime_type.blank?
+            mime_type = Kithe::MediainfoAnalyzer.new.call(io)
+          end
+
+          mime_type = "application/octet-stream" if mime_type.blank?
+
+          mime_type
+        end
+      end
+    end
+    register_plugin(:kithe_determine_mime_type, KitheDetermineMimeType)
+  end
+end

data/lib/shrine/plugins/kithe_persisted_derivatives.rb

@@ -0,0 +1,161 @@
+class Shrine
+  module Plugins
+    # Some convenience methods for adding/changing derivatives in
+    # concurrency-safe manner:
+    #
+    #   * Won't make a change if the underlying original has changed
+    #     so doesn't match the one you wanted to remove.
+    #   * Won't over-write changes made concurrently in the db by other processes
+    #   * Will always make sure to clean up any temporary files on all error
+    #     and falure conditions.
+    #
+    # Shrine has some building blocks for this, which we use, but it's a bit tricky
+    # to put them together to be generically reliable, as we think we've done here.
+    #
+    # All these methods will cause your Asset model to be saved, because of how
+    # the shrine atomic helpers work. So these methods will by default raise
+    # a TypeError if your Asset model has any unrelated outstanding changes,
+    # but you can tell it to save anyway with `allow_other_changes: true`.
+    #
+    # ## Shrine references:
+    #
+    # https://shrinerb.com/docs/plugins/derivatives
+    # https://shrinerb.com/docs/processing
+    class KithePersistedDerivatives
+      module AttacherMethods
+        # Like the shrine `add_derivatives` method, but also *persists* the
+        # derivatives (saves to db), in a realiably concurrency-safe way.
+        #
+        # Generally can take any options that shrine `add_derivatives`
+        # can take, including custom `storage` or `metadata` arguments.
+        #
+        # Like shrine add_derivatives, it will assume the files passed in are
+        # temporary, and delete them for you. If you want to disable this behavior:
+        #
+        #     attacher.add_persisted_derivatives({key: io}, delete: false)
+        #
+        # In some cases the derivatives can't be persisted because the underlying
+        # database has changed such that they would not be applicable. In those
+        # cases `false` will be return value, otherwise returns the new derivatives
+        # just as shrine `add_derivatives`
+        #
+        # Because the concurrent-safe persistence method will save the associated model --
+        # and save without ActiveRecord validation -- it is not safe to
+        # add_persisted_derivatives on a model with other unsaved changes. The
+        # method will by default refuse to do so, throwing a TypeError. If you'd
+        # like to force it, pass `allow_other_changes: true` as an argument.
+        #
+        # Also takes care of deleting any replaced derivative files, that are no longer
+        # referenced by the model. Shrine by default does not do this:
+        # https://github.com/shrinerb/shrine/issues/468
+        #
+        # All deletions are inline. In general this could be a fairly expensive operation,
+        # it can be wise to do it in a bg job.
+        def add_persisted_derivatives(local_files, **options)
+          other_changes_allowed = !!options.delete(:allow_other_changes)
+          if record && !other_changes_allowed && record.changed?
+            raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
+          end
+
+          existing_derivative_files = nil
+
+          # upload to storage
+          new_derivatives = upload_derivatives(local_files, **options)
+
+          begin
+            atomic_persist do |reloaded_attacher|
+              # record so we can delete any replaced ones...
+              existing_derivative_files = map_derivative(reloaded_attacher.derivatives).collect { |path, file| file }
+
+              # make sure we don't override derivatives created in other jobs, by
+              # first using the current up-to-date derivatives from db,
+              # then merging our changes in on top.
+              set_derivatives(reloaded_attacher.derivatives)
+              merge_derivatives(new_derivatives)
+            end
+          rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound => e
+            # underlying file has changed or model has been deleted, inappropriate
+            # to add the derivatives, we can just silently drop them, but clean
+            # up after ourselves.
+            delete_derivatives(local_files) unless options[:delete] == false
+            delete_derivatives(new_derivatives)
+
+            return false
+          rescue StandardError => e
+            # unexpected error, clean up our files and re-raise
+            delete_derivatives(local_files) unless options[:delete] == false
+            delete_derivatives(new_derivatives)
+            raise e
+          end
+
+          # Take care of deleting from storage any derivatives that were replaced.
+          current_derivative_files = map_derivative(derivatives).collect { |path, file| file }
+          replaced_files = existing_derivative_files - current_derivative_files
+          delete_derivatives(replaced_files)
+
+          new_derivatives
+        end
+
+        # Like the shrine `create_derivatives` method, but persists the created derivatives
+        # to the database in a concurrency-safe way.
+        #
+        # Can take all options that shrine `create_derivatives` can take, including custom
+        # processors, custom storage key, and arbitrary custom processor arguments.
+        #
+        #     asset.file_attacher.create_persisted_derivatives
+        #     asset.file_attacher.create_persisted_derivatives(storage: :custom_key)
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives)
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, some_arg: "value")
+        #     asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, alternate_source_file)
+        #
+        # Also has an `allow_other_changes` argument, see #add_persisted_derivatives.
+        def create_persisted_derivatives(*args, storage: nil, allow_other_changes: false, **options)
+          return false unless file
+
+          local_files = process_derivatives(*args, **options)
+          add_persisted_derivatives(local_files, storage: storage, allow_other_changes: allow_other_changes)
+        end
+
+        # Kind of like built-in Shrine #remove_derivatives, but also takes care of
+        # persisting AND deleting the removed derivative file from storage --
+        # all in concurrency-safe way, including not making sure to overwrite
+        # any unrelated derivatives someone else was adding.
+        #
+        # Can take the same sorts of path arguments as Shrine derivative #remove_derivatives
+        #
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb)
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb)
+        #     asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb, allow_other_changes: true)
+        def remove_persisted_derivatives(*paths, **options)
+          return if paths.empty?
+
+          other_changes_allowed = !!options.delete(:allow_other_changes)
+          if record && !other_changes_allowed && record.changed?
+            raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
+          end
+
+          removed_derivatives = nil
+          atomic_persist do |reloaded_attacher|
+            set_derivatives(reloaded_attacher.derivatives)
+            removed_derivatives = remove_derivatives(*paths, delete: false)
+          end
+
+          if removed_derivatives
+            map_derivative(removed_derivatives) do |_, derivative|
+              derivative.delete if derivative
+            end
+          end
+
+          removed_derivatives
+        rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound
+          # original was already deleted or changed, the derivatives wer'e trying to delete.
+          # It should be fine to do nothing, the process that deleted or changed
+          # the model should already have deleted all these derivatives.
+          # But we'll return false as a signel.
+          return false
+        end
+      end
+    end
+    register_plugin(:kithe_persisted_derivatives, KithePersistedDerivatives)
+  end
+end

data/lib/shrine/plugins/kithe_promotion_callbacks.rb

@@ -24,6 +24,10 @@ class Shrine
     # was convenient and avoided confusion to isolate wrapping in a class method that can be used
     # anywhere, and only depends on args passed in, no implicit state anywhere.
     class KithePromotionCallbacks
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :kithe_promotion_directives
+      end
+
       # promotion logic differs somewhat in different modes of use (bg or inline promotion),
       # so we extract the wrapping logic here. Exactly what the logic wrapped is can
       # differ.

data/lib/shrine/plugins/kithe_promotion_directives.rb

@@ -89,15 +89,15 @@ class Shrine
         #     some_model.save!
         def set_promotion_directives(hash)
           # ActiveJob sometimes has trouble if there are symbols in there, somewhat
-          # unpredictably.
-          hash = hash.collect { |k, v| [k.to_s, v === Symbol ? v.to_s : v.to_s]}.to_h
+          # unpredictably. And for other reasons, standardize on everything a string.
+          hash = hash.collect { |k, v| [k.to_s, v.to_s]}.to_h
 
           unrecognized = hash.keys.collect(&:to_sym) - KithePromotionDirectives.allowed_promotion_directives
           unless unrecognized.length == 0
             raise ArgumentError.new("Unrecognized promotion directive key: #{unrecognized.join('')}")
           end
 
-          promotion_directives.merge!(hash)
+          context[:promotion_directives] = promotion_directives.merge(hash).freeze
         end
 
         # context[:promotion_directives], lazily initializing to hash for convenience.
@@ -105,6 +105,36 @@ class Shrine
           context[:promotion_directives] ||= {}
         end
       end
+
+      # VERY hacky way to try to preserve promotion_directives on Asset.reload.
+      #
+      # This may not be necessary in a future shrine version if shrine resolves
+      # issue. See: https://github.com/shrinerb/shrine/issues/463
+      #
+      # It is the activerecord plugin implementation that erases all shrine context
+      # (and thus our promotion directives) on reload.
+      # https://github.com/shrinerb/shrine/blob/b5fc2e1432e51e6fde87c120bc6cf6abeb286c68/lib/shrine/plugins/activerecord.rb#L56-L60
+      #
+      # It is quite tricky to override the activerecord plugin's own override, because
+      # of the way shrine does these overrides. We've figured out a pretty crazy way
+      # below.
+      module AttachmentMethods
+        def included(model)
+          super
+
+          original_reload = instance_method(:reload)
+
+          define_method :reload do |*args|
+            previous_promotion_directives = file_attacher.promotion_directives
+
+            result = original_reload.bind(self).call(*args)
+            file_attacher.set_promotion_directives(previous_promotion_directives)
+
+            result
+          end
+        end
+      end
+
     end
     register_plugin(:kithe_promotion_directives, KithePromotionDirectives)
   end

data/lib/shrine/plugins/kithe_storage_location.rb

@@ -2,25 +2,74 @@ require 'shrine/storage/url'
 
 class Shrine
   module Plugins
-    # Set file location to "asset/#{asset_uuid_id}/#{unique_file_id}" -- regardless of
-    # asset sub-class, since they all have unique ids, just all under asset/.
+    # Set custom storage locations/paths for both the original file which is the main
+    # file in the shrine attachment at Asset#file, and any shrine derivatives.
+    #
+    # Shrine's default is to just put both of these at top-level `[randomID].suffix`. We
+    # instead:
+    #
+    # ## Original file
+    #
+    # Stored at `asset/#{asset_uuid_id}/#{unique_file_id}.suffix` -- regardless of
+    # asset sub-class, since they all have unique ids, just all under asset/. (In retrospect,
+    # maybe shoudl have left `asset/` off, and let consumer specify a prefix when configuring
+    # storage).
     #
     # If no Asset pk is available (direct upload or unsaved Asset), will be stored just
     # under "asset/#{unique_file_id}.#{suffix}"
     #
     # We are choosing to store under Asset UUID PK instead of friendlier_id, friendlier_id
     # is good for public URLs and UI, but actual PK is more reliable/immutable.
+    #
+    # ## Derivatives
+    #
+    # Stored at `#{asset_uuid_id}/derivative_key/#{unique_file_id}.suffix`.
+    #
+    # If asset uuid pk is not available, will raise a TypeError and refuse to store
+    # derivative. (This may have to be thought through more.)
+    #
+    # If you want an additional prefix, supply it hwen configuring kithe_derivatives
+    # storage.
     module KitheStorageLocation
       module InstanceMethods
-        def generate_location(io, context)
+        def generate_location(io, derivative: nil, **context)
+          original = super
+
+          if derivative
+            _kithe_generate_derivative_location(io, original: original, derivative: derivative, **context)
+          else
+            _kithe_generate_main_location(io, original: original, **context)
+          end
+        end
+
+        private
+
+        def _kithe_generate_main_location(io, original:, **context)
           # If it doesn't have a id, we're probably storing in cache, possibly as part
           # of direct upload endpoint. A better path will be created on store.
           id = context[:record].id if context[:record].respond_to?(:id)
 
-          basename = super
+          basename = original
 
           ["asset", id, basename].compact.join("/")
         end
+
+        # Usually NOT in the same bucket/prefix as the originals/main attachments.
+        # You can set a prefix yourself in your shrine storage config if you want them
+        # on the same bucket, and probably should.
+        def _kithe_generate_derivative_location(io, original:, derivative:, record:, **context)
+          # for now to be save, insist the record exist and have an id so we can get the
+          # correct derivative location. This is consistent with kithe 1.x behavior. We can
+          # enhance later maybe.
+          unless record && record.id
+            raise TypeError.new("Can't determine correct derivative location without a persisted record. Record: #{record}")
+          end
+          unless derivative && original
+            raise ArgumentError.new("Missing required argument")
+          end
+
+          [record.id, derivative, original].join("/")
+        end
       end
     end
     register_plugin(:kithe_storage_location, KitheStorageLocation)

data/lib/tasks/kithe_tasks.rake

@@ -8,11 +8,10 @@ namespace :kithe do
     options = {}
     OptionParser.new do |opts|
       opts.banner = "Usage: ./bin/rake kithe:create_derivatives -- [options]"
-      opts.on("--derivatives TYPES", "comma-seperated list of type keys") { |ids| options[:derivative_keys] = ids.split(",")}
-      opts.on("--lazy","Lazy create") { options[:lazy] = true }
-      opts.on("--asset-id FRIENDLIER_IDS", "comma-seperated list of asset (friendlier) ids") { |ids| options[:asset_ids] = ids.split(",") }
-      opts.on("--work-id FRIENDLIER_IDS", "comma-seperated list of work (friendlier) ids") { |ids| options[:work_ids] = ids.split(",") }
-      opts.on("--mark-derivatives-created", "set derivatives_created? flag on assets") { |ids| options[:mark_derivatives_created] = true }
+      opts.on("--derivatives=TYPES", "comma-seperated list of type keys") { |ids| options[:derivative_keys] = ids.split(",")}
+      opts.on("--lazy", "Lazy create") { options[:lazy] = true }
+      opts.on("--asset-id=FRIENDLIER_IDS", "comma-seperated list of asset (friendlier) ids") { |ids| options[:asset_ids] = ids.split(",") }
+      opts.on("--work-id=FRIENDLIER_IDS", "comma-seperated list of work (friendlier) ids") { |ids| options[:work_ids] = ids.split(",") }
     end.tap do |parser|
       parser.parse!(parser.order(ARGV) {})
     end
@@ -22,17 +21,20 @@ namespace :kithe do
       scope = scope.joins(:parent).where("parents_kithe_models.friendlier_id":  options[:work_ids])
     end
     scope = scope.where(friendlier_id: options[:asset_ids]) if options[:asset_ids]
-    scope = scope.includes(:derivatives) if options[:lazy]
 
     progress_bar = ProgressBar.create(total: scope.count, format: Kithe::STANDARD_PROGRESS_BAR_FORMAT)
 
     scope.find_each do |asset|
-      progress_bar.title = asset.friendlier_id
-      asset.create_derivatives(
-        only: options[:derivative_keys],
-        lazy: !!options[:lazy],
-        mark_created: options[:mark_derivatives_created]
-      )
+      begin
+        progress_bar.title = asset.friendlier_id
+        asset.create_derivatives(
+          only: options[:derivative_keys],
+          lazy: !!options[:lazy]
+        )
+      rescue Shrine::FileNotFound => e
+        progress_bar.log("original missing for #{asset.friendlier_id}")
+        # it's cool, skip it
+      end
       progress_bar.increment
     end
   end
@@ -43,9 +45,14 @@ namespace :kithe do
     task :lazy_defaults => :environment do
       progress_bar = ProgressBar.create(total: Kithe::Asset.count, format: Kithe::STANDARD_PROGRESS_BAR_FORMAT)
 
-      Kithe::Asset.includes(:derivatives).find_each do |asset|
-        progress_bar.title = asset.friendlier_id
-        asset.create_derivatives(lazy: true)
+      Kithe::Asset.find_each do |asset|
+        begin
+          progress_bar.title = asset.friendlier_id
+          asset.create_derivatives(lazy: true)
+        rescue Shrine::FileNotFound => e
+          progress_bar.log("original missing for #{asset.friendlier_id}")
+          # it's cool, skip it
+        end
         progress_bar.increment
       end
     end