kithe 2.0.0.pre.alpha2 → 2.0.0.pre.beta1

data/app/models/kithe/asset/derivative_definition.rb

@@ -2,38 +2,37 @@
 # class, it's what's created when you call Kithe::Asset#define_derivative
 class Kithe::Asset::DerivativeDefinition
   attr_reader :key, :content_type, :default_create, :proc, :storage_key
-  def initialize(key:, storage_key:, proc:, content_type: nil, default_create: true)
-    @key = key
+  def initialize(key:, proc:, content_type: nil, default_create: true)
+    @key = key.to_sym
     @content_type = content_type
-    @storage_key = storage_key
     @default_create = default_create
     @proc = proc
   end
 
-  def call(original_file:,record:)
-    if proc_accepts_record_keyword?
-      proc.call(original_file, record: record)
+  def call(original_file:,attacher:)
+    if proc_accepts_keyword?(:attacher)
+      proc.call(original_file, attacher: attacher)
     else
       proc.call(original_file)
     end
   end
 
   # Do content-type restrictions defined for this definition match a given asset?
-  def applies_to?(asset)
+  def applies_to_content_type?(original_content_type)
     return true if content_type.nil?
 
-    return true if content_type == asset.content_type
+    return true if content_type == original_content_type
 
-    return false if asset.content_type.nil?
+    return false if original_content_type.nil?
 
-    return true if (content_type.kind_of?(Array) && content_type.include?(asset.content_type))
+    return true if (content_type.kind_of?(Array) && content_type.include?(original_content_type))
 
-    content_type == asset.content_type.sub(%r{/.+\Z}, '')
+    content_type == original_content_type.sub(%r{/.+\Z}, '')
   end
 
   private
 
-  def proc_accepts_record_keyword?
-    proc.parameters.include?([:key, :record]) || proc.parameters.include?([:keyreq, :record])
+  def proc_accepts_keyword?(kwarg)
+    proc.parameters.include?([:key, kwarg]) || proc.parameters.include?([:keyreq, kwarg]) || proc.parameters.find {|a| a.first == :keyrest}
   end
 end

data/app/models/kithe/asset/set_shrine_uploader.rb

@@ -0,0 +1,64 @@
+# Our Kithe::Asset model class is meant to be a superclass of a local application asset class, which we
+# can call `Asset`, although an app can call it whatever they like.
+#
+# Kithe::Asset sets it's own shrine uploader class, with a typical shrine:
+#
+#     include Kithe::AssetUploader::Attachment(:file)
+#
+# An application Asset subclass will inherit this uploader, which is convenient for getting
+# started quickly. But an application will likely want to define its own local uploader
+# class, to define it's own metadata, derivatives, and any other custom beahvior.
+#
+# There isn't an obvious built-into-shrine way to do that, but it turns out simply overriding
+# class and instance `*_attacher` methods seems to work out well. See:
+# https://discourse.shrinerb.com/t/model-sub-classes-with-uploader-sub-classes/208
+#
+# So a local application can define it's own shrine uploader, which is highly recommended to
+# be a sub-class of Kithe::AssetUploader to ensure it has required and useful
+# Kithe behavior:
+#
+#     # ./app/uploaders/asset_uploader.rb
+#     class AssetUploader < Kithe::AssetUploader
+#       # maybe we want some custom metadata
+#       add_metadata :something do |io|
+#         whatever
+#       end
+#     end
+#
+# And then set it in ti's custom local Asset class:
+#
+#     # ./app/models/asset.rb
+#     class Asset < Kithe::Asset
+#       set_shrine_uploader(AssetUploader)
+#     end
+#
+# If a local app has it's own inheritance hieararchy of children below that (eg) Asset class,
+# they can each (optionally) also override with a custom Uploader. It is recommended that
+# the Uploader inheritance hieararchy match the model inheritance hieararchy, to have
+# all behavior consistent. For instance:
+#
+#     class AudioAssetUploader < AssetUploader
+#     end
+#
+#     class AudioAsset < Asset
+#       set_shrine_uploader(AudioAssetUploader)
+#     end
+#
+module Kithe::Asset::SetShrineUploader
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def set_shrine_uploader(uploader_class)
+      subclass_attachment = uploader_class::Attachment.new(:file)
+
+      define_singleton_method :file_attacher do |**options|
+        subclass_attachment.send(:class_attacher, **options)
+      end
+
+      define_method :file_attacher do |**options|
+        subclass_attachment.send(:attacher, self, **options)
+      end
+    end
+  end
+
+end

data/app/models/kithe/collection.rb

@@ -1,14 +1,8 @@
 class Kithe::Collection < Kithe::Model
-  # Collections don't have derivatives, but we want to allow Rails eager loading
-  # of association on hetereogenous fetches of Kithe::Model, so this is clever.
-  has_many :derivatives, -> { none }
-  private :derivatives, :derivatives=, :derivative_ids, :derivative_ids=
-
   after_initialize do
     self.kithe_model_type = "collection" if self.kithe_model_type.nil?
   end
   before_validation do
     self.kithe_model_type = "collection" if self.kithe_model_type.nil?
   end
-
 end

data/app/models/kithe/model.rb

@@ -10,15 +10,6 @@ class Kithe::Model < ActiveRecord::Base
   include AttrJson::Record::Dirty
   include Kithe::Indexable
 
-  # A handy scope for eager-loading all representatives and all of their derivatives.
-  #
-  # Works on hetereogenous collections of Works and Assets -- the Assets need
-  # :derivatives directly referenced (since they don't really have a leaf_representative assoc),
-  # the works need :leaf_representative => :derivatives.
-  #
-  # Loading all three of these on result sets of hundreds of values is still relatively quick.
-  scope :with_representative_derivatives, -> { includes(:derivatives, leaf_representative: :derivatives) }
-
   # While Rails STI means the actual specific class is in `type`, sometimes
   # it can be convenient to fetch on a top category of Kithe::Model without using
   # Rails STI.
@@ -100,18 +91,6 @@ class Kithe::Model < ActiveRecord::Base
     in_memory
   end
 
-  # hacky :(
-  def derivatives(*args)
-    raise TypeError.new("Only valid on Kithe::Asset") unless self.kind_of?(Kithe::Asset)
-    super
-  end
-  # hacky :(
-  def derivatives=(*args)
-    raise TypeError.new("Only valid on Kithe::Asset") unless self.kind_of?(Kithe::Asset)
-    super
-  end
-
-
   # insist that leaf_representative is an Asset, otherwise return nil.
   # nil means there is no _asset_ leaf, and lets caller rely on leaf being
   # an asset.

data/app/models/kithe/work.rb

@@ -1,9 +1,4 @@
 class Kithe::Work < Kithe::Model
-  # Works don't have derivatives, but we want to allow Rails eager loading
-  # of association on hetereogenous fetches of Kithe::Model, so this is clever.
-  has_many :derivatives, -> { none }
-  private :derivatives, :derivatives=, :derivative_ids, :derivative_ids=
-
   after_initialize do
     self.kithe_model_type = "work" if self.kithe_model_type.nil?
   end

data/app/uploaders/kithe/asset_uploader.rb

@@ -20,7 +20,7 @@ module Kithe
   # FUTURE: Look at using client-side-calculated checksums to verify end-to-end.
   # https://github.com/shrinerb/shrine/wiki/Using-Checksums-in-Direct-Uploads
   #
-  # When magicc-byte analyzer can't determine mime type, will fall back to  `mediainfo`
+  # When magic-byte analyzer can't determine mime type, will fall back to  `mediainfo`
   # CLI _if_ `Kithe.use_mediainfo` is true (defaults to true if mediainfo CLI is
   # available). (We need better ways to customize uploader.)
   class AssetUploader < Shrine
@@ -34,100 +34,37 @@ module Kithe
     # promotion, possibly in the background.
     plugin :refresh_metadata
 
-    # Marcel analyzer is pure-ruby and fast. It's from Basecamp and is what
-    # ActiveStorage uses. It is very similar to :mimemagic (and uses mimemagic
-    # under the hood), but mimemagic seems not to be maintained with up to date
-    # magic db? https://github.com/minad/mimemagic/pull/66
-    plugin :determine_mime_type, analyzer: -> (io, analyzers) do
-      mime_type = analyzers[:marcel].call(io)
-
-      # But marcel is not able to catch some of our MP3s as audio/mpeg,
-      # let's try mediainfo command line. mediainfo is one of the tools
-      # the Harvard Fits tool uses. https://github.com/MediaArea/MediaInfo
-      if Kithe.use_mediainfo && mime_type == "application/octet-stream" || mime_type.blank?
-        mime_type = Kithe::MediainfoAnalyzer.new.call(io)
-      end
-
-      mime_type = "application/octet-stream" if mime_type.blank?
-
-      mime_type
-    end
-
     # Will save height and width to metadata for image types. (Won't for non-image types)
     # ignore errors (often due to storing a non-image file), consistent with shrine 2.x behavior.
     plugin :store_dimensions, on_error: :ignore
 
-    # promotion and deletion will (sometimes) be in background.
-    plugin :backgrounding
-
-    # Useful in case consumers want it, and doesn't harm anything to be available.
-    # https://github.com/shrinerb/shrine/blob/master/doc/plugins/rack_response.md
-    plugin :rack_response
+    plugin :infer_extension, inferrer: :mini_mime
 
-    # Normally we promote in background with backgrounding, but the set_promotion_directives
-    # feature can be used to make promotion not happen at all, or happen in foreground.
-    #     asset.file_attacher.set_promotion_directives(promote: false)
-    #     asset.file_attacher.set_promotion_directives(promote: "inline")
-    Attacher.promote_block do
-      Kithe::TimingPromotionDirective.new(key: :promote, directives: self.promotion_directives) do |directive|
-        if directive.inline?
-          promote
-        elsif directive.background?
-          # What shrine normally expects for backgrounding, plus promotion_directives
-          Kithe::AssetPromoteJob.perform_later(self.class.name, record.class.name, record.id, name.to_s, file_data, self.promotion_directives)
-        end
-      end
-    end
+    # Just leave it here for otheres please
+    plugin :add_metadata
 
-    # Delete using shrine backgrounding, but can be effected
-    # by promotion_directives[:delete], similar to promotion above.
-    # Yeah, not really a "promotion" directive, oh well.
-    Attacher.destroy_block do
-      Kithe::TimingPromotionDirective.new(key: :delete, directives: self.promotion_directives) do |directive|
-        if directive.inline?
-          destroy
-        elsif directive.background?
-          # What shrine normally expects for backgrounding
-          Kithe::AssetDeleteJob.perform_later(self.class.name, data)
-        end
-      end
-    end
 
-    plugin :add_metadata
 
-    # Makes files stored as /asset/#{asset_pk}/#{random_uuid}.#{original_suffix}
-    plugin :kithe_storage_location
 
-    # Allows you to assign hashes like:
-    #    { "id" => "http://url", "storage" => "remote_url", headers: { "Authorization" => "Bearer whatever"}}
-    # (headers optional), for fetching remote urls on promotion. Useful with browse-everything.
-    # WARNING: There's no whitelist, will accept any url. Is this a problem?
-    plugin :kithe_accept_remote_url
+    # kithe-standard logic for sniffing mime type.
+    plugin :kithe_determine_mime_type
 
-    # We want to store md5 and sha1 checksums (legacy compat), as well as
-    # sha512 (more recent digital preservation recommendation: https://ocfl.io/draft/spec/#digests)
-    #
-    # We only calculate them on `store` action to avoid double-computation, and because for
-    # direct uploads/backgrounding, we haven't actually gotten the file in our hands to compute
-    # checksums until then anyway.
-    plugin :signature
-    add_metadata do |io, context|
-      if context[:action] != :cache
-        {
-          md5: calculate_signature(io, :md5),
-          sha1: calculate_signature(io, :sha1),
-          sha512: calculate_signature(io, :sha512)
-        }
-      end
-    end
-    metadata_method :md5, :sha1, :sha512
+    # Determines storage path/location/id, so files will be stored as:
+    #      /asset/#{asset_pk}/#{random_uuid}.#{original_suffix}
+    plugin :kithe_storage_location
 
+    # Set up logic for shrine backgrounding, which in kithe can be set by promotion_directives
+    plugin :kithe_controllable_backgrounding
 
     # Gives us (set_)promotion_directives methods on our attacher to
     # house lifecycle directives, about whether promotion, deletion,
     # derivatives happen in foreground, background, or not at all.
     plugin :kithe_promotion_directives
 
+    # Makes our before/after promotion callbacks get called.
     plugin :kithe_promotion_callbacks
+
+    # some configuration and convenience methods for shrine derivatives.
+    plugin :kithe_derivatives
   end
 end

data/lib/kithe/version.rb

@@ -1,3 +1,6 @@
 module Kithe
-  VERSION = '2.0.0-alpha2'
+  # not sure why rubygems turned our alphas into 2.0.0.pre.alpha1, inserting
+  # "pre". We need to do same thing with betas to get version orderings
+  # appropriate.
+  VERSION = '2.0.0.pre.beta1'
 end

data/lib/shrine/plugins/kithe_checksum_signatures.rb

@@ -0,0 +1,41 @@
+class Shrine
+  module Plugins
+    # Using the shrine signature and add_metadata plugins, ensure that the shrine standard
+    # digest/checksum signatures are recorded in metadata.
+    #
+    # This plugin is NOT included in Kithe::AssetUploader by default, include it in your
+    # local uploader if desired.
+    #
+    # We want to store md5 and sha1 checksums (legacy compat), as well as
+    # sha512 (more recent digital preservation recommendation: https://ocfl.io/draft/spec/#digests)
+    #
+    # We only calculate them only on promotion action (not cache action), to avoid needlessly
+    # expensive double-computation, and because for direct uploads/backgrounding, we haven't
+    # actually gotten the file in our hands to compute checksums until then anyway.
+    #
+    # the add_metadata plugin's `metadata_method` is used to make md5, sha1, and sha512 methods
+    # available on the Attacher. (They also end up delegated from the Asset model)
+    class KitheChecksumSignatures
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :add_metadata
+        uploader.plugin :signature
+      end
+
+      def self.configure(uploader, opts = {})
+        uploader.class_eval do
+          add_metadata do |io, derivative:nil, **context|
+            if context[:action] != :cache && derivative.nil?
+              {
+                md5: calculate_signature(io, :md5),
+                sha1: calculate_signature(io, :sha1),
+                sha512: calculate_signature(io, :sha512)
+              }
+            end
+          end
+          metadata_method :md5, :sha1, :sha512
+        end
+      end
+    end
+    register_plugin(:kithe_checksum_signatures, KitheChecksumSignatures)
+  end
+end

data/lib/shrine/plugins/kithe_controllable_backgrounding.rb

@@ -0,0 +1,53 @@
+class Shrine
+  module Plugins
+
+    # Set up shrine `backgrounding`, where promotion and deletion can happen in a background job.
+    #
+    # https://shrinerb.com/docs/getting-started#backgrounding
+    # https://shrinerb.com/docs/plugins/backgrounding
+    #
+    # By default, kithe does promotion and deletion in kithe-provided ActiveJob classes.
+    #
+    # But this plugin implements code to let you use kithe_promotion_directives to make them happen
+    # inline instead, or disable them.
+    #
+    #     asset.file_attacher.set_promotion_directives(promote: false)
+    #     asset.file_attacher.set_promotion_directives(promote: :inline)
+    #     asset.file_attacher.set_promotion_directives(promote: "inline")
+    #
+    #     asset.file_attacher.set_promotion_directives(delete: :inline)
+    class KitheControllableBackgrounding
+      def self.load_dependencies(uploader, *)
+        uploader.plugin :backgrounding
+        uploader.plugin :kithe_promotion_directives
+      end
+
+      def self.configure(uploader, options = {})
+
+        # promote using shrine backgrounding, but can be effected by promotion_directives[:promote]
+        uploader::Attacher.promote_block do
+          Kithe::TimingPromotionDirective.new(key: :promote, directives: self.promotion_directives) do |directive|
+            if directive.inline?
+              promote
+            elsif directive.background?
+              # What shrine normally expects for backgrounding, plus promotion_directives
+              Kithe::AssetPromoteJob.perform_later(self.class.name, record.class.name, record.id, name.to_s, file_data, self.promotion_directives)
+            end
+          end
+        end
+
+        uploader::Attacher.destroy_block do
+          Kithe::TimingPromotionDirective.new(key: :delete, directives: self.promotion_directives) do |directive|
+            if directive.inline?
+              destroy
+            elsif directive.background?
+              # What shrine normally expects for backgrounding
+              Kithe::AssetDeleteJob.perform_later(self.class.name, data)
+            end
+          end
+        end
+      end
+    end
+    register_plugin(:kithe_controllable_backgrounding, KitheControllableBackgrounding)
+  end
+end

data/lib/shrine/plugins/kithe_derivative_definitions.rb

@@ -0,0 +1,101 @@
+class Shrine
+  module Plugins
+    class KitheDerivativeDefinitions
+      def self.configure(uploader, *opts)
+        # use Rails class_attribute to conveniently have a class-level place
+        # to store our derivative definitions that are inheritable and overrideable.
+        # We store it on the Attacher class, because that's where shrine
+        # puts derivative processor definitions, so seems appropriate.
+        uploader::Attacher.class_attribute :kithe_derivative_definitions, instance_writer: false, default: []
+
+        # Register our derivative processor, that will create our registered derivatives,
+        # with our custom options.
+        uploader::Attacher.derivatives(:kithe_derivatives) do |original, **options|
+          Kithe::Asset::DerivativeCreator.new(self.class.kithe_derivative_definitions,
+            source_io: original,
+            shrine_attacher: self,
+            only: options[:only],
+            except: options[:except],
+            lazy: options[:lazy]
+          ).call
+        end
+      end
+
+      module AttacherClassMethods
+        # Establish a derivative definition that will be used to create a derivative
+        # when #create_derivatives is called, for instance automatically after promotion.
+        #
+        # The most basic definition consists of a derivative key, and a ruby block that
+        # takes the original file, transforms it, and returns a ruby File or other
+        # (shrine-compatible) IO-like object. It will usually be done inside a custom Asset
+        # class definition.
+        #
+        #     class Asset < Kithe::Asset
+        #       define_derivative :thumbnail do |original_file|
+        #       end
+        #     end
+        #
+        # The original_file passed in will be a ruby File object that is already open for reading. If
+        # you need a local file path for your transformation, just use `original_file.path`.
+        #
+        # The return value can be any IO-like object. If it is a ruby File or Tempfile,
+        # that temporary file will be deleted for you after the derivative has been created. If you
+        # have to make any intermediate files, you are responsible for cleaning them up. Ruby stdlib
+        # Tempfile and Dir.mktmpdir may be useful.
+        #
+        # If in order to do your transformation you need additional information about the original,
+        # just add a `record:` keyword argument to your block, and the Asset object will be passed in:
+        #
+        #     define_derivative :thumbnail do |original_file, record:|
+        #        record.width, record.height, record.content_type # etc
+        #     end
+        #
+        # Derivatives are normally uploaded to the Shrine storage labeled :kithe_derivatives,
+        # but a definition can specify an alternate Shrine storage id. (specified shrine storage key
+        # is applied on derivative creation; if you change it with existing derivatives, they should
+        # remain, and be accessible, where they were created; there is no built-in solution at present
+        # for moving them).
+        #
+        #     define_derivative :thumbnail, storage_key: :my_thumb_storage do |original| # ...
+        #
+        # You can also set `default_create: false` if you want a particular definition not to be
+        # included in a no-arg `asset.create_derivatives` that is normally triggered on asset creation.
+        #
+        # And you can set content_type to either a specific type like `image/jpeg` (or array of such) or a general type
+        # like `image`, if you want to define a derivative generation routine for only certain types.
+        # If multiple blocks for the same key are defined, with different content_type restrictions,
+        # the most specific one will be used.  That is, for a JPG, `image/jpeg` beats `image` beats no restriction.
+        def define_derivative(key, content_type: nil, default_create: true, &block)
+          # Make sure we dup the array to handle sub-classes on class_attribute
+          self.kithe_derivative_definitions = self.kithe_derivative_definitions.dup.push(
+            Kithe::Asset::DerivativeDefinition.new(
+              key: key,
+              content_type: content_type,
+              default_create: default_create,
+              proc: block
+            )
+          ).freeze
+        end
+
+        # Returns all derivative keys registered with a definition, as array of strings
+        def defined_derivative_keys
+          self.kithe_derivative_definitions.collect(&:key).uniq.collect(&:to_s)
+        end
+
+        # If you have a subclass that has inherited derivative definitions, you can
+        # remove them -- only by key, will remove any definitions with that key regardless
+        # of content_type restrictions.
+        #
+        # This could be considered rather bad OO design, you might want to consider
+        # a different class hieararchy where you don't have to do this. But it's here.
+        def remove_derivative_definition!(*keys)
+          keys = keys.collect(&:to_sym)
+          self.kithe_derivative_definitions = self.kithe_derivative_definitions.reject do |defn|
+            keys.include?(defn.key.to_sym)
+          end.freeze
+        end
+      end
+    end
+    register_plugin(:kithe_derivative_definitions, KitheDerivativeDefinitions)
+  end
+end