kithe 2.0.0.pre.alpha2 → 2.0.2

data/app/models/kithe/asset/derivative_definition.rb

@@ -2,38 +2,37 @@
 # class, it's what's created when you call Kithe::Asset#define_derivative
 class Kithe::Asset::DerivativeDefinition
   attr_reader :key, :content_type, :default_create, :proc, :storage_key
-  def initialize(key:, storage_key:, proc:, content_type: nil, default_create: true)
-    @key = key
+  def initialize(key:, proc:, content_type: nil, default_create: true)
+    @key = key.to_sym
     @content_type = content_type
-    @storage_key = storage_key
     @default_create = default_create
     @proc = proc
   end
 
-  def call(original_file:,record:)
-    if proc_accepts_record_keyword?
-      proc.call(original_file, record: record)
+  def call(original_file:,attacher:)
+    if proc_accepts_keyword?(:attacher)
+      proc.call(original_file, attacher: attacher)
     else
       proc.call(original_file)
     end
   end
 
   # Do content-type restrictions defined for this definition match a given asset?
-  def applies_to?(asset)
+  def applies_to_content_type?(original_content_type)
     return true if content_type.nil?
 
-    return true if content_type == asset.content_type
+    return true if content_type == original_content_type
 
-    return false if asset.content_type.nil?
+    return false if original_content_type.nil?
 
-    return true if (content_type.kind_of?(Array) && content_type.include?(asset.content_type))
+    return true if (content_type.kind_of?(Array) && content_type.include?(original_content_type))
 
-    content_type == asset.content_type.sub(%r{/.+\Z}, '')
+    content_type == original_content_type.sub(%r{/.+\Z}, '')
   end
 
   private
 
-  def proc_accepts_record_keyword?
-    proc.parameters.include?([:key, :record]) || proc.parameters.include?([:keyreq, :record])
+  def proc_accepts_keyword?(kwarg)
+    proc.parameters.include?([:key, kwarg]) || proc.parameters.include?([:keyreq, kwarg]) || proc.parameters.find {|a| a.first == :keyrest}
   end
 end

data/app/models/kithe/asset/set_shrine_uploader.rb

@@ -0,0 +1,64 @@
+# Our Kithe::Asset model class is meant to be a superclass of a local application asset class, which we
+# can call `Asset`, although an app can call it whatever they like.
+#
+# Kithe::Asset sets it's own shrine uploader class, with a typical shrine:
+#
+#     include Kithe::AssetUploader::Attachment(:file)
+#
+# An application Asset subclass will inherit this uploader, which is convenient for getting
+# started quickly. But an application will likely want to define its own local uploader
+# class, to define it's own metadata, derivatives, and any other custom beahvior.
+#
+# There isn't an obvious built-into-shrine way to do that, but it turns out simply overriding
+# class and instance `*_attacher` methods seems to work out well. See:
+# https://discourse.shrinerb.com/t/model-sub-classes-with-uploader-sub-classes/208
+#
+# So a local application can define it's own shrine uploader, which is highly recommended to
+# be a sub-class of Kithe::AssetUploader to ensure it has required and useful
+# Kithe behavior:
+#
+#     # ./app/uploaders/asset_uploader.rb
+#     class AssetUploader < Kithe::AssetUploader
+#       # maybe we want some custom metadata
+#       add_metadata :something do |io|
+#         whatever
+#       end
+#     end
+#
+# And then set it in ti's custom local Asset class:
+#
+#     # ./app/models/asset.rb
+#     class Asset < Kithe::Asset
+#       set_shrine_uploader(AssetUploader)
+#     end
+#
+# If a local app has it's own inheritance hieararchy of children below that (eg) Asset class,
+# they can each (optionally) also override with a custom Uploader. It is recommended that
+# the Uploader inheritance hieararchy match the model inheritance hieararchy, to have
+# all behavior consistent. For instance:
+#
+#     class AudioAssetUploader < AssetUploader
+#     end
+#
+#     class AudioAsset < Asset
+#       set_shrine_uploader(AudioAssetUploader)
+#     end
+#
+module Kithe::Asset::SetShrineUploader
+  extend ActiveSupport::Concern
+
+  class_methods do
+    def set_shrine_uploader(uploader_class)
+      subclass_attachment = uploader_class::Attachment.new(:file)
+
+      define_singleton_method :file_attacher do |**options|
+        subclass_attachment.send(:class_attacher, **options)
+      end
+
+      define_method :file_attacher do |**options|
+        subclass_attachment.send(:attacher, self, **options)
+      end
+    end
+  end
+
+end

data/app/models/kithe/collection.rb

@@ -1,14 +1,8 @@
 class Kithe::Collection < Kithe::Model
-  # Collections don't have derivatives, but we want to allow Rails eager loading
-  # of association on hetereogenous fetches of Kithe::Model, so this is clever.
-  has_many :derivatives, -> { none }
-  private :derivatives, :derivatives=, :derivative_ids, :derivative_ids=
-
   after_initialize do
     self.kithe_model_type = "collection" if self.kithe_model_type.nil?
   end
   before_validation do
     self.kithe_model_type = "collection" if self.kithe_model_type.nil?
   end
-
 end

data/app/models/kithe/model.rb

@@ -10,15 +10,6 @@ class Kithe::Model < ActiveRecord::Base
   include AttrJson::Record::Dirty
   include Kithe::Indexable
 
-  # A handy scope for eager-loading all representatives and all of their derivatives.
-  #
-  # Works on hetereogenous collections of Works and Assets -- the Assets need
-  # :derivatives directly referenced (since they don't really have a leaf_representative assoc),
-  # the works need :leaf_representative => :derivatives.
-  #
-  # Loading all three of these on result sets of hundreds of values is still relatively quick.
-  scope :with_representative_derivatives, -> { includes(:derivatives, leaf_representative: :derivatives) }
-
   # While Rails STI means the actual specific class is in `type`, sometimes
   # it can be convenient to fetch on a top category of Kithe::Model without using
   # Rails STI.
@@ -100,18 +91,6 @@ class Kithe::Model < ActiveRecord::Base
     in_memory
   end
 
-  # hacky :(
-  def derivatives(*args)
-    raise TypeError.new("Only valid on Kithe::Asset") unless self.kind_of?(Kithe::Asset)
-    super
-  end
-  # hacky :(
-  def derivatives=(*args)
-    raise TypeError.new("Only valid on Kithe::Asset") unless self.kind_of?(Kithe::Asset)
-    super
-  end
-
-
   # insist that leaf_representative is an Asset, otherwise return nil.
   # nil means there is no _asset_ leaf, and lets caller rely on leaf being
   # an asset.

data/app/models/kithe/work.rb

@@ -1,9 +1,4 @@
 class Kithe::Work < Kithe::Model
-  # Works don't have derivatives, but we want to allow Rails eager loading
-  # of association on hetereogenous fetches of Kithe::Model, so this is clever.
-  has_many :derivatives, -> { none }
-  private :derivatives, :derivatives=, :derivative_ids, :derivative_ids=
-
   after_initialize do
     self.kithe_model_type = "work" if self.kithe_model_type.nil?
   end

data/app/uploaders/kithe/asset_uploader.rb

@@ -20,7 +20,7 @@ module Kithe
   # FUTURE: Look at using client-side-calculated checksums to verify end-to-end.
   # https://github.com/shrinerb/shrine/wiki/Using-Checksums-in-Direct-Uploads
   #
-  # When magicc-byte analyzer can't determine mime type, will fall back to  `mediainfo`
+  # When magic-byte analyzer can't determine mime type, will fall back to  `mediainfo`
   # CLI _if_ `Kithe.use_mediainfo` is true (defaults to true if mediainfo CLI is
   # available). (We need better ways to customize uploader.)
   class AssetUploader < Shrine
@@ -34,100 +34,37 @@ module Kithe
     # promotion, possibly in the background.
     plugin :refresh_metadata
 
-    # Marcel analyzer is pure-ruby and fast. It's from Basecamp and is what
-    # ActiveStorage uses. It is very similar to :mimemagic (and uses mimemagic
-    # under the hood), but mimemagic seems not to be maintained with up to date
-    # magic db? https://github.com/minad/mimemagic/pull/66
-    plugin :determine_mime_type, analyzer: -> (io, analyzers) do
-      mime_type = analyzers[:marcel].call(io)
-
-      # But marcel is not able to catch some of our MP3s as audio/mpeg,
-      # let's try mediainfo command line. mediainfo is one of the tools
-      # the Harvard Fits tool uses. https://github.com/MediaArea/MediaInfo
-      if Kithe.use_mediainfo && mime_type == "application/octet-stream" || mime_type.blank?
-        mime_type = Kithe::MediainfoAnalyzer.new.call(io)
-      end
-
-      mime_type = "application/octet-stream" if mime_type.blank?
-
-      mime_type
-    end
-
     # Will save height and width to metadata for image types. (Won't for non-image types)
     # ignore errors (often due to storing a non-image file), consistent with shrine 2.x behavior.
     plugin :store_dimensions, on_error: :ignore
 
-    # promotion and deletion will (sometimes) be in background.
-    plugin :backgrounding
-
-    # Useful in case consumers want it, and doesn't harm anything to be available.
-    # https://github.com/shrinerb/shrine/blob/master/doc/plugins/rack_response.md
-    plugin :rack_response
+    plugin :infer_extension, inferrer: :mini_mime
 
-    # Normally we promote in background with backgrounding, but the set_promotion_directives
-    # feature can be used to make promotion not happen at all, or happen in foreground.
-    #     asset.file_attacher.set_promotion_directives(promote: false)
-    #     asset.file_attacher.set_promotion_directives(promote: "inline")
-    Attacher.promote_block do
-      Kithe::TimingPromotionDirective.new(key: :promote, directives: self.promotion_directives) do |directive|
-        if directive.inline?
-          promote
-        elsif directive.background?
-          # What shrine normally expects for backgrounding, plus promotion_directives
-          Kithe::AssetPromoteJob.perform_later(self.class.name, record.class.name, record.id, name.to_s, file_data, self.promotion_directives)
-        end
-      end
-    end
+    # Just leave it here for otheres please
+    plugin :add_metadata
 
-    # Delete using shrine backgrounding, but can be effected
-    # by promotion_directives[:delete], similar to promotion above.
-    # Yeah, not really a "promotion" directive, oh well.
-    Attacher.destroy_block do
-      Kithe::TimingPromotionDirective.new(key: :delete, directives: self.promotion_directives) do |directive|
-        if directive.inline?
-          destroy
-        elsif directive.background?
-          # What shrine normally expects for backgrounding
-          Kithe::AssetDeleteJob.perform_later(self.class.name, data)
-        end
-      end
-    end
 
-    plugin :add_metadata
 
-    # Makes files stored as /asset/#{asset_pk}/#{random_uuid}.#{original_suffix}
-    plugin :kithe_storage_location
 
-    # Allows you to assign hashes like:
-    #    { "id" => "http://url", "storage" => "remote_url", headers: { "Authorization" => "Bearer whatever"}}
-    # (headers optional), for fetching remote urls on promotion. Useful with browse-everything.
-    # WARNING: There's no whitelist, will accept any url. Is this a problem?
-    plugin :kithe_accept_remote_url
+    # kithe-standard logic for sniffing mime type.
+    plugin :kithe_determine_mime_type
 
-    # We want to store md5 and sha1 checksums (legacy compat), as well as
-    # sha512 (more recent digital preservation recommendation: https://ocfl.io/draft/spec/#digests)
-    #
-    # We only calculate them on `store` action to avoid double-computation, and because for
-    # direct uploads/backgrounding, we haven't actually gotten the file in our hands to compute
-    # checksums until then anyway.
-    plugin :signature
-    add_metadata do |io, context|
-      if context[:action] != :cache
-        {
-          md5: calculate_signature(io, :md5),
-          sha1: calculate_signature(io, :sha1),
-          sha512: calculate_signature(io, :sha512)
-        }
-      end
-    end
-    metadata_method :md5, :sha1, :sha512
+    # Determines storage path/location/id, so files will be stored as:
+    #      /asset/#{asset_pk}/#{random_uuid}.#{original_suffix}
+    plugin :kithe_storage_location
 
+    # Set up logic for shrine backgrounding, which in kithe can be set by promotion_directives
+    plugin :kithe_controllable_backgrounding
 
     # Gives us (set_)promotion_directives methods on our attacher to
     # house lifecycle directives, about whether promotion, deletion,
     # derivatives happen in foreground, background, or not at all.
     plugin :kithe_promotion_directives
 
+    # Makes our before/after promotion callbacks get called.
     plugin :kithe_promotion_callbacks
+
+    # some configuration and convenience methods for shrine derivatives.
+    plugin :kithe_derivatives
   end
 end

data/lib/kithe.rb

@@ -68,28 +68,30 @@ module Kithe
   # The settings need to live here not in Kithe::Indexable, to avoid terrible
   # Rails dev-mode class-reloading weirdnesses. This module is not reloaded.
   class << self
-    attr_accessor :indexable_settings
+    attr_writer :indexable_settings
   end
-  self.indexable_settings = IndexableSettings.new(
-    solr_url: "http://localhost:8983/solr/default",
-    model_name_solr_field: "model_name_ssi",
-    solr_id_value_attribute: "id",
-    writer_class_name: "Traject::SolrJsonWriter",
-    writer_settings: {
-      # as default we tell the solrjsonwriter to use no threads,
-      # no batching. softCommit on every update. Least surprising
-      # default configuration.
-      "solr_writer.thread_pool" => 0,
-      "solr_writer.batch_size" => 1,
-      "solr_writer.solr_update_args" => { softCommit: true },
-      "solr_writer.http_timeout" => 3,
-      "logger" => Rails.logger,
+  def self.indexable_settings
+    @indexable_settings ||= IndexableSettings.new(
+      solr_url: "http://localhost:8983/solr/default",
+      model_name_solr_field: "model_name_ssi",
+      solr_id_value_attribute: "id",
+      writer_class_name: "Traject::SolrJsonWriter",
+      writer_settings: {
+        # as default we tell the solrjsonwriter to use no threads,
+        # no batching. softCommit on every update. Least surprising
+        # default configuration.
+        "solr_writer.thread_pool" => 0,
+        "solr_writer.batch_size" => 1,
+        "solr_writer.solr_update_args" => { softCommit: true },
+        "solr_writer.http_timeout" => 3,
+        "logger" => Rails.logger,
 
-      # MAYBE? no skippable exceptions please
-      # "solr_writer.skippable_exceptions" => []
-    },
-    disable_callbacks: false
-  )
+        # MAYBE? no skippable exceptions please
+        # "solr_writer.skippable_exceptions" => []
+      },
+      disable_callbacks: false
+    )
+  end
 
   class << self
     # Currently used by Kithe::AssetUploader, a bit of a hacky

data/{app/models → lib}/kithe/config_base.rb

@@ -9,7 +9,7 @@ module Kithe
   #
   # You may also want to consider [railsconfig](https://github.com/railsconfig/config)
   #
-  # Kithe::Config:
+  # Kithe::ConfigBase:
   #
   # * uses an explicit declared list of allowable config keys, no silent typos
   # * can read from a local YAML file or ENV, by default letting ENV override local YAML file values.
@@ -97,6 +97,11 @@ module Kithe
   # This doesn't use any locking for concurrent initial loads, which is technically not
   # great, but probably shouldn't be a problem in practice, especially in MRI. Trying to
   # do proper locking with lazy load was too hard for me right now.
+  #
+  # ## Auto-loading
+  #
+  # This is intentionally NOT in an auto-loaded directory, so it can be used more
+  # easily in Rails initialization without problems. https://github.com/rails/rails/issues/40904
   class ConfigBase
     include Singleton
 

data/lib/kithe/engine.rb

@@ -1,5 +1,19 @@
 require 'shrine'
 
+# Gem "F(x)" or `fx` gem will get schema.rb to include locally-defined custom postgres functions
+# and triggers, like we use. So apps can keep using schema.rb instead of structure.sql,
+# and still have our custom functions preserved. We need to require it explicitly
+# since it'll be an indirect dependency of the end app.
+#
+# But we need to patch it to create functions first so we can use them as default values
+# https://github.com/teoljungberg/fx/issues/33
+# https://github.com/teoljungberg/fx/pull/53
+require 'fx'
+require 'kithe/patch_fx'
+
+# not auto-loaded, let's just load it for backwards compat though
+require "kithe/config_base"
+
 module Kithe
   class Engine < ::Rails::Engine
     config.generators do |g|
@@ -8,8 +22,5 @@ module Kithe
       g.assets false
       g.helper false
     end
-
-    # should only affect kithe development
-    config.active_record.schema_format = :sql
   end
 end

data/lib/kithe/indexable_settings.rb

@@ -8,7 +8,7 @@ module Kithe
       @writer_class_name = writer_class_name
       @writer_settings = writer_settings
       @model_name_solr_field = model_name_solr_field
-      @solr_id_value_attribute = solr_id_value_attribute
+      @solr_id_value_attribute = solr_id_value_attribute || 'id'
     end
 
     # Use configured solr_url, and merge together with configured

data/lib/kithe/patch_fx.rb

@@ -0,0 +1,39 @@
+# fx is a gem that lets Rails schema.rb capture postgres functions and triggers
+#
+# For it to work for our use case, we need it to define functions BEFORE tables when
+# doing a `rake db:schema:load`, so we can refer to functions as default values in our
+# tables.
+#
+# This is a known issue in fx, with a PR, but isn't yet merged/released, so we hack
+# in a patch to force it. Better than forking.
+#
+# Based on: https://github.com/teoljungberg/fx/pull/53/
+#
+# We try to write future-compat code assuming that will be merged eventually....
+
+require 'fx'
+
+if Fx.configuration.respond_to?(:dump_functions_at_beginning_of_schema)
+  # we have the feature!
+
+  Fx.configure do |config|
+    config.dump_functions_at_beginning_of_schema = true
+  end
+
+else
+  # Fx does not have the feature, we have to patch it in
+
+  require 'fx/schema_dumper/function'
+
+  module Fx
+    module SchemaDumper
+      module Function
+        def tables(stream)
+          functions(stream)
+          super
+        end
+      end
+    end
+  end
+
+end