kithe 2.0.0.pre.alpha2 → 2.0.2

data/spec/shrine/kithe_storage_location_spec.rb

@@ -4,29 +4,57 @@ require 'shrine/plugins/kithe_storage_location'
 describe Shrine::Plugins::KitheStorageLocation do
   let(:uploader) { test_uploader { plugin :kithe_storage_location } }
 
-  it "uploads with a record with id" do
-    uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
+  describe "for main file" do
+    it "uploads with a record with id" do
+      uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
 
-    expect(uploaded_file.id).to match %r{\Aasset/81060886-4f93-42e7-ace7-ab51399f4808/[0-9a-f]+}
-  end
+      expect(uploaded_file.id).to match %r{\Aasset/81060886-4f93-42e7-ace7-ab51399f4808/[0-9a-f]+}
+    end
 
-  it "has suffix with a record id and filename" do
-    uploaded_file = uploader.upload(fakeio(filename: "foo.jpg"), record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
+    it "has suffix with a record id and filename" do
+      uploaded_file = uploader.upload(fakeio(filename: "foo.jpg"), record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
 
-    expect(uploaded_file.id).to match %r{\Aasset/81060886-4f93-42e7-ace7-ab51399f4808/[0-9a-f]+\.jpg}
-  end
+      expect(uploaded_file.id).to match %r{\Aasset/81060886-4f93-42e7-ace7-ab51399f4808/[0-9a-f]+\.jpg}
+    end
 
-  it "uploads with no record" do
-    uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
+    it "uploads with no record" do
+      uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
 
-    expect(uploaded_file.id).to match %r{\Aasset/[0-9a-f]+}
-  end
+      expect(uploaded_file.id).to match %r{\Aasset/[0-9a-f]+}
+    end
 
-  it "uploads with record with no id" do
-    uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(), name: :file)
+    it "uploads with record with no id" do
+      uploaded_file = uploader.upload(fakeio, record: OpenStruct.new(), name: :file)
 
-    expect(uploaded_file.id).to match %r{\Aasset/[0-9a-f]+}
+      expect(uploaded_file.id).to match %r{\Aasset/[0-9a-f]+}
+    end
   end
 
+  describe "for shrine derivatives" do
+    let(:image_path) { Kithe::Engine.root.join("spec/test_support/images/1x1_pixel.jpg") }
+
+    let(:uploader) do
+      test_uploader do
+        plugin :kithe_storage_location
+      end
+    end
 
+    it "uses good path for derivative" do
+      uploaded_file = uploader.upload(fakeio("foo.jpg"), derivative: :fixed, record: OpenStruct.new(id: "81060886-4f93-42e7-ace7-ab51399f4808"), name: :file)
+
+      expect(uploaded_file.id).to match %r{\A81060886-4f93-42e7-ace7-ab51399f4808/fixed/[0-9a-f]+}
+    end
+
+    it "raises with no record" do
+      expect {
+        uploader.upload(fakeio("foo.jpg"), record: nil, derivative: :fixed, name: :file)
+      }.to raise_error(TypeError)
+    end
+
+    it "raises with record with no id" do
+      expect {
+        uploader.upload(fakeio("foo.jpg"), record: OpenStruct.new, derivative: :fixed, name: :file)
+      }.to raise_error(TypeError)
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -123,22 +123,3 @@ RSpec.configure do |config|
 =end
 end
 
-# Workaround ruby 2.7.0 StringIO enccoding weirdness, which hopefully
-# will be fixed in shrine 3.x before we get there. if you can remove this patch
-# and tests still pass, you're good.
-#
-# https://github.com/shrinerb/shrine/pull/443
-#
-require 'sane_patch'
-SanePatch.patch("shrine", "< 3.2.2") do
-  require 'shrine/storage/memory'
-  class Shrine::Storage::Memory
-    def open(id, *)
-      str = store.fetch(id)
-      StringIO.new(str).set_encoding(str.encoding, str.encoding)
-    end
-  end
-end
-
-
-

data/spec/test_support/shrine_spec_support.rb

@@ -59,8 +59,9 @@ require "stringio"
 class FakeIO
   attr_reader :original_filename, :content_type
 
-  def initialize(content, filename: nil, content_type: nil)
+  def initialize(content, filename: nil, content_type: nil, encoding: "BINARY")
     @io = StringIO.new(content)
+    @io.set_encoding(encoding, encoding) # weird ruby workaround
     @original_filename = filename
     @content_type = content_type
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kithe
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre.alpha2
+  version: 2.0.2
 platform: ruby
 authors:
 - Jonathan Rochkind
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-18 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -19,7 +19,7 @@ dependencies:
         version: 5.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '6.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: 5.2.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: attr_json
   requirement: !ruby/object:Gem::Requirement
@@ -182,6 +182,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: traject
   requirement: !ruby/object:Gem::Requirement
@@ -272,20 +292,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: shrine-memory
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -314,7 +320,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: 
+description:
 email:
 - jrochkind@sciencehistory.org
 executables: []
@@ -341,10 +347,8 @@ files:
 - app/models/kithe/asset.rb
 - app/models/kithe/asset/derivative_creator.rb
 - app/models/kithe/asset/derivative_definition.rb
-- app/models/kithe/asset/derivative_updater.rb
+- app/models/kithe/asset/set_shrine_uploader.rb
 - app/models/kithe/collection.rb
-- app/models/kithe/config_base.rb
-- app/models/kithe/derivative.rb
 - app/models/kithe/mediainfo_analyzer.rb
 - app/models/kithe/model.rb
 - app/models/kithe/model_contains.rb
@@ -355,7 +359,6 @@ files:
 - app/simple_form_enhancements/kithe/form_builder.rb
 - app/simple_form_enhancements/kithe/repeatable_input_generator.rb
 - app/uploaders/kithe/asset_uploader.rb
-- app/uploaders/kithe/derivative_uploader.rb
 - app/validators/array_inclusion_validator.rb
 - config/locales/en.yml
 - config/routes.rb
@@ -370,12 +373,20 @@ files:
 - lib/kithe.rb
 - lib/kithe/blacklight_tools/bulk_loading_search_service.rb
 - lib/kithe/blacklight_tools/search_service_bulk_load.rb
+- lib/kithe/config_base.rb
 - lib/kithe/engine.rb
 - lib/kithe/indexable_settings.rb
+- lib/kithe/patch_fx.rb
 - lib/kithe/sti_preload.rb
 - lib/kithe/version.rb
 - lib/shrine/plugins/kithe_accept_remote_url.rb
+- lib/shrine/plugins/kithe_checksum_signatures.rb
+- lib/shrine/plugins/kithe_controllable_backgrounding.rb
+- lib/shrine/plugins/kithe_derivative_definitions.rb
+- lib/shrine/plugins/kithe_derivatives.rb
+- lib/shrine/plugins/kithe_determine_mime_type.rb
 - lib/shrine/plugins/kithe_multi_cache.rb
+- lib/shrine/plugins/kithe_persisted_derivatives.rb
 - lib/shrine/plugins/kithe_promotion_callbacks.rb
 - lib/shrine/plugins/kithe_promotion_directives.rb
 - lib/shrine/plugins/kithe_storage_location.rb
@@ -395,6 +406,7 @@ files:
 - spec/dummy/app/jobs/application_job.rb
 - spec/dummy/app/mailers/application_mailer.rb
 - spec/dummy/app/models/application_record.rb
+- spec/dummy/app/models/plain_active_record.rb
 - spec/dummy/app/views/layouts/application.html.erb
 - spec/dummy/app/views/layouts/mailer.html.erb
 - spec/dummy/app/views/layouts/mailer.text.erb
@@ -431,7 +443,7 @@ files:
 - spec/dummy/config/routes.rb
 - spec/dummy/config/spring.rb
 - spec/dummy/config/storage.yml
-- spec/dummy/db/structure.sql
+- spec/dummy/db/schema.rb
 - spec/dummy/log/development.log
 - spec/dummy/log/test.log
 - spec/dummy/package.json
@@ -448,19 +460,24 @@ files:
 - spec/indexing/indexable_spec.rb
 - spec/indexing/indexer_spec.rb
 - spec/indexing/obj_extract_spec.rb
-- spec/models/kithe/asset/asset_create_derivatives_spec.rb
+- spec/models/kithe/asset/asset_derivatives_spec.rb
 - spec/models/kithe/asset/asset_promotion_hooks_spec.rb
+- spec/models/kithe/asset/set_shrine_uploader_spec.rb
 - spec/models/kithe/asset_spec.rb
 - spec/models/kithe/collection_spec.rb
 - spec/models/kithe/config_spec.rb
-- spec/models/kithe/derivative_spec.rb
 - spec/models/kithe/mediainfo_analyzer_spec.rb
 - spec/models/kithe/model_spec.rb
 - spec/models/kithe/parameters_spec.rb
 - spec/models/kithe/representatives_spec.rb
 - spec/models/kithe/work_spec.rb
+- spec/models/kithe_spec.rb
 - spec/rails_helper.rb
+- spec/shrine/kithe_accept_remote_url_spec.rb
+- spec/shrine/kithe_checksum_signatures_spec.rb
+- spec/shrine/kithe_derivative_definitions_spec.rb
 - spec/shrine/kithe_multi_cache_spec.rb
+- spec/shrine/kithe_persisted_derivatives_spec.rb
 - spec/shrine/kithe_storage_location_spec.rb
 - spec/simple_form_enhancements/repeatable_input_generator_spec.rb
 - spec/spec_helper.rb
@@ -468,6 +485,7 @@ files:
 - spec/test_support/audio/ice_cubes.mp3
 - spec/test_support/images/1x1_pixel.jpg
 - spec/test_support/images/2x2_pixel.jpg
+- spec/test_support/images/3x3_pixel.jpg
 - spec/test_support/images/photo_800x586.jpg
 - spec/test_support/shrine_spec_support.rb
 - spec/test_support/temporary_class_for_specs.rb
@@ -476,7 +494,7 @@ homepage: https://github.com/sciencehistory/kithe
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -487,18 +505,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Shareable tools/components for building a digital collections app in Rails.
 test_files:
 - spec/spec_helper.rb
 - spec/dummy/app/mailers/application_mailer.rb
+- spec/dummy/app/models/plain_active_record.rb
 - spec/dummy/app/models/application_record.rb
 - spec/dummy/app/jobs/application_job.rb
 - spec/dummy/app/controllers/application_controller.rb
@@ -553,7 +571,7 @@ test_files:
 - spec/dummy/public/404.html
 - spec/dummy/public/apple-touch-icon-precomposed.png
 - spec/dummy/package.json
-- spec/dummy/db/structure.sql
+- spec/dummy/db/schema.rb
 - spec/dummy/log/test.log
 - spec/dummy/log/development.log
 - spec/dummy/tmp/development_secret.txt
@@ -561,23 +579,29 @@ test_files:
 - spec/derivative_transformers/ffmpeg_transformer_spec.rb
 - spec/models/kithe/collection_spec.rb
 - spec/models/kithe/config_spec.rb
+- spec/models/kithe/asset/asset_derivatives_spec.rb
 - spec/models/kithe/asset/asset_promotion_hooks_spec.rb
-- spec/models/kithe/asset/asset_create_derivatives_spec.rb
+- spec/models/kithe/asset/set_shrine_uploader_spec.rb
 - spec/models/kithe/representatives_spec.rb
 - spec/models/kithe/work_spec.rb
 - spec/models/kithe/mediainfo_analyzer_spec.rb
 - spec/models/kithe/parameters_spec.rb
-- spec/models/kithe/derivative_spec.rb
 - spec/models/kithe/asset_spec.rb
 - spec/models/kithe/model_spec.rb
+- spec/models/kithe_spec.rb
 - spec/shrine/kithe_multi_cache_spec.rb
 - spec/shrine/kithe_storage_location_spec.rb
+- spec/shrine/kithe_persisted_derivatives_spec.rb
+- spec/shrine/kithe_derivative_definitions_spec.rb
+- spec/shrine/kithe_accept_remote_url_spec.rb
+- spec/shrine/kithe_checksum_signatures_spec.rb
 - spec/factories/kithe_works.rb
 - spec/factories/kithe_collections.rb
 - spec/factories/kithe_assets.rb
 - spec/test_support/temporary_class_for_specs.rb
 - spec/test_support/shrine_spec_support.rb
 - spec/test_support/images/2x2_pixel.jpg
+- spec/test_support/images/3x3_pixel.jpg
 - spec/test_support/images/1x1_pixel.jpg
 - spec/test_support/images/photo_800x586.jpg
 - spec/test_support/audio/README.md

data/app/models/kithe/asset/derivative_updater.rb

@@ -1,119 +0,0 @@
-# A service object to add an IO stream as a derivative with a certain key, to a asset.
-# Adds a Derivative database object for such. This class is normally only used from
-# Asset#update_derivative, it's a helper object, you aren't expected to use it independently.
-#
-# This would be very straightforward if it weren't for taking account of a couple concurrency race
-# conditions involving data integrity:
-#
-# 1. There should be only one derivative for a given asset/key pair. This is enforced by
-#   a DB constraint. If the record already exists, we want to update the current record,
-#   otherwise add a new one. We want to do this in a race-condition safe way, with possibly
-#   multiple processes editing db.
-#
-# 2. The DB should at no point in time contain a derivative generated for an _old_ version
-#   of the asset. If an asset#file is changed, it's existing derivatives need to be deleted,
-#   and this needs to happen in a race-condition safe way when something may be trying to
-#   add a derivative concurrently.
-#
-# I believe we have solved those challenges, but it leads to a bit tricky code. We use
-# a kind of "optimistic" approach to (1) (try to insert, if you get a uniqueness violation
-# try to find and use the record that's already there). And a "pessimistic" approach to (2),
-# where we actually briefly take out a DB pessimistic lock to make sure the asset#file hasn't
-# changed, and can't until we're done updating.  (using sha512 as a marker, which is why you
-# can't add an asset until it has a sha512 in it's metadata, usually post-promotion).
-#
-# If we made a given Asset objects's file bytestream immutable, this would all be a lot simpler;
-# we wouldn't need to worry about (2) at all, and maybe not even (1). We might consider that, but
-# for now we're tackling the hard way.
-#
-class Kithe::Asset::DerivativeUpdater
-  attr_reader :asset, :key, :io, :storage_key, :metadata, :max_optimistic_tries
-
-  def initialize(asset, key, io, storage_key: :kithe_derivatives, metadata: {})
-    @asset = asset
-    @key = key
-    @io = io
-    @storage_key = storage_key
-    @metadata = metadata
-
-    @max_optimistic_tries = 3
-
-    unless asset_has_persisted_sha512?
-      raise ArgumentError.new("Can not safely add derivative to an asset without a persisted sha512 value")
-    end
-  end
-
-  def update
-    deriv = Kithe::Derivative.new(key: key.to_s, asset: asset)
-
-    # skip cache phase, right to specified storage, but with metadata extraction.
-    uploader = deriv.file_attacher.shrine_class.new(storage_key)
-
-    # add our derivative key to context when uploading, so Kithe::DerivativeUploader can
-    # use it if needed.
-    uploaded_file = uploader.upload(io, record: deriv, metadata: metadata, kithe_derivative_key: key)
-    optimistically_save_derivative(uploaded_file: uploaded_file, derivative: deriv)
-  end
-
-  # Attaches UploadedFile to Derivative and tries to save it -- if we get a
-  # unique constraint violation because a Derivative for that asset/key already existed,
-  # we fetch that alredy existing one from the db and update it's actual bytestream.
-  #
-  # This method calls itself recursively to do that. Gives up after max_optimistic_tries,
-  # at which point it'll just raise the constraint violation exception.
-  def optimistically_save_derivative(uploaded_file:, derivative:, tries: 0)
-    derivative.file_attacher.change(uploaded_file)
-    save_deriv_ensuring_unchanged_asset(derivative)
-  rescue ActiveRecord::RecordNotUnique => e
-    if tries < max_optimistic_tries
-      # find the one that's already there, try to attach our new file
-      # to that one
-      derivative = Kithe::Derivative.where(key: key.to_s, asset: asset).first || derivative
-      optimistically_save_derivative(uploaded_file: uploaded_file, derivative: derivative, tries: tries + 1)
-    else
-      uploaded_file.delete if uploaded_file
-      raise e
-    end
-  rescue StandardError => e
-    # aggressively clean up our file on errors!
-    uploaded_file.delete if uploaded_file
-    raise e
-  end
-
-  # Save a Derivative model with some fancy DB footwork to ensure at the time
-  # we save it, the original asset file it is based on is still in db unchanged,
-  # in a concurrency-safe way.
-  #
-  # We re-fetch to ensure asset still exists, with sha512 we expect. (kithe model ensures sha512
-  # exists in shrine metadata). With a pessmistic lock in a transaction. This ensures that at
-  # the point we save the new derivative, the db is still in a state where the original file
-  # the derivative relates to is still in the db.
-  #
-  # Can raise a ActiveRecord::RecordNotUnique, if derivative unique constraint is violated,
-  # that is handled above here.
-  def save_deriv_ensuring_unchanged_asset(deriv)
-    # fancy throw/catch keep our abort rescue from being in the transaction
-    catch(:kithe_unchanged_abort) do
-      Kithe::Asset.transaction do
-        # the file we're trying to add a derivative to doesn't exist anymore, forget it
-        unless asset.acquire_lock_on_sha
-          throw :kithe_unchanged_abort
-        end
-
-        deriv.save!
-        return deriv
-      end
-    end
-
-    # If we made it here, we've aborted
-    deriv.file.delete
-    return nil
-  end
-
-  def asset_has_persisted_sha512?
-    asset.persisted? && asset.sha512.present? &&
-    !( asset.file_data_changed? &&
-         asset.file_data_change.first.try(:dig, "metadata", "sha512") !=
-           asset.file_data_change.second.try(:dig, "metadata", "sha512"))
-  end
-end