iiif_print 1.0.0

data/spec/services/iiif_print/pluggable_derivative_service_spec.rb

@@ -0,0 +1,178 @@
+require 'fileutils'
+require 'spec_helper'
+
+RSpec.describe IiifPrint::PluggableDerivativeService do
+  let(:persisted_file_set) do
+    fs = FileSet.new
+    work.title = ['This is a page!']
+    work.members.push(fs)
+    fs.instance_variable_set(:@mime_type, 'image/tiff')
+    fs.save!(validate: false)
+    work.save!(validate: false)
+    fs
+  end
+
+  let(:fixture_path) do
+    File.join(
+      IiifPrint::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  describe "service registration" do
+    # integration test with Hyrax, verify services is registered
+
+    it "is registered with Hyrax" do
+      expect(Hyrax::DerivativeService.services).to include described_class
+    end
+
+    it "is the first valid service found" do
+      file_set = double(FileSet,
+                        class: FileSet,
+                        mime_type: 'application/pdf',
+                        parent: MyIiifConfiguredWorkWithAllDerivativeServices.new)
+      found = Hyrax::DerivativeService.for(file_set)
+      expect(found).to be_a described_class
+    end
+  end
+
+  context "when the FileSet's parent is not IiifPrint configured" do
+    before do
+      allow(persisted_file_set).to receive(:in_works).and_return([work])
+    end
+
+    let(:work) { MyWork.new }
+
+    describe "#plugins" do
+      it "uses the default derivatives service" do
+        file_set = double(FileSet,
+                          class: FileSet,
+                          mime_type: 'application/pdf',
+                          parent: MyWork.new)
+        service = described_class.new(file_set)
+        expect(service.plugins).to eq [Hyrax::FileSetDerivativesService]
+      end
+    end
+  end
+
+  context "when the FileSet's parent is IiifPrint configured" do
+    describe "calls the configured derivative plugins" do
+      before do
+        allow(persisted_file_set).to receive(:in_works).and_return([work])
+        allow_any_instance_of(Hyrax::FileSetDerivativesService).to receive(:send)
+      end
+
+      let(:work) { MyIiifConfiguredWork.new }
+      let(:plugin) { FakeDerivativeService.new }
+
+      it "calls each plugin on create" do
+        service = described_class.new(persisted_file_set, plugins: [plugin])
+        expect do
+          service.create_derivatives('not_a_real_filename')
+        end.to change(plugin, :create_called).by(1)
+      end
+
+      def touch_fake_derivative_file(file_set, ext)
+        path = Hyrax::DerivativePath.derivative_path_for_reference(file_set, ext)
+        FileUtils.mkdir_p(File.join(path.split('/')[0..-2]))
+        FileUtils.touch(path)
+      end
+
+      it "does not re-create existing derivative" do
+        service = described_class.new(persisted_file_set, plugins: [plugin])
+        expect(persisted_file_set.id).not_to be_nil
+        expect do
+          touch_fake_derivative_file(persisted_file_set, plugin.target_extension)
+          service.create_derivatives('/nonsense/source/path/ignored ')
+        end.not_to change(plugin, :create_called)
+      end
+
+      it "calls each plugin on cleanup" do
+        service = described_class.new(persisted_file_set, plugins: [plugin])
+        expect { service.cleanup_derivatives }.to change(plugin, :cleanup_called).by(1)
+      end
+    end
+
+    context "integration tests for plugins" do
+      before do
+        allow(persisted_file_set).to receive(:in_works).and_return([work])
+      end
+
+      let(:work) { MyIiifConfiguredWorkWithAllDerivativeServices.new }
+
+      describe "calls all derivative plugins" do
+        def source_image(name)
+          File.join(fixture_path, name)
+        end
+
+        def derivatives_for(file_set)
+          Hyrax::DerivativePath.derivatives_for_reference(file_set)
+        end
+
+        def expected_plugins
+          [
+            Hyrax::FileSetDerivativesService,
+            IiifPrint::JP2DerivativeService,
+            IiifPrint::PDFDerivativeService,
+            IiifPrint::TextExtractionDerivativeService,
+            IiifPrint::TIFFDerivativeService
+          ]
+        end
+
+        # The expected set of Plugins that will run for file set
+        it "has expected valid plugins configured" do
+          plugins = described_class.new(persisted_file_set).plugins
+          fs = persisted_file_set
+          services = plugins.map { |plugin| plugin.new(fs) }.select(&:valid?)
+          expect(services.length).to eq 5
+          used_plugins = services.map(&:class)
+          expected_plugins.each do |plugin|
+            expect(used_plugins).to include plugin
+          end
+        end
+
+        it "creates expected derivatives from TIFF source" do
+          svc = described_class.new(persisted_file_set)
+          svc.create_derivatives(source_image('4.1.07.tiff'))
+          made = derivatives_for(persisted_file_set)
+          made.each { |path| expect(File.exist?(path)) }
+          extensions = made.map { |path| path.split('.')[-1] }
+          expect(extensions).to include 'pdf'
+          expect(extensions).to include 'jp2'
+          expect(extensions).not_to include 'tiff'
+          # Thumbnail, created by Hyrax:
+          expect(extensions).to include 'jpeg'
+        end
+      end
+
+      describe "ingest integration" do
+        def log_attachment(file_set)
+          # create a log entry for the fileset given destination name 'jp2'
+          IiifPrint::DerivativeAttachment.create(
+            fileset_id: file_set.id,
+            path: '/some/arbitrary/path/to.jp2',
+            destination_name: 'jp2'
+          )
+        end
+
+        def jp2_plugin?(plugins)
+          r = plugins.select { |p| p.is_a? IiifPrint::JP2DerivativeService }
+          !r.empty?
+        end
+
+        it "will not attempt creating over pre-made derivative" do
+          service = described_class.new(persisted_file_set)
+          # this should be respected, evaluate by obtaining filtered
+          #   services list, which must omit JP2DerivativeService
+          plugins = service.services(:create_derivatives)
+          # initially has jp2 plugin
+          expect(jp2_plugin?(plugins)).to be true
+          # blacklist jp2 by effect of log entry of pre-made attachment
+          log_attachment(service.file_set)
+          # omits, after logging intent of previous attachment:
+          plugins = service.services(:create_derivatives)
+          expect(jp2_plugin?(plugins)).to be false
+        end
+      end
+    end
+  end
+end

data/spec/services/iiif_print/text_extraction_derivative_service_spec.rb

@@ -0,0 +1,82 @@
+require 'nokogiri'
+require 'spec_helper'
+require 'misc_shared'
+
+RSpec.describe IiifPrint::TextExtractionDerivativeService do
+  include_context "shared setup"
+
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:work) do
+    work = NewspaperPage.create(title: ["Hello"])
+    work.members << valid_file_set
+    work.save!
+  end
+
+  let(:minimal_alto) do
+    File.join(fixture_path, 'minimal-alto.xml')
+  end
+
+  let(:altoxsd) do
+    xsdpath = File.join(fixture_path, 'alto-2-0.xsd')
+    Nokogiri::XML::Schema(File.read(xsdpath))
+  end
+
+  describe "Creates ALTO derivative" do
+    def source_image(name)
+      File.join(fixture_path, name)
+    end
+
+    def expected_path(file_set, ext)
+      Hyrax::DerivativePath.derivative_path_for_reference(file_set, ext)
+    end
+
+    def validate_alto(filename)
+      altoxsd.validate(filename)
+    end
+
+    def derivative_exists(ext)
+      path = expected_path(valid_file_set, ext)
+      expect(File.exist?(path)).to be true
+      expect(File.size(path)).to be > 0
+    end
+
+    xit "creates, stores valid ALTO and plain-text derivatives" do
+      # these are in same test to avoid duplicate OCR operation
+      service = described_class.new(valid_file_set)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+      # ALTO derivative file exists at expected path and validates:
+      altoxsd.validate(expected_path(valid_file_set, 'xml'))
+      # Plain text exists as non-empty file:
+      derivative_exists('txt')
+      derivative_exists('json')
+      json_path = expected_path(valid_file_set, 'json')
+      loaded_result = JSON.parse(File.read(json_path))
+      expect(loaded_result['coords'].length).to be > 1
+    end
+
+    xit "usually uses OCR, when no existing text" do
+      service = described_class.new(valid_file_set)
+      # here, service will delegate create_derivatives to OCR impl method:
+      expect(service).to receive(:create_derivatives_from_ocr)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+    end
+
+    xit "defers to existing ALTO sources, when present" do
+      # Attach some ALTO to a work
+      derivatives = IiifPrint::Data::WorkDerivatives.of(
+        work,
+        valid_file_set
+      )
+      derivatives.attach(minimal_alto, 'xml')
+      # In this case, service will not call the OCR implementation method:
+      service = described_class.new(valid_file_set)
+      expect(service).not_to receive(:create_derivatives_from_ocr)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+    end
+  end
+end

data/spec/services/iiif_print/text_formats_from_alto_service_spec.rb

@@ -0,0 +1,127 @@
+require 'nokogiri'
+require 'spec_helper'
+require 'misc_shared'
+
+RSpec.describe IiifPrint::TextFormatsFromALTOService do
+  include_context "shared setup"
+
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:work) do
+    work = NewspaperPage.create(title: ["Hello"])
+    work.members << valid_file_set
+    work.save!
+    work
+  end
+
+  let(:minimal_alto) do
+    File.join(fixture_path, 'minimal-alto.xml')
+  end
+
+  def log_incoming_attachment(fsid)
+    IiifPrint::DerivativeAttachment.create!(
+      fileset_id: fsid,
+      path: minimal_alto,
+      destination_name: 'xml'
+    )
+  end
+
+  def derivatives_of(work, fileset)
+    IiifPrint::Data::WorkDerivatives.of(work, fileset)
+  end
+
+  describe "Saves other formats from ALTO" do
+    xit "saves JSON, text from existing ALTO derivative" do
+      derivatives = derivatives_of(work, valid_file_set)
+      expect(derivatives.keys.size).to eq 0
+      derivatives.attach(minimal_alto, 'xml')
+      expect(derivatives.keys.size).to eq 1
+      service = described_class.new(valid_file_set)
+      service.create_derivatives('/some/random/primary/path/does_not/matter')
+      derivatives.load_paths
+      expect(derivatives.keys.size).to eq 3
+      expect(derivatives.keys).to include 'json', 'txt'
+    end
+
+    xit "saves JSON, text from incoming ALTO derivative" do
+      derivatives = derivatives_of(work, valid_file_set)
+      expect(derivatives.keys.size).to eq 0
+      log_incoming_attachment(valid_file_set.id)
+      service = described_class.new(valid_file_set)
+      service.create_derivatives('/some/random/primary/path/does_not/matter')
+      # reload keys to check derivatives:
+      derivatives.load_paths
+      expect(derivatives.keys).to include 'json', 'txt'
+    end
+  end
+
+  describe "scaling matters" do
+    # we need an ingested, characterized file:
+    do_now_jobs = [
+      IngestLocalFileJob,
+      IngestJob,
+      InheritPermissionsJob,
+      CharacterizeJob
+    ]
+    # we omit CreateDerivativesJob from above, as obviously duplicative and
+    # therefore potential cause of problems here.
+
+    # remove any previous test run (development) artifacts in file
+    #   attachment logging tables
+    before(:all) do
+      IiifPrint::DerivativeAttachment.all.delete_all
+      IiifPrint::IngestFileRelation.all.delete_all
+    end
+
+    let(:work) do
+      work = NewspaperPage.create(title: ["Hello"])
+      work
+    end
+
+    let(:tiff_path) { File.join(fixture_path, 'ocr_gray.tiff') }
+    let(:ocr_alto_path) do
+      File.join(fixture_path, 'ocr_alto_scaled_4pts_per_px.xml')
+    end
+
+    def attach_primary_file(work)
+      IiifPrint::Data::WorkFiles.assign!(to: work, path: tiff_path)
+      work.reload
+      pcdm_file = IiifPrint::Data::WorkFiles.of(work).values[0].unwrapped
+      expect(pcdm_file).not_to be_nil
+      # we have image dimensions (px) to work with:
+      expect(pcdm_file.width[0].to_i).to be_an Integer
+      expect(pcdm_file.height[0].to_i).to be_an Integer
+    end
+
+    def derivatives_of(work)
+      IiifPrint::Data::WorkFiles.of(work).derivatives
+    end
+
+    def attach_alto(work)
+      derivatives = derivatives_of(work)
+      derivatives.attach(ocr_alto_path, 'xml')
+      # has a path to now-stored derivative:
+      expect(derivatives.path('xml')).not_to be_nil
+    end
+
+    xit "scales ALTO points to original image", perform_enqueued: do_now_jobs do
+      attach_primary_file(work)
+      attach_alto(work)
+      work.reload
+      file_set = work.ordered_members.to_a.find { |m| m.is_a? FileSet }
+      service = described_class.new(file_set)
+      service.create_derivatives('/a/path/here/needed/but/will/not/matter')
+      coords = JSON.parse(derivatives_of(work).data('json'))
+      word = coords['coords'].select { |k, _v| k == 'Bethesda' }
+      # test against known scaled coordinate of OCR data:
+      #   This roughly matches unscaled ALTO data for token 'Bethesda'
+      #   in spec/fixtures/files/ocr_alto.xml, with the disclaimer that
+      #   round-trip rounding error of 1px is noted for VPOS.
+      expect(word['Bethesda']).to eq [[16, 665, 78, 16]]
+    end
+  end
+end

data/spec/services/iiif_print/tiff_derivative_service_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+RSpec.describe IiifPrint::TIFFDerivativeService do
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:fixture_path) do
+    File.join(
+      IiifPrint::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  describe "Creates TIFF derivatives" do
+    def source_image(name)
+      File.join(fixture_path, name)
+    end
+
+    def expected_path(file_set)
+      Hyrax::DerivativePath.derivative_path_for_reference(file_set, 'tiff')
+    end
+
+    def get_res(path)
+      tool = IiifPrint::ImageTool.new(path)
+      "#{tool.metadata[:width]}x#{tool.metadata[:height]}"
+    end
+
+    def check_dpi_match(orig, dest)
+      # check ppi, but skip pdf to avoid ghostscript warnings to stderr
+      expect(get_res(orig)).to eq get_res(dest) unless orig.end_with?('pdf')
+    end
+
+    def makes_tiff(filename)
+      path = source_image(filename)
+      expected = expected_path(valid_file_set)
+      expect(File.exist?(expected)).to be false
+      svc = described_class.new(valid_file_set)
+      svc.create_derivatives(path)
+      expect(File.exist?(expected)).to be true
+      mime = IiifPrint::ImageTool.new(expected).metadata[:content_type]
+      expect(mime).to eq 'image/tiff'
+      check_dpi_match(path, expected)
+      svc.cleanup_derivatives
+    end
+
+    # for cases where primary file is TIFF already
+    def avoids_duplicative_creation(filename)
+      expected = expected_path(valid_file_set)
+      expect(File.exist?(expected)).to be false
+      svc = described_class.new(valid_file_set)
+      svc.create_derivatives(source_image(filename))
+      expect(File.exist?(expected)).not_to be true
+    end
+
+    it "Does not make TIFF derivatives when primary is TIFF" do
+      avoids_duplicative_creation('ocr_mono.tiff')
+      avoids_duplicative_creation('ocr_gray.tiff')
+    end
+
+    it "creates TIFF from PDF source, robust to multi-page" do
+      makes_tiff('sample-color-newsletter.pdf')
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,181 @@
+require 'json'
+
+# testing environent:
+ENV['RAILS_ENV'] ||= 'test'
+
+require 'coveralls'
+Coveralls.wear!
+
+require 'shoulda/matchers'
+Shoulda::Matchers.configure do |config|
+  config.integrate do |with|
+    with.test_framework :rspec
+  end
+end
+
+# engine_cart:
+require 'bundler/setup'
+require 'engine_cart'
+EngineCart.load_application!
+
+require 'rspec/rails'
+require 'support/iiif_print_models'
+require 'support/controller_level_helpers'
+require 'rspec/active_model/mocks'
+
+ActiveJob::Base.queue_adapter = :test
+
+RSpec.configure do |config|
+  # enable FactoryBot:
+  require 'factory_bot'
+  config.include FactoryBot::Syntax::Methods
+  # auto-detect and load all factories in spec/factories:
+  FactoryBot.find_definitions
+
+  config.infer_spec_type_from_file_location!
+
+  # Transactional
+  config.use_transactional_fixtures = false
+  config.include Devise::Test::ControllerHelpers, type: :controller
+
+  # ensure Hyrax has active sipity workflow for default admin set:
+  config.before(:suite) do
+    require 'active_fedora/cleaner'
+    require 'database_cleaner'
+
+    # By default, Hyrax uses a database minter class.  That's the preferred pathway (because you are
+    # tracking minting state in the database).  However, for testing purposes we don't need to / nor
+    # want to install the minter migrations.  Hence we're favoring this approach.
+    minter_class = ::Noid::Rails::Minter::File
+    ::Noid::Rails.config.minter_class = minter_class
+    Hyrax.config.noid_minter_class = minter_class
+
+    ActiveFedora::Cleaner.clean!
+    DatabaseCleaner.clean_with(:truncation)
+
+    begin
+      # TODO: switch the below methods to use the appropriate services
+      # rather than the deprecated methods currently being used.
+      # ensure permission template actually exists in RDBMS:
+      id = 'admin_set/default'
+      no_template = Hyrax::PermissionTemplate.find_by(source_id: id).nil?
+      Hyrax::PermissionTemplate.create!(source_id: id) if no_template
+      # ensure workflows exist, presumes permission template does first:
+      Hyrax::Workflow::WorkflowImporter.load_workflows
+      # Default admin set needs to exist in Fedora, with relation to its
+      #   PermissionTemplate object:
+      begin
+        admin_set = AdminSet.find(AdminSet.find_or_create_default_admin_set_id)
+        admin_set.save!
+      rescue ActiveRecord::RecordNotUnique
+        admin_set = AdminSet.find(AdminSet::DEFAULT_ID)
+      end
+      permission_template = admin_set.permission_template
+      workflow = permission_template.available_workflows.where(
+        name: 'default'
+      ).first
+      Sipity::Workflow.activate!(
+        permission_template: permission_template,
+        workflow_id: workflow.id
+      )
+    rescue Faraday::ConnectionFailed
+      STDERR.puts "Attempting to run test suite without Fedora and/or Solr..."
+    end
+  end
+
+  # :perform_enqueued config setting below copied from Hyrax spec_helper.rb
+  config.before(:example, :perform_enqueued) do |example|
+    ActiveJob::Base.queue_adapter.filter = example.metadata[:perform_enqueued].try(:to_a)
+    ActiveJob::Base.queue_adapter.perform_enqueued_jobs = true
+    ActiveJob::Base.queue_adapter.perform_enqueued_at_jobs = true
+  end
+  config.after(:example, :perform_enqueued) do
+    ActiveJob::Base.queue_adapter.filter = nil
+    ActiveJob::Base.queue_adapter.enqueued_jobs = []
+    ActiveJob::Base.queue_adapter.performed_jobs = []
+    ActiveJob::Base.queue_adapter.perform_enqueued_jobs = false
+    ActiveJob::Base.queue_adapter.perform_enqueued_at_jobs = false
+  end
+  config.after(:suite) do # or :each or :all
+    FileUtils.rm_rf(Dir[Rails.root.join('tmp', 'derivatives', '*')])
+  end
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  # The settings below are suggested to provide a good initial experience
+  # with RSpec, but feel free to customize to your heart's content.
+
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  # config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  # config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  # config.disable_monkey_patching!
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  # if config.files_to_run.one?
+  # Use the documentation formatter for detailed output,
+  # unless a formatter has already been configured
+  # (e.g. via a command-line flag).
+  #  config.default_formatter = "doc"
+  # end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+end

data/spec/support/controller_level_helpers.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# copied from Hyrax
+
+module ControllerLevelHelpers
+  # This provides some common mock methods for view tests.
+  # These are normally provided by the controller.
+  module ControllerViewHelpers
+    def search_state
+      @search_state ||= CatalogController.search_state_class.new(params, blacklight_config, controller)
+    end
+
+    # This allows you to set the configuration
+    # @example: view.blacklight_config = Blacklight::Configuration.new
+    attr_writer :blacklight_config
+
+    def blacklight_config
+      @blacklight_config ||= CatalogController.blacklight_config
+    end
+
+    def blacklight_configuration_context
+      @blacklight_configuration_context ||= Blacklight::Configuration::Context.new(controller)
+    end
+  end
+
+  def initialize_controller_helpers(helper)
+    helper.extend ControllerViewHelpers
+  end
+end