newspaper_works 0.1.0

data/spec/services/newspaper_works/pluggable_derivative_service_spec.rb

@@ -0,0 +1,204 @@
+require 'fileutils'
+require 'spec_helper'
+
+RSpec.describe NewspaperWorks::PluggableDerivativeService do
+  let(:valid_file_set) { FileSet.new }
+
+  let(:persisted_file_set) do
+    fs = FileSet.new
+    work = NewspaperPage.new
+    work.title = ['This is a page!']
+    work.members.push(fs)
+    fs.instance_variable_set(:@mime_type, 'image/tiff')
+    fs.save!(validate: false)
+    work.save!(validate: false)
+    fs
+  end
+
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  # cache and restore originally described derivative service plugins
+  before do
+    @orig_plugins = described_class.plugins
+  end
+  after do
+    described_class.plugins = @orig_plugins
+  end
+
+  describe ".plugins=" do
+    it "allows setting of derivative service plugins" do
+      expect(described_class.plugins).to eq @orig_plugins
+      described_class.plugins = [Hyrax::FileSetDerivativesService] * 2
+      expect(described_class.plugins).to eq [Hyrax::FileSetDerivativesService] * 2
+    end
+  end
+
+  describe "calls all derivative plugins" do
+    class FakeDerivativeService
+      @create_called = 0
+      @cleanup_called = 0
+      class << self
+        attr_accessor :create_called, :cleanup_called
+
+        def target_ext
+          'txt'
+        end
+      end
+
+      def initialize(fileset)
+        @fileset = fileset
+        @created = false
+      end
+
+      def valid?
+        true
+      end
+
+      def create_derivatives(filename)
+        self.class.create_called += 1
+        filename
+      end
+
+      def cleanup_derivatives
+        self.class.cleanup_called += 1
+      end
+    end
+
+    def touch_fake_derivative_file(file_set, ext)
+      path = Hyrax::DerivativePath.derivative_path_for_reference(file_set, ext)
+      FileUtils.mkdir_p(File.join(path.split('/')[0..-2]))
+      FileUtils.touch(path)
+    end
+
+    it "calls each plugin on create" do
+      create_calls = FakeDerivativeService.create_called
+      described_class.plugins = [FakeDerivativeService, FakeDerivativeService]
+      service = described_class.new(FileSet.new)
+      service.create_derivatives('not_a_real_filename')
+      expect(FakeDerivativeService.create_called).to eq create_calls + 2
+    end
+
+    it "does not re-create existing derivative" do
+      create_calls = FakeDerivativeService.create_called
+      described_class.plugins = [FakeDerivativeService]
+      service = described_class.new(persisted_file_set)
+      expect(persisted_file_set.id).not_to be_nil
+      # Fake is configured to have 'txt' destination_path, let's create a
+      #   destination file in Hyrax's opinionated plate for dest. name.
+      touch_fake_derivative_file(persisted_file_set, 'txt')
+      service.create_derivatives('/nonsense/source/path/ignored')
+      # create calls logged by fake should not increment,
+      #   as PluggableDerivativeService should have skipped calling
+      #   plugin's create_derivatives method w/ presence of existing derivative
+      expect(FakeDerivativeService.create_called).to eq create_calls
+    end
+
+    it "calls each plugin on cleanup" do
+      expect(FakeDerivativeService.cleanup_called).to eq 0
+      described_class.plugins = [FakeDerivativeService, FakeDerivativeService]
+      service = described_class.new(FileSet.new)
+      service.cleanup_derivatives
+      expect(FakeDerivativeService.cleanup_called).to eq 2
+    end
+
+    it "test meta: spec restores original plugins" do
+      # verify `after do` clean up of plugins array to original value
+      plugins = described_class.plugins
+      expect(plugins.length).to eq @orig_plugins.length
+      expect(plugins).to include Hyrax::FileSetDerivativesService
+    end
+  end
+
+  describe "service registration" do
+    # integration test with Hyrax, verify services is registered
+
+    it "is registered with Hyrax" do
+      expect(Hyrax::DerivativeService.services).to include described_class
+    end
+
+    it "is the first valide service found" do
+      found = Hyrax::DerivativeService.for(FileSet.new)
+      expect(found.class).to be described_class
+    end
+  end
+
+  # integration tests for plugins
+  describe "runs multiple plugins, makes multiple derivatives" do
+    def source_image(name)
+      File.join(fixture_path, name)
+    end
+
+    def derivatives_for(file_set)
+      Hyrax::DerivativePath.derivatives_for_reference(file_set)
+    end
+
+    def expected_plugins
+      [
+        Hyrax::FileSetDerivativesService,
+        NewspaperWorks::JP2DerivativeService,
+        NewspaperWorks::PDFDerivativeService,
+        NewspaperWorks::TextExtractionDerivativeService,
+        NewspaperWorks::TIFFDerivativeService
+      ]
+    end
+
+    # The expected set of Plugins that will run for file set
+    it "has expected valid plugins configured" do
+      plugins = described_class.plugins
+      fs = persisted_file_set
+      services = plugins.map { |plugin| plugin.new(fs) }.select(&:valid?)
+      expect(services.length).to eq 5
+      used_plugins = services.map(&:class)
+      expected_plugins.each do |plugin|
+        expect(used_plugins).to include plugin
+      end
+    end
+
+    it "creates expected derivatives from TIFF source" do
+      svc = described_class.new(persisted_file_set)
+      svc.create_derivatives(source_image('4.1.07.tiff'))
+      made = derivatives_for(persisted_file_set)
+      made.each { |path| expect(File.exist?(path)) }
+      extensions = made.map { |path| path.split('.')[-1] }
+      expect(extensions).to include 'pdf'
+      expect(extensions).to include 'jp2'
+      expect(extensions).not_to include 'tiff'
+      # Thumbnail, created by Hyrax:
+      expect(extensions).to include 'jpeg'
+    end
+  end
+
+  describe "ingest integration" do
+    def log_attachment(file_set)
+      # create a log entry for the fileset given destination name 'jp2'
+      NewspaperWorks::DerivativeAttachment.create(
+        fileset_id: file_set.id,
+        path: '/some/arbitrary/path/to.jp2',
+        destination_name: 'jp2'
+      )
+    end
+
+    def jp2_plugin?(plugins)
+      r = plugins.select { |p| p.class == NewspaperWorks::JP2DerivativeService }
+      !r.empty?
+    end
+
+    it "will not attempt creating over pre-made derivative" do
+      service = described_class.new(persisted_file_set)
+      # this should be respected, evaluate by obtaining filtered
+      #   services list, which must omit JP2DerivativeService
+      plugins = service.services(:create_derivatives)
+      # initially has jp2 plugin
+      expect(jp2_plugin?(plugins)).to be true
+      # blacklist jp2 by effect of log entry of pre-made attachment
+      log_attachment(service.file_set)
+      # omits, after logging intent of previous attachment:
+      plugins = service.services(:create_derivatives)
+      expect(jp2_plugin?(plugins)).to be false
+    end
+  end
+end

data/spec/services/newspaper_works/text_extraction_derivative_service_spec.rb

@@ -0,0 +1,82 @@
+require 'nokogiri'
+require 'spec_helper'
+require 'misc_shared'
+
+RSpec.describe NewspaperWorks::TextExtractionDerivativeService do
+  include_context "shared setup"
+
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:work) do
+    work = NewspaperPage.create(title: ["Hello"])
+    work.members << valid_file_set
+    work.save!
+  end
+
+  let(:minimal_alto) do
+    File.join(fixture_path, 'minimal-alto.xml')
+  end
+
+  let(:altoxsd) do
+    xsdpath = File.join(fixture_path, 'alto-2-0.xsd')
+    Nokogiri::XML::Schema(File.read(xsdpath))
+  end
+
+  describe "Creates ALTO derivative" do
+    def source_image(name)
+      File.join(fixture_path, name)
+    end
+
+    def expected_path(file_set, ext)
+      Hyrax::DerivativePath.derivative_path_for_reference(file_set, ext)
+    end
+
+    def validate_alto(filename)
+      altoxsd.validate(filename)
+    end
+
+    def derivative_exists(ext)
+      path = expected_path(valid_file_set, ext)
+      expect(File.exist?(path)).to be true
+      expect(File.size(path)).to be > 0
+    end
+
+    it "creates, stores valid ALTO and plain-text derivatives" do
+      # these are in same test to avoid duplicate OCR operation
+      service = described_class.new(valid_file_set)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+      # ALTO derivative file exists at expected path and validates:
+      altoxsd.validate(expected_path(valid_file_set, 'xml'))
+      # Plain text exists as non-empty file:
+      derivative_exists('txt')
+      derivative_exists('json')
+      json_path = expected_path(valid_file_set, 'json')
+      loaded_result = JSON.parse(File.read(json_path))
+      expect(loaded_result['coords'].length).to be > 1
+    end
+
+    it "usually uses OCR, when no existing text" do
+      service = described_class.new(valid_file_set)
+      # here, service will delegate create_derivatives to OCR impl method:
+      expect(service).to receive(:create_derivatives_from_ocr)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+    end
+
+    it "defers to existing ALTO sources, when present" do
+      # Attach some ALTO to a work
+      derivatives = NewspaperWorks::Data::WorkDerivatives.of(
+        work,
+        valid_file_set
+      )
+      derivatives.attach(minimal_alto, 'xml')
+      # In this case, service will not call the OCR implementation method:
+      service = described_class.new(valid_file_set)
+      expect(service).not_to receive(:create_derivatives_from_ocr)
+      service.create_derivatives(source_image('ocr_mono.tiff'))
+    end
+  end
+end

data/spec/services/newspaper_works/text_formats_from_alto_service_spec.rb

@@ -0,0 +1,129 @@
+require 'nokogiri'
+require 'spec_helper'
+require 'misc_shared'
+
+RSpec.describe NewspaperWorks::TextFormatsFromALTOService do
+  include_context "shared setup"
+
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:work) do
+    work = NewspaperPage.create(title: ["Hello"])
+    work.members << valid_file_set
+    work.save!
+    work
+  end
+
+  let(:minimal_alto) do
+    File.join(fixture_path, 'minimal-alto.xml')
+  end
+
+  def log_incoming_attachment(fsid)
+    NewspaperWorks::DerivativeAttachment.create!(
+      fileset_id: fsid,
+      path: minimal_alto,
+      destination_name: 'xml'
+    )
+  end
+
+  def derivatives_of(work, fileset)
+    NewspaperWorks::Data::WorkDerivatives.of(work, fileset)
+  end
+
+  describe "Saves other formats from ALTO" do
+    it "saves JSON, text from existing ALTO derivative" do
+      derivatives = derivatives_of(work, valid_file_set)
+      expect(derivatives.keys.size).to eq 0
+      derivatives.attach(minimal_alto, 'xml')
+      expect(derivatives.keys.size).to eq 1
+      service = described_class.new(valid_file_set)
+      service.create_derivatives('/some/random/primary/path/does_not/matter')
+      derivatives.load_paths
+      expect(derivatives.keys.size).to eq 3
+      expect(derivatives.keys).to include 'json', 'txt'
+    end
+
+    it "saves JSON, text from incoming ALTO derivative" do
+      derivatives = derivatives_of(work, valid_file_set)
+      expect(derivatives.keys.size).to eq 0
+      log_incoming_attachment(valid_file_set.id)
+      service = described_class.new(valid_file_set)
+      service.create_derivatives('/some/random/primary/path/does_not/matter')
+      # reload keys to check derivatives:
+      derivatives.load_paths
+      expect(derivatives.keys).to include 'json', 'txt'
+    end
+  end
+
+  describe "scaling matters" do
+    # we need an ingested, characterized file:
+    do_now_jobs = [
+      IngestLocalFileJob,
+      IngestJob,
+      InheritPermissionsJob,
+      CharacterizeJob
+    ]
+    # we omit CreateDerivativesJob from above, as obviously duplicative and
+    # therefore potential cause of problems here.
+
+    # remove any previous test run (development) artifacts in file
+    #   attachment logging tables
+    before(:all) do
+      NewspaperWorks::DerivativeAttachment.all.delete_all
+      NewspaperWorks::IngestFileRelation.all.delete_all
+    end
+
+    let(:work) do
+      work = NewspaperPage.create(title: ["Hello"])
+      work
+    end
+
+    let(:tiff_path) { File.join(fixture_path, 'ocr_gray.tiff') }
+    let(:ocr_alto_path) do
+      File.join(fixture_path, 'ocr_alto_scaled_4pts_per_px.xml')
+    end
+
+    def attach_primary_file(work)
+      attachment = NewspaperWorks::Data::WorkFiles.of(work)
+      attachment.assign(tiff_path)
+      attachment.commit!
+      work.reload
+      pcdm_file = NewspaperWorks::Data::WorkFiles.of(work).values[0].unwrapped
+      expect(pcdm_file).not_to be_nil
+      # we have image dimensions (px) to work with:
+      expect(pcdm_file.width[0].to_i).to be_an Integer
+      expect(pcdm_file.height[0].to_i).to be_an Integer
+    end
+
+    def derivatives_of(work)
+      NewspaperWorks::Data::WorkFiles.of(work).derivatives
+    end
+
+    def attach_alto(work)
+      derivatives = derivatives_of(work)
+      derivatives.attach(ocr_alto_path, 'xml')
+      # has a path to now-stored derivative:
+      expect(derivatives.path('xml')).not_to be_nil
+    end
+
+    it "scales ALTO points to original image", perform_enqueued: do_now_jobs do
+      attach_primary_file(work)
+      attach_alto(work)
+      work.reload
+      file_set = work.ordered_members.to_a.select { |m| m.class == FileSet }[0]
+      service = described_class.new(file_set)
+      service.create_derivatives('/a/path/here/needed/but/will/not/matter')
+      coords = JSON.parse(derivatives_of(work).data('json'))
+      word = coords['coords'].select { |k, _v| k == 'Bethesda' }
+      # test against known scaled coordinate of OCR data:
+      #   This roughly matches unscaled ALTO data for token 'Bethesda'
+      #   in spec/fixtures/files/ocr_alto.xml, with the disclaimer that
+      #   round-trip rounding error of 1px is noted for VPOS.
+      expect(word['Bethesda']).to eq [[16, 665, 78, 16]]
+    end
+  end
+end

data/spec/services/newspaper_works/tiff_derivative_service_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+RSpec.describe NewspaperWorks::TIFFDerivativeService do
+  let(:valid_file_set) do
+    file_set = FileSet.new
+    file_set.save!(validate: false)
+    file_set
+  end
+
+  let(:fixture_path) do
+    File.join(
+      NewspaperWorks::GEM_PATH, 'spec', 'fixtures', 'files'
+    )
+  end
+
+  describe "Creates TIFF derivatives" do
+    def source_image(name)
+      File.join(fixture_path, name)
+    end
+
+    def expected_path(file_set)
+      Hyrax::DerivativePath.derivative_path_for_reference(file_set, 'tiff')
+    end
+
+    def get_res(path)
+      lines = `gm identify -verbose #{path}`.lines
+      lines.select { |line| line.strip.start_with?('Geometry') }[0].strip
+    end
+
+    def check_dpi_match(orig, dest)
+      # check ppi, but skip pdf to avoid ghostscript warnings to stderr
+      expect(get_res(orig)).to eq get_res(dest) unless orig.end_with?('pdf')
+    end
+
+    def makes_tiff(filename)
+      expected = expected_path(valid_file_set)
+      expect(File.exist?(expected)).to be false
+      svc = described_class.new(valid_file_set)
+      svc.create_derivatives(source_image(filename))
+      expect(File.exist?(expected)).to be true
+      desc = `gm identify #{expected}`
+      expect(desc).to include 'TIFF'
+      check_dpi_match(source_image(filename), expected)
+      svc.cleanup_derivatives
+    end
+
+    it "creates gray TIFF derivative from one-bit source" do
+      makes_tiff('page1.tiff')
+    end
+
+    it "creates gray TIFF from grayscale source" do
+      makes_tiff('lowres-gray-via-ndnp-sample.tiff')
+    end
+
+    it "creates TIFF from PDF source, robust to multi-page" do
+      makes_tiff('sample-color-newsletter.pdf')
+    end
+  end
+end