iiif_print 1.0.0 → 1.1.0

data/lib/iiif_print/jobs/create_relationships_job.rb

@@ -1,41 +1,67 @@
 module IiifPrint
   module Jobs
-    # Break a pdf into individual pages
+    # Link newly created child works to the parent
     class CreateRelationshipsJob < IiifPrint::Jobs::ApplicationJob
-      # Link newly created child works to the parent
-      # @param user: [User] user
+      include Hyrax::Lockable
+
+      RETRY_MAX = 10
+
       # @param parent_id: [<String>] parent work id
       # @param parent_model: [<String>] parent model
       # @param child_model: [<String>] child model
-      def perform(user:, parent_id:, parent_model:, child_model:)
-        if completed_child_data_for(parent_id, child_model)
+      # @param retries: [<Integer>] count used during rescheduling to prevent infinite retries
+      def perform(parent_id:, parent_model:, child_model:, retries: 0, **)
+        @parent_id = parent_id
+        @parent_model = parent_model
+        @child_model = child_model
+        @retries = retries + 1
+
+        @number_of_successes = 0
+        @number_of_failures = 0
+        @parent_record_members_added = false
+        @errors = []
+
+        # Because we need our children in the correct order, we can't create any
+        # relationships until all child works have been created.
+        if completed_child_data
           # add the members
-          parent_work = parent_model.constantize.find(parent_id)
-          create_relationships(user: user, parent: parent_work, ordered_children: @child_works)
-          @pending_children.each(&:destroy)
+          add_children_to_parent
+          if @number_of_failures.zero? && @number_of_successes == @pending_children.count
+            # remove pending relationships upon valid completion
+            @pending_children.each(&:destroy)
+          elsif @number_of_failures.zero? && @number_of_successes > @pending_children.count
+            # remove pending relationships but raise error that too many relationships formed
+            @pending_children.each(&:destroy)
+            raise "CreateRelationshipsJob for parent id: #{@parent_id} " \
+                  "added #{@number_of_successes} children, " \
+                  "expected #{@pending_children} children."
+          else
+            # report failures & keep pending relationships
+            raise "CreateRelationshipsJob failed for parent id: #{@parent_id} " \
+                  "had #{@number_of_successes} successes & #{@number_of_failures} failures, " \
+                  "with errors: #{@errors}. Wanted #{@pending_children} children."
+          end
         else
-          # reschedule the job and end this one normally
-          #
-          # TODO: Depending on how things shake out, we could be infinitely rescheduling this job.
-          # Consider a time to live parameter.
-          reschedule(user: user, parent_id: parent_id, parent_model: parent_model, child_model: child_model)
+          # if we aren't ready yet, reschedule the job and end this one normally
+          reschedule_job
         end
       end
 
       private
 
-      # load @child_works, and return true or false
-      def completed_child_data_for(parent_id, child_model)
+      # load @child_works and @pending children, and
+      # return boolean indicating whether all chilren are present
+      def completed_child_data
         @child_works = []
         found_all_children = true
 
         # find and sequence all pending children
-        @pending_children = IiifPrint::PendingRelationship.where(parent_id: parent_id).order('child_order asc')
+        @pending_children = IiifPrint::PendingRelationship.where(parent_id: @parent_id).order('child_order asc')
 
         # find child works (skip out if any haven't yet been created)
         @pending_children.each do |child|
           # find by title... if any aren't found, the child works are not yet ready
-          found_children = find_children_by_title_for(child.child_title, child_model)
+          found_children = find_children_by_title_for(child.child_title, @child_model)
           found_all_children = false if found_children.empty?
           break unless found_all_children == true
           @child_works += found_children
@@ -49,30 +75,53 @@ module IiifPrint
         model.constantize.where(title: title)
       end
 
-      def reschedule(user:, parent_id:, parent_model:, child_model:)
+      def add_children_to_parent
+        parent_work = @parent_model.constantize.find(@parent_id)
+        create_relationships(parent: parent_work, ordered_children: @child_works)
+      end
+
+      def reschedule_job
+        return if @retries > RETRY_MAX
         CreateRelationshipsJob.set(wait: 10.minutes).perform_later(
-          user: user,
-          parent_id: parent_id,
-          parent_model: parent_model,
-          child_model: child_model
+          parent_id: @parent_id,
+          parent_model: @parent_model,
+          child_model: @child_model,
+          retries: @retries
         )
       end
 
-      def create_relationships(user:, parent:, ordered_children:)
-        records_hash = {}
-        ordered_children.map(&:id).each_with_index do |child_id, i|
-          records_hash[i.to_s] = { id: child_id }
+      def create_relationships(parent:, ordered_children:)
+        acquire_lock_for(parent.id) do
+          # Not sure uncached is needed here, but kept
+          # for consistency with Bulkrax's relationships logic
+          ActiveRecord::Base.uncached do
+            ordered_children.each do |child|
+              add_to_work(child_record: child, parent_record: parent)
+              @number_of_successes += 1
+            rescue => e
+              @number_of_failures += 1
+              @errors << e
+            end
+          end
+          parent.save! if @parent_record_members_added && @number_of_failures.zero?
         end
-        attrs = { work_members_attributes: records_hash }
-        parent.try(:reindex_extent=, Hyrax::Adapters::NestingIndexAdapter::LIMITED_REINDEX)
-        env = Hyrax::Actors::Environment.new(parent, Ability.new(user), attrs)
 
-        Hyrax::CurationConcern.actor.update(env)
-        # need to reindex all file_sets to make all ancestors are indexed
+        # Bulkrax no longer reindexes file_sets, but IiifPrint needs both to add is_page_of_ssim for universal viewer.
+        # This is where child works need to be indexed (AFTER the parent save), as opposed to how Bulkrax does it.
         ordered_children.each do |child_work|
+          child_work.update_index
           child_work.file_sets.each(&:update_index) if child_work.respond_to?(:file_sets)
         end
       end
+
+      def add_to_work(child_record:, parent_record:)
+        return true if parent_record.ordered_members.to_a.include?(child_record)
+
+        parent_record.ordered_members << child_record
+        @parent_record_members_added = true
+        # Bulkrax does child_record.save! here, but it makes no sense
+        # as there is nothing to save or index at this point.
+      end
     end
   end
 end

data/lib/iiif_print/jobs/request_split_pdf_job.rb

@@ -0,0 +1,31 @@
+module IiifPrint
+  module Jobs
+    ##
+    # Encapsulates logic for cleanup when the PDF is destroyed after pdf splitting into child works
+    class RequestSplitPdfJob < IiifPrint::Jobs::ApplicationJob
+      ##
+      # @param file_set [FileSet]
+      # @param user [User]
+      # rubocop:disable Metrics/MethodLength
+      def perform(file_set:, user:)
+        return true unless file_set.pdf?
+
+        work = IiifPrint.parent_for(file_set)
+
+        # Woe is ye who changes the configuration of the model, thus removing the splitting.
+        raise WorkNotConfiguredToSplitFileSetError.new(work: work, file_set: file_set) unless work&.iiif_print_config&.pdf_splitter_job&.presence
+
+        # clean up any existing spawned child works of this file_set
+        IiifPrint::SplitPdfs::DestroyPdfChildWorksService.conditionally_destroy_spawned_children_of(
+          file_set: file_set,
+          work: work
+        )
+
+        location = Hyrax::WorkingDirectory.find_or_retrieve(file_set.files.first.id, file_set.id)
+
+        IiifPrint.conditionally_submit_split_for(work: work, file_set: file_set, locations: [location], user: user)
+      end
+      # rubocop:enable Metrics/MethodLength
+    end
+  end
+end

data/lib/iiif_print/lineage_service.rb

@@ -2,7 +2,7 @@ module IiifPrint
   # The purpose of this module is to encode lineage related services:
   #
   # - {.ancestor_ids_for}
-  # - {.descendent_file_set_ids_for}
+  # - {.descendent_member_ids_for}
   #
   # The ancestor and descendent_file_sets are useful for ensuring we index together related items.
   # For example, when I have a work that is a book, and one file set per page of that book, when I
@@ -18,24 +18,45 @@ module IiifPrint
     def self.ancestor_ids_for(object)
       ancestor_ids ||= []
       object.in_works.each do |work|
-        ancestor_ids << work.id
+        ancestor_ids << ancestry_identifier_for(work)
         ancestor_ids += ancestor_ids_for(work) if work.is_child
       end
       ancestor_ids.flatten.compact.uniq
     end
 
+    ##
+    # @api public
+    #
+    # Given the :work return it's identifier
+    #
+    # @param [Object]
+    # @return [String]
+    def self.ancestry_identifier_for(work)
+      IiifPrint.config.ancestory_identifier_function.call(work)
+    end
+
     ##
     # @param object [#ordered_works, #file_sets, #member_ids]
-    # @return [Array<String>] the ids of associated file sets
-    def self.descendent_file_set_ids_for(object)
+    # @return [Array<String>] the ids of associated file sets and child works
+    #
+    # @see
+    # https://github.com/samvera/hyrax/blob/2b807fe101176d594129ef8a8fe466d3d03a372b/app/indexers/hyrax/work_indexer.rb#L15-L18
+    # for "clarification" of the comingling of file_set_ids and member_ids
+    def self.descendent_member_ids_for(object)
       # enables us to return parents when searching for child OCR
-      file_set_ids = object.file_sets.map(&:id)
+      #
+      # https://github.com/samvera/hydra-works/blob/c9b9dd0cf11de671920ba0a7161db68ccf9b7f6d/lib/hydra/works/models/concerns/work_behavior.rb#L90-L92
+      #
+      # The Hydara::Works implementation of file_set_ids is "members.select(&:file_set?).map(&:id)";
+      # so no sense doing `object.file_set_ids + object.member_ids`
+      file_set_ids = object.member_ids
       object.ordered_works&.each do |child|
-        file_set_ids += descendent_file_set_ids_for(child)
+        file_set_ids += descendent_member_ids_for(child)
       end
-      # enables us to return parents when searching for child metadata
-      file_set_ids += object.member_ids
       file_set_ids.flatten.uniq.compact
     end
+    class << self
+      alias descendent_file_set_ids_for descendent_member_ids_for
+    end
   end
 end

data/lib/iiif_print/metadata.rb

@@ -17,48 +17,43 @@ module IiifPrint
       @base_url = base_url
     end
 
-    attr_reader :work, :version, :fields
+    attr_reader :work, :version, :fields, :current_ability
 
     def build_metadata
-      send("build_metadata_for_v#{version}")
-    end
-
-    private
-
-    def build_metadata_for_v2
       fields.map do |field|
-        label = Hyrax::Renderers::AttributeRenderer.new(field.name, nil).label
-        if field.name == :collection && member_of_collection?
-          viewable_collections = Hyrax::CollectionMemberService.run(work, @current_ability)
-          next if viewable_collections.empty?
-          { 'label' => label,
-            'value' => make_collection_link(viewable_collections) }
-        else
-          next if field_is_empty?(field)
-          { 'label' => label,
-            'value' => cast_to_value(field_name: field.name, options: field.options) }
+        values = values_for(field_name: field)
+        if field.name == :collection && member_of_collection? && viewable_collections.present?
+          { 'label' => metadata_map(field, :label),
+            'value' => metadata_map(field, :collection) }
+        elsif values.present? && !empty_string?(values)
+          { 'label' => metadata_map(field, :label),
+            'value' => metadata_map(field, :value) }
         end
       end.compact
     end
 
-    def build_metadata_for_v3
-      fields.map do |field|
-        values = Array(work.try(field.name)).map { |value| scrub(value.to_s) }
-        next if values.empty?
-        {
-          'label' => {
-            # Since we're using I18n to translate the field, we're setting the locale used in the translation.
-            I18n.locale.to_s => [Hyrax::Renderers::AttributeRenderer.new(field.name, nil).label]
-          },
-          'value' => {
-            'none' => values
-          }
-        }
-      end.compact
+    private
+
+    def metadata_map(field, property)
+      if version == 2
+        case property
+        when :label      then field.label
+        when :value      then cast_to_value(field_name: field.name, options: field.options)
+        when :collection then make_collection_link(viewable_collections)
+        end
+      elsif version == 3
+        case property
+        when :label      then { I18n.locale.to_s => [field.label] }
+        when :value      then { 'none' => cast_to_value(field_name: field.name, options: field.options) }
+        when :collection then { 'none' => make_collection_link(viewable_collections) }
+        end
+      end
     end
 
-    def field_is_empty?(field)
-      Array(work.try(field.name)).empty?
+    # Bulkrax imports values as [""] if there isn't a value but still a header,
+    # these fields should not show in the metadata pane
+    def empty_string?(values)
+      values.uniq.size == 1 ? values.first == "" : false
     end
 
     def member_of_collection?
@@ -71,21 +66,41 @@ module IiifPrint
 
     def cast_to_value(field_name:, options:)
       if options&.[](:render_as) == :faceted
-        values_for(field_name: field_name).map do |value|
-          search_field = field_name.to_s + "_sim"
-          path = Rails.application.routes.url_helpers.search_catalog_path(
-            "f[#{search_field}][]": value, locale: I18n.locale
-          )
-          path += '&include_child_works=true' if work["is_child_bsi"] == true
-          "<a href='#{File.join(@base_url, path)}'>#{value}</a>"
-        end
+        faceted_values_for(field_name: field_name)
+      elsif qa_field?(field_name: options&.dig(:render_as) || field_name)
+        authority_values_for(field_name: field_name)
       else
         make_link(values_for(field_name: field_name))
       end
     end
 
+    def faceted_values_for(field_name:)
+      values_for(field_name: field_name).map do |value|
+        search_field = field_name.to_s + "_sim"
+        path = Rails.application.routes.url_helpers.search_catalog_path(
+          "f[#{search_field}][]": value, locale: I18n.locale
+        )
+        path += '&include_child_works=true' if work["is_child_bsi"] == true
+        "<a href='#{File.join(@base_url, path)}'>#{value}</a>"
+      end
+    end
+
+    def qa_field?(field_name:, questioning_authority_fields: IiifPrint.config.questioning_authority_fields)
+      questioning_authority_fields.include?(field_name.to_s)
+    end
+
+    def authority_values_for(field_name:)
+      authority = Qa::Authorities::Local.subauthority_for(field_name.to_s.pluralize)
+      values_for(field_name: field_name).map do |value|
+        id, term = authority.find(value).values_at('id', 'term')
+        "<a href='#{id}'>#{term}</a>"
+      end
+    end
+
     def values_for(field_name:)
-      Array(work.send(field_name))
+      field_name = field_name.try(:name) || field_name
+      # TODO: we are assuming tesim or dtsi (for dates), might want to account for other suffixes in the future
+      Array(work["#{field_name}_tesim"] || work["#{field_name}_dtsi"]&.to_date.try(:to_formatted_s, :standard))
     end
 
     def make_collection_link(collection_documents)
@@ -94,11 +109,16 @@ module IiifPrint
       end
     end
 
-    # @note This method turns link looking strings into links
+    def viewable_collections
+      Hyrax::CollectionMemberService.run(SolrDocument.find(work.id), current_ability)
+    end
+
+    # @note This method turns link looking strings into links and assumes https if not protocol was given
     def make_link(texts)
       texts.map do |t|
         t.to_s.gsub(MAKE_LINK_REGEX) do |url|
-          "<a href='#{url}' target='_blank'>#{url}</a>"
+          protocol = url.start_with?('www.') ? 'https://' : ''
+          "<a href='#{protocol}#{url}' target='_blank'>#{url}</a>"
         end
       end
     end
@@ -106,10 +126,9 @@ module IiifPrint
     MAKE_LINK_REGEX = %r{
       \b
       (
-        (?: [a-z][\w-]+:
-          (?: /{1,3} | [a-z0-9%] ) |
-            www\d{0,3}[.] |
-            [a-z0-9.\-]+[.][a-z]{2,4}/
+        (?:
+          (?:https?://) |
+          (?:www\.)
         )
         (?:
           [^\s()<>]+ | \(([^\s()<>]+|(\([^\s()<>]+\)))*\)

data/lib/iiif_print/split_pdfs/base_splitter.rb

@@ -0,0 +1,142 @@
+require 'open3'
+require 'securerandom'
+require 'tmpdir'
+require 'iiif_print/split_pdfs/pdf_image_extraction_service'
+
+module IiifPrint
+  module SplitPdfs
+    # @abstract
+    #
+    # The purpose of this class is to split the PDF into constituent image files.
+    #
+    # @see .call
+    class BaseSplitter
+      ##
+      # @api public
+      #
+      # @param path [String] local path to the PDF that we will split.
+      # @return [Enumerable]
+      #
+      # @see #each
+      #
+      # @note We're including the ** args to provide method conformity; other services require
+      #       additional information (such as the FileSet)
+      #
+      # @see IiifPrint::SplitPdfs::DerivativeRodeoSplitter
+      def self.call(path, **)
+        new(path).to_a
+      end
+
+      class_attribute :image_extension
+      class_attribute :compression, default: nil
+      class_attribute :quality, default: nil
+
+      def initialize(path, tmpdir: Dir.mktmpdir, default_dpi: 400)
+        @baseid = SecureRandom.uuid
+        @pdfpath = path
+        @pdfinfo = IiifPrint::SplitPdfs::PdfImageExtractionService.new(pdfpath)
+        @tmpdir = tmpdir
+        @default_dpi = default_dpi
+      end
+
+      # In creating {#each} we get many of the methods of array operation (e.g. #to_a).
+      include Enumerable
+
+      # @api public
+      #
+      # @yieldparam [String] the path to the page's tiff.
+      def each
+        entries.each do |e|
+          yield(e)
+        end
+      end
+
+      # @api private
+      #
+      # TODO: put this test somewhere to prevent invalid pdfs from crashing the image service.
+      def invalid_pdf?
+        return true if pdfinfo.color.include?(nil) || pdfinfo.width.nil? || pdfinfo.height.nil? || pdfinfo.page_count.zero?
+        false
+      end
+
+      attr_reader :pdfinfo, :tmpdir, :baseid, :default_dpi, :pdfpath
+      private :pdfinfo, :tmpdir, :baseid, :default_dpi, :pdfpath
+
+      private
+
+      # entries for each page
+      def entries
+        return @entries if defined? @entries
+
+        @entries = Array.wrap(gsconvert)
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      # ghostscript convert all pages to TIFF
+      def gsconvert
+        output_base = File.join(tmpdir, "#{baseid}-page%d.#{image_extension}")
+        # NOTE: you must call gsdevice before compression, as compression is
+        # updated during the gsdevice call.
+        cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} -dTextAlphaBits=4"
+        cmd += " -sCompression=#{compression}" if compression?
+        cmd += " -dJPEGQ=#{quality}" if quality?
+        cmd += " -sOutputFile=#{output_base} -r#{ppi} -f #{pdfpath}"
+        filenames = []
+
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          page_number = 0
+          stdout.read.split("\n").each do |line|
+            next unless line.start_with?('Page ')
+
+            page_number += 1
+            filenames << File.join(tmpdir, "#{baseid}-page#{page_number}.#{image_extension}")
+          end
+        end
+
+        filenames
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def gsdevice
+        raise NotImplementedError
+      end
+
+      PAGE_COUNT_REGEXP = %r{^Pages: +(\d+)$}.freeze
+
+      def pagecount
+        return @pagecount if defined? @pagecount
+
+        cmd = "pdfinfo #{pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          match = PAGE_COUNT_REGEXP.match(stdout.read)
+          @pagecount = match[1].to_i
+        end
+        @pagecount
+      end
+
+      def ppi
+        if looks_scanned?
+          # For scanned media, defer to detected image PPI:
+          pdfinfo.ppi
+        else
+          # 400 dpi for something that does not look like scanned media:
+          default_dpi
+        end
+      end
+
+      def looks_scanned?
+        max_image_px = pdfinfo.width * pdfinfo.height
+        # single 10mp+ image per page?
+        single_image_per_page? && max_image_px > 1024 * 1024 * 10
+      end
+
+      def single_image_per_page?
+        pdfinfo.page_count == pagecount
+      end
+    end
+  end
+end
+
+require "iiif_print/split_pdfs/pages_to_jpgs_splitter"
+require "iiif_print/split_pdfs/pages_to_pngs_splitter"
+require "iiif_print/split_pdfs/pages_to_tiffs_splitter"