iiif_print 1.0.0 → 2.0.0

data/lib/iiif_print/split_pdfs/base_splitter.rb

@@ -0,0 +1,153 @@
+require 'open3'
+require 'securerandom'
+require 'tmpdir'
+require 'iiif_print/split_pdfs/pdf_image_extraction_service'
+
+module IiifPrint
+  module SplitPdfs
+    # @abstract
+    #
+    # The purpose of this class is to split the PDF into constituent image files.
+    #
+    # @see .call
+    class BaseSplitter
+      ##
+      # @api public
+      #
+      # @param path [String] local path to the PDF that we will split.
+      # @return [Enumerable]
+      #
+      # @see #each
+      #
+      # @note We're including the ** args to provide method conformity; other services require
+      #       additional information (such as the FileSet)
+      #
+      # @see IiifPrint::SplitPdfs::DerivativeRodeoSplitter
+      def self.call(path, **)
+        new(path).to_a
+      end
+
+      ##
+      # @api public
+      #
+      # Added to allow for fine-tuning of splitting decision such as tenant-based omission
+      # @see https://github.com/samvera/hyku/blob/main/app/services/iiif_print/tenant_config.rb
+      #
+      # @return [Boolean] returns false to not limit the splitting of PDFs
+      def self.never_split_pdfs?
+        false
+      end
+
+      class_attribute :image_extension
+      class_attribute :compression, default: nil
+      class_attribute :quality, default: nil
+
+      def initialize(path, tmpdir: Dir.mktmpdir, default_dpi: 400)
+        @baseid = SecureRandom.uuid
+        @pdfpath = path
+        @pdfinfo = IiifPrint::SplitPdfs::PdfImageExtractionService.new(pdfpath)
+        @tmpdir = tmpdir
+        @default_dpi = default_dpi
+      end
+
+      # In creating {#each} we get many of the methods of array operation (e.g. #to_a).
+      include Enumerable
+
+      # @api public
+      #
+      # @yieldparam [String] the path to the page's tiff.
+      def each
+        entries.each do |e|
+          yield(e)
+        end
+      end
+
+      # @api private
+      #
+      # TODO: put this test somewhere to prevent invalid pdfs from crashing the image service.
+      def invalid_pdf?
+        return true if pdfinfo.color.include?(nil) || pdfinfo.width.nil? || pdfinfo.height.nil? || pdfinfo.page_count.zero?
+        false
+      end
+
+      attr_reader :pdfinfo, :tmpdir, :baseid, :default_dpi, :pdfpath
+      private :pdfinfo, :tmpdir, :baseid, :default_dpi, :pdfpath
+
+      private
+
+      # entries for each page
+      def entries
+        return @entries if defined? @entries
+
+        @entries = Array.wrap(gsconvert)
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      # ghostscript convert all pages to TIFF
+      def gsconvert
+        output_base = File.join(tmpdir, "#{baseid}-page%d.#{image_extension}")
+        # NOTE: you must call gsdevice before compression, as compression is
+        # updated during the gsdevice call.
+        cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} -dTextAlphaBits=4"
+        cmd += " -sCompression=#{compression}" if compression?
+        cmd += " -dJPEGQ=#{quality}" if quality?
+        cmd += " -sOutputFile=#{output_base} -r#{ppi} -f #{pdfpath}"
+        filenames = []
+
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          page_number = 0
+          stdout.read.split("\n").each do |line|
+            next unless line.start_with?('Page ')
+
+            page_number += 1
+            filenames << File.join(tmpdir, "#{baseid}-page#{page_number}.#{image_extension}")
+          end
+        end
+
+        filenames
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      def gsdevice
+        raise NotImplementedError
+      end
+
+      PAGE_COUNT_REGEXP = %r{^Pages: +(\d+)$}.freeze
+
+      def pagecount
+        return @pagecount if defined? @pagecount
+
+        cmd = "pdfinfo #{pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          match = PAGE_COUNT_REGEXP.match(stdout.read)
+          @pagecount = match[1].to_i
+        end
+        @pagecount
+      end
+
+      def ppi
+        if looks_scanned?
+          # For scanned media, defer to detected image PPI:
+          pdfinfo.ppi
+        else
+          # 400 dpi for something that does not look like scanned media:
+          default_dpi
+        end
+      end
+
+      def looks_scanned?
+        max_image_px = pdfinfo.width * pdfinfo.height
+        # single 10mp+ image per page?
+        single_image_per_page? && max_image_px > 1024 * 1024 * 10
+      end
+
+      def single_image_per_page?
+        pdfinfo.page_count == pagecount
+      end
+    end
+  end
+end
+
+require "iiif_print/split_pdfs/pages_to_jpgs_splitter"
+require "iiif_print/split_pdfs/pages_to_pngs_splitter"
+require "iiif_print/split_pdfs/pages_to_tiffs_splitter"

data/lib/iiif_print/split_pdfs/child_work_creation_from_pdf_service.rb

@@ -1,29 +1,49 @@
 # frozen_string_literal: true
 
-# Encapsulates methods used for pdf splitting into child works
 module IiifPrint
   module SplitPdfs
+    ##
+    # Encapsulates methods used for pdf splitting into child works.
+    #
+    # The primary point of entry is {.conditionally_enqueue}.
     class ChildWorkCreationFromPdfService
-      # Load an array of paths to pdf files
-      # @param [Array > Hyrax::Upload file ids]
-      # @return [Array > String] file paths to temp directory
-      def self.pdf_paths(files:)
-        upload_ids = filter_file_ids(files)
-        return [] if upload_ids.empty?
-        uploads = Hyrax::UploadedFile.find(upload_ids)
-        paths = uploads.map(&method(:upload_path))
-        pdfs_only_for(paths)
-      end
+      ##
+      # Responsible for conditionally enqueueing the PDF splitting job.  The conditions attempt to
+      # sniff out whether the given file was a PDF.
+      #
+      # @param file_set [FileSet] What is the containing file set for the provided file.
+      # @param file [#path, #id]
+      # @param user [User] Who did the upload?
+      # @param import_url [NilClass, String] Provided when we're dealing with a file provided via a
+      #        URL.
+      # @param work [Hydra::PCDM::Work] An optional parameter that saves us a bit of time in not
+      #        needing to query for the parent of the given :file_set (see {.parent_for})
+      #
+      # @return [Symbol] when we don't enqueue the job
+      # @return [TrueClass] when we actually enqueue the job underlying job.
+      # rubocop:disable Metrics/MethodLength
+      def self.conditionally_enqueue(file_set:, file:, user:, import_url: nil, work: nil)
+        work ||= IiifPrint.parent_for(file_set)
 
-      # Is child work splitting defined for model?
-      # @param [GenericWork, etc] A valid type of hyrax work
-      # @return [Boolean]
-      def self.iiif_print_split?(work:)
-        # defined only if work has include IiifPrint.model_configuration with pdf_split_child_model
-        return true if work.try(:iiif_print_config)&.pdf_split_child_model
-        false
+        return :no_split_for_parent unless iiif_print_split?(work: work)
+        return :no_pdfs_to_split_for_import_url if import_url && !pdfs?(paths: [import_url])
+
+        file_locations = if import_url
+                           # TODO: Fix this logic, currently unsupported in Bulkrax
+                           [Hyrax::WorkingDirectory.find_or_retrieve(file.id, file_set.id)]
+                         else
+                           pdf_paths(file: file)
+                         end
+        return :no_pdfs_to_split if file_locations.empty?
+
+        IiifPrint.conditionally_submit_split_for(work: work, file_set: file_set, locations: file_locations, user: user)
+        :enqueued
       end
+      # rubocop:enable Metrics/MethodLength
 
+      ##
+      # @api private
+      #
       # Are there any PDF files?
       # @param [Array > String] paths to PDFs
       # @return [Boolean]
@@ -33,42 +53,68 @@ module IiifPrint
         true
       end
 
-      # Submit the job to split PDF into child works
+      ##
+      # @api private
+      # Load an array of paths to pdf files
+      # @param [Array > Hyrax::Upload file ids]
+      # @return [Array > String] file paths to temp directory
+      def self.pdf_paths(file:)
+        return [] unless file
+
+        if file.class < Valkyrie::Resource
+          # assuming that if one PDF is uploaded to a Valkyrie resource then all of them should be
+          paths = [Hyrax.storage_adapter.file_path(file.file_identifier)]
+          pdfs_only_for(paths)
+        else
+          upload_ids = filter_file_ids(file.id.to_s)
+          return [] if upload_ids.empty?
+
+          uploads = Hyrax::UploadedFile.find(upload_ids)
+          paths = uploads.map(&method(:upload_path))
+          pdfs_only_for(paths)
+        end
+      end
+
+      ##
+      # @api private
+      #
+      # Is child work splitting defined for model?
       # @param [GenericWork, etc] A valid type of hyrax work
-      # @param [Array<String>] paths to PDF attachments
-      # @param [User] user
-      # @param [Integer] number of pdfs already on existing work's filesets (not yet implemented)
-      def self.queue_job(work:, file_locations:, user:, admin_set_id:)
-        work.iiif_print_config.pdf_splitter_job.perform_later(
-          work,
-          file_locations,
-          user,
-          admin_set_id,
-          count_existing_pdfs(work)
-        )
+      # @return [Boolean]
+      def self.iiif_print_split?(work:)
+        config = work.try(:iiif_print_config)
+        return false unless config
+        return false if config.pdf_splitter_service.try(:never_split_pdfs?)
+        # defined only if work has include IiifPrint.model_configuration with pdf_split_child_model
+        return true if config&.pdf_split_child_model
+        false
       end
 
+      ##
+      # @api private
       def self.filter_file_ids(input)
         Array.wrap(input).select(&:present?)
       end
 
+      ##
+      # @api private
+      #
       # Given Hyrax::Upload object, return path to file on local filesystem
       def self.upload_path(upload)
         # so many layers to this onion:
+        # TODO: Write a recursive function to keep calling file until
+        # the file doesn't respond to file then return that file.
         upload.file.file.file
       end
 
-      # TODO: implement a method to count existing PDFs on a work to support
-      #       adding more PDFs to an existing work.
-      def self.count_existing_pdfs(_work)
-        0
-      end
-
+      ##
+      # @api private
+      #
       # TODO: Consider other methods to identify a PDF file.
       #       This sub-selection may need to be moved to use mimetype if there
       #       is a need to support paths not ending in .pdf (i.e. remote_urls)
       def self.pdfs_only_for(paths)
-        paths.select { |path| path.end_with?('.pdf', '.PDF') }
+        paths.select { |path| IiifPrint.split_for_path_suffix?(path) }
       end
     end
   end

data/lib/iiif_print/split_pdfs/derivative_rodeo_splitter.rb

@@ -0,0 +1,166 @@
+module IiifPrint
+  module SplitPdfs
+    ##
+    # This class wraps the DerivativeRodeo::Generators::PdfSplitGenerator to find preprocessed
+    # images, or split a PDF if there are no preprocessed images.
+    #
+    # We have already attached the original file to the file_set.  We want to convert that original
+    # file that's attached to a input_uri (e.g. "file://path/to/original-file" as in what we have
+    # written to Fedora as the PDF)
+    #
+    # @see .call
+    class DerivativeRodeoSplitter
+      ##
+      # @param filename [String] the local path to the PDFDerivativeServicele
+      # @param file_set [FileSet] file set containing the PDF file to split
+      #
+      # @return [Array<String>] paths to images split from each page of PDF file
+      #
+      # @see IiifPrint::SplitPdfs::BaseSplitter
+      def self.call(filename, file_set:)
+        new(filename, file_set: file_set).split_files
+      end
+
+      ##
+      # @param filename [String] path to the original file.  Note that we use {#filename} to
+      #        derivate {#input_uri}
+      # @param file_set [FileSet] the container for the original file and its derivatives.
+      #
+      # @param output_tmp_dir [String] where we will be writing things.  In using `Dir.mktmpdir`
+      #        we're creating a sudirectory on `Dir.tmpdir`
+      def initialize(filename, file_set:, output_tmp_dir: Dir.tmpdir)
+        @filename = filename
+        @file_set = file_set
+
+        @input_uri = "file://#{filename}"
+
+        # We are writing the images to a local location that CarrierWave can upload.  This is a
+        # local file, internal to IiifPrint; it looks like SpaceStone/DerivativeRodeo lingo, but
+        # that's just a convenience.
+        output_template_path = File.join(output_tmp_dir, '{{ dir_parts[-1..-1] }}', '{{ filename }}')
+
+        @output_location_template = "file://#{output_template_path}"
+      end
+
+      attr_reader :filename, :file_set
+
+      ##
+      # This is where, in "Fedora" we have the original file.  This is not the original file in the
+      # pre-processing location but instead the long-term location of the file in the application
+      # that mounts IIIF Print.
+      #
+      # @return [String]
+      attr_reader :input_uri
+
+      ##
+      # This is the location where we're going to write the derivatives that will "go into Fedora";
+      # it is a local location, one that IIIF Print's mounting application can directly do
+      # "File.read"
+      #
+      # @return [String]
+      attr_reader :output_location_template
+
+      ##
+      # Where can we find the file that represents the pre-processing template.  In this case, the
+      # original PDF file.
+      #
+      # The logic handles a case where SpaceStone successfully fetched the file to then perform
+      # processing.
+      #
+      # For example, SpaceStone::Serverless will pre-process derivatives and write them into an S3
+      # bucket that we then use for IIIF Print.
+      #
+      # @note The preprocessed_location_template should end in `.pdf`.  The
+      #       DerivativeRodeo::BaseGenerator::PdfSplitGenerator#derive_preprocessed_template_from
+      #       will coerce the template into one that represents the split pages.
+      #
+      # @return [String]
+      #
+      # @see https://github.com/scientist-softserv/space_stone-serverless/blob/7f46dd5b218381739cd1c771183f95408a4e0752/awslambda/handler.rb#L58-L63
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
+      def preprocessed_location_template
+        return @preprocessed_location_template if defined?(@preprocessed_location_template)
+
+        derivative_rodeo_candidate = IiifPrint::DerivativeRodeoService.derivative_rodeo_uri(file_set: file_set, filename: filename)
+
+        @preprocessed_location_template =
+          if derivative_rodeo_candidate.blank?
+            message = "#{self.class}##{__method__} could not establish derivative_rodeo_candidate for " \
+                      "#{file_set.class} ID=#{file_set&.id} #to_param=#{file_set&.to_param} with filename #{filename.inspect}.  " \
+                      "Move along little buddy."
+            Rails.logger.debug(message)
+            nil
+          elsif rodeo_conformant_uri_exists?(derivative_rodeo_candidate)
+            Rails.logger.debug("#{self.class}##{__method__} found existing file at location #{derivative_rodeo_candidate}.  High five partner!")
+            derivative_rodeo_candidate
+          elsif file_set.import_url
+            message = "#{self.class}##{__method__} did not find #{derivative_rodeo_candidate.inspect} to exist.  " \
+                      "Moving on to check the #{file_set.class}#import_url of #{file_set.import_url.inspect}"
+            Rails.logger.warn(message)
+            handle_original_file_not_in_derivative_rodeo
+          else
+            message = "#{self.class}##{__method__} could not find an existing file at #{derivative_rodeo_candidate} " \
+                      "nor a remote_url for #{file_set.class} ID=#{file_set.id} #to_param=#{file_set&.to_param}.  " \
+                      "Returning `nil' as we have no possible preprocess.  " \
+                      "Maybe the input_uri #{input_uri.inspect} will be adequate."
+            Rails.logger.warn(message)
+            nil
+          end
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+
+      ##
+      # @api private
+      #
+      # When the file does not exist in the pre-processed location (e.g. "SpaceStone") we need to
+      # ensure that we have something locally.  We copy the {FileSet#import_url} to the {#input_uri}
+      # location.
+      #
+      # @return [String] should be the {#input_uri}
+      # @raise [DerivativeRodeo::Errors::FileMissingError] when the input_uri does not exist
+      def handle_original_file_not_in_derivative_rodeo
+        # A quick short-circuit.  Don't attempt to copy.  Likely already covered by the DerivativeRodeo::Generators::CopyGenerator
+        return input_uri if rodeo_conformant_uri_exists?(input_uri)
+
+        message = "#{self.class}##{__method__} found #{file_set.class}#import_url of #{file_set.import_url.inspect} to exist.  " \
+                  "Perhaps there was a problem in SpaceStone downloading the file?  " \
+                  "Regardless, we'll use DerivativeRodeo::Generators::CopyGenerator to ensure #{input_uri.inspect} exists.  " \
+                  "However, we'll almost certainly be generating child pages locally."
+        Rails.logger.info(message)
+
+        # This ensures that we have a copy of the file_set.import_uri at the input_uri location;
+        # we likely have this.
+        DerivativeRodeo::Generators::CopyGenerator.new(
+          input_uris: [file_set.import_url],
+          output_location_template: input_uri
+        ).generated_uris.first
+      end
+      # private :handle_original_file_not_in_derivative_rodeo
+
+      def rodeo_conformant_uri_exists?(uri)
+        DerivativeRodeo::StorageLocations::BaseLocation.from_uri(uri).exist?
+      end
+      private :rodeo_conformant_uri_exists?
+
+      ##
+      # @return [Array<Strings>] the paths to each of the images split off from the PDF.
+      def split_files
+        DerivativeRodeo::Generators::PdfSplitGenerator.new(
+          input_uris: [input_uri],
+          output_location_template: output_location_template,
+          preprocessed_location_template: preprocessed_location_template
+        ).generated_files.map(&:file_path)
+      rescue => e
+        message = "#{self.class}##{__method__} encountered `#{e.class}' “#{e}” for " \
+                  "input_uri: #{input_uri.inspect}, " \
+                  "output_location_template: #{output_location_template.inspect}, and " \
+                  "preprocessed_location_template: #{preprocessed_location_template.inspect}."
+        exception = RuntimeError.new(message)
+        exception.set_backtrace(e.backtrace)
+        raise exception
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/destroy_pdf_child_works_service.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module IiifPrint
+  module SplitPdfs
+    ## Encapsulates logic for cleanup when the PDF is destroyed after pdf splitting into child works
+    class DestroyPdfChildWorksService
+      ## @api public
+      # @param file_set [FileSet] What is the containing file set for the provided file.
+      # @param work [Hydra::PCDM::Work] Parent of the fileset being deleted
+      def self.conditionally_destroy_spawned_children_of(file_set:, work:, user: nil)
+        child_model = work.try(:iiif_print_config)&.pdf_split_child_model
+        return unless child_model
+        return unless IiifPrint.pdf?(file_set)
+
+        # NOTE: The IiifPrint::PendingRelationship is an ActiveRecord object; hence we don't need to
+        # leverage an adapter.
+        IiifPrint::PendingRelationship.where(parent_id: work.id, file_id: file_set.id).find_each(&:destroy)
+        IiifPrint.destroy_children_split_from(file_set: file_set, work: work, model: child_model, user: user)
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/pages_to_jpgs_splitter.rb

@@ -0,0 +1,19 @@
+module IiifPrint
+  module SplitPdfs
+    # @abstract
+    #
+    # The purpose of this class is to split the PDF into constituent jpg files.
+    #
+    # @see #each
+    class PagesToJpgsSplitter < BaseSplitter
+      self.image_extension = 'jpg'
+      self.quality = '50'
+
+      private
+
+      def gsdevice
+        'jpeg'
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/pages_to_pngs_splitter.rb

@@ -0,0 +1,26 @@
+module IiifPrint
+  module SplitPdfs
+    # @abstract
+    #
+    # The purpose of this class is to split the PDF into constituent png files.
+    #
+    # @see #each
+    class PagesToPngsSplitter < BaseSplitter
+      self.image_extension = 'png'
+
+      private
+
+      def gsdevice
+        color, _channels, bpc = pdfinfo.color
+        device = nil
+        # 1 Bit Grayscale, if applicable:
+        device = 'pngmonod' if color == 'gray' && bpc == 1
+        # 8 Bit Grayscale, if applicable:
+        device = 'pnggray' if color == 'gray' && bpc > 1
+        # otherwise 24 Bit RGB:
+        device = 'png16m' if device.nil?
+        device
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/pages_to_tiffs_splitter.rb

@@ -0,0 +1,41 @@
+module IiifPrint
+  module SplitPdfs
+    # The purpose of this class is to split the PDF into constituent TIFF files.
+    #
+    # @see #each
+    class PagesToTiffsSplitter < BaseSplitter
+      self.image_extension = 'tiff'
+      DEFAULT_COMPRESSION = 'lzw'.freeze
+      self.compression = DEFAULT_COMPRESSION
+
+      private
+
+      def gsdevice
+        color, channels, bpc = pdfinfo.color
+        device = nil
+        if color == 'gray'
+          # CCITT Group 4 Black and White, if applicable:
+          if bpc == 1
+            device = 'tiffg4'
+            self.compression = 'g4'
+          elsif bpc > 1
+            # 8 Bit Grayscale, if applicable:
+            device = 'tiffgray'
+          end
+        end
+
+        # otherwise color:
+        device = colordevice(channels, bpc) if device.nil?
+        device
+      end
+
+      def colordevice(channels, bpc)
+        bits = bpc * channels
+        # will be either 8bpc/16bpd color TIFF,
+        #   with any CMYK source transformed to 8bpc RBG
+        bits = 24 unless [24, 48].include? bits
+        "tiff#{bits}nc"
+      end
+    end
+  end
+end