iiif_print 1.0.0

data/lib/iiif_print/metadata.rb

@@ -0,0 +1,125 @@
+module IiifPrint
+  # rubocop:disable Metrics/ClassLength
+  class Metadata
+    def self.build_metadata_for(work:, version:, fields:, current_ability:, base_url:)
+      new(work: work,
+          version: version,
+          fields: fields,
+          current_ability: current_ability,
+          base_url: base_url).build_metadata
+    end
+
+    def initialize(work:, version:, fields:, current_ability:, base_url:)
+      @work = work
+      @version = version.to_i
+      @fields = fields
+      @current_ability = current_ability
+      @base_url = base_url
+    end
+
+    attr_reader :work, :version, :fields
+
+    def build_metadata
+      send("build_metadata_for_v#{version}")
+    end
+
+    private
+
+    def build_metadata_for_v2
+      fields.map do |field|
+        label = Hyrax::Renderers::AttributeRenderer.new(field.name, nil).label
+        if field.name == :collection && member_of_collection?
+          viewable_collections = Hyrax::CollectionMemberService.run(work, @current_ability)
+          next if viewable_collections.empty?
+          { 'label' => label,
+            'value' => make_collection_link(viewable_collections) }
+        else
+          next if field_is_empty?(field)
+          { 'label' => label,
+            'value' => cast_to_value(field_name: field.name, options: field.options) }
+        end
+      end.compact
+    end
+
+    def build_metadata_for_v3
+      fields.map do |field|
+        values = Array(work.try(field.name)).map { |value| scrub(value.to_s) }
+        next if values.empty?
+        {
+          'label' => {
+            # Since we're using I18n to translate the field, we're setting the locale used in the translation.
+            I18n.locale.to_s => [Hyrax::Renderers::AttributeRenderer.new(field.name, nil).label]
+          },
+          'value' => {
+            'none' => values
+          }
+        }
+      end.compact
+    end
+
+    def field_is_empty?(field)
+      Array(work.try(field.name)).empty?
+    end
+
+    def member_of_collection?
+      work[:member_of_collection_ids_ssim]&.present?
+    end
+
+    def scrub(value)
+      Loofah.fragment(value).scrub!(:whitewash).to_s
+    end
+
+    def cast_to_value(field_name:, options:)
+      if options&.[](:render_as) == :faceted
+        values_for(field_name: field_name).map do |value|
+          search_field = field_name.to_s + "_sim"
+          path = Rails.application.routes.url_helpers.search_catalog_path(
+            "f[#{search_field}][]": value, locale: I18n.locale
+          )
+          path += '&include_child_works=true' if work["is_child_bsi"] == true
+          "<a href='#{File.join(@base_url, path)}'>#{value}</a>"
+        end
+      else
+        make_link(values_for(field_name: field_name))
+      end
+    end
+
+    def values_for(field_name:)
+      Array(work.send(field_name))
+    end
+
+    def make_collection_link(collection_documents)
+      collection_documents.map do |collection|
+        "<a href='#{File.join(@base_url, 'collections', collection.id)}'>#{collection.title.first}</a>"
+      end
+    end
+
+    # @note This method turns link looking strings into links
+    def make_link(texts)
+      texts.map do |t|
+        t.to_s.gsub(MAKE_LINK_REGEX) do |url|
+          "<a href='#{url}' target='_blank'>#{url}</a>"
+        end
+      end
+    end
+
+    MAKE_LINK_REGEX = %r{
+      \b
+      (
+        (?: [a-z][\w-]+:
+          (?: /{1,3} | [a-z0-9%] ) |
+            www\d{0,3}[.] |
+            [a-z0-9.\-]+[.][a-z]{2,4}/
+        )
+        (?:
+          [^\s()<>]+ | \(([^\s()<>]+|(\([^\s()<>]+\)))*\)
+        )+
+        (?:
+          \(([^\s()<>]+|(\([^\s()<>]+\)))*\) |
+          [^\s`!()\[\]{};:'".,<>?«»〝〞‘‛]
+        )
+      )
+    }ix.freeze
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/iiif_print/pdf_derivative_service.rb

@@ -0,0 +1,42 @@
+require 'open3'
+
+module IiifPrint
+  class PDFDerivativeService < BaseDerivativeService
+    self.target_extension = 'pdf'.freeze
+
+    # PDF (JPEG, 8 bit grayscale), 150ppi
+    GRAY_PDF_CMD = 'convert %<source_file>s ' \
+                   '-resize 1800 -density 150 ' \
+                   '-depth 8 -colorspace Gray ' \
+                   '-compress jpeg %<out_file>s'.freeze
+
+    # sRBG color PDF (JPEG, 8 bits per channel), 150ppi
+    COLOR_PDF_CMD = 'convert %<source_file>s ' \
+                    '-resize 1800 -density 150 ' \
+                    '-depth 8 ' \
+                    '-compress jpeg %<out_file>s'.freeze
+
+    def initialize(file_set)
+      super(file_set)
+    end
+
+    # Get conversion command; command varies on whether or not we have
+    #   JP2 source, and whether we have color or grayscale material.
+    def convert_cmd
+      template = use_color? ? COLOR_PDF_CMD : GRAY_PDF_CMD
+      format(template, source_file: @source_path, out_file: @dest_path)
+    end
+
+    def create_derivatives(filename)
+      # Base class takes care of loading @source_path, @dest_path
+      super(filename)
+
+      # no creation if pdf master
+      return if mime_type == 'application/pdf'
+
+      # Get and run conversion command
+      return jp2_convert if mime_type == 'image/jp2'
+      im_convert
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/child_work_creation_from_pdf_service.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+# Encapsulates methods used for pdf splitting into child works
+module IiifPrint
+  module SplitPdfs
+    class ChildWorkCreationFromPdfService
+      # Load an array of paths to pdf files
+      # @param [Array > Hyrax::Upload file ids]
+      # @return [Array > String] file paths to temp directory
+      def self.pdf_paths(files:)
+        upload_ids = filter_file_ids(files)
+        return [] if upload_ids.empty?
+        uploads = Hyrax::UploadedFile.find(upload_ids)
+        paths = uploads.map(&method(:upload_path))
+        pdfs_only_for(paths)
+      end
+
+      # Is child work splitting defined for model?
+      # @param [GenericWork, etc] A valid type of hyrax work
+      # @return [Boolean]
+      def self.iiif_print_split?(work:)
+        # defined only if work has include IiifPrint.model_configuration with pdf_split_child_model
+        return true if work.try(:iiif_print_config)&.pdf_split_child_model
+        false
+      end
+
+      # Are there any PDF files?
+      # @param [Array > String] paths to PDFs
+      # @return [Boolean]
+      def self.pdfs?(paths:)
+        pdf_paths = pdfs_only_for(paths)
+        return false unless pdf_paths.count.positive?
+        true
+      end
+
+      # Submit the job to split PDF into child works
+      # @param [GenericWork, etc] A valid type of hyrax work
+      # @param [Array<String>] paths to PDF attachments
+      # @param [User] user
+      # @param [Integer] number of pdfs already on existing work's filesets (not yet implemented)
+      def self.queue_job(work:, file_locations:, user:, admin_set_id:)
+        work.iiif_print_config.pdf_splitter_job.perform_later(
+          work,
+          file_locations,
+          user,
+          admin_set_id,
+          count_existing_pdfs(work)
+        )
+      end
+
+      def self.filter_file_ids(input)
+        Array.wrap(input).select(&:present?)
+      end
+
+      # Given Hyrax::Upload object, return path to file on local filesystem
+      def self.upload_path(upload)
+        # so many layers to this onion:
+        upload.file.file.file
+      end
+
+      # TODO: implement a method to count existing PDFs on a work to support
+      #       adding more PDFs to an existing work.
+      def self.count_existing_pdfs(_work)
+        0
+      end
+
+      # TODO: Consider other methods to identify a PDF file.
+      #       This sub-selection may need to be moved to use mimetype if there
+      #       is a need to support paths not ending in .pdf (i.e. remote_urls)
+      def self.pdfs_only_for(paths)
+        paths.select { |path| path.end_with?('.pdf', '.PDF') }
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/pages_into_images_service.rb

@@ -0,0 +1,130 @@
+require 'open3'
+require 'securerandom'
+require 'tmpdir'
+require 'iiif_print/split_pdfs/pdf_image_extraction_service'
+
+module IiifPrint
+  module SplitPdfs
+    class PagesIntoImagesService
+      include Enumerable
+
+      def initialize(path)
+        @baseid = SecureRandom.uuid
+        @pdfpath = path
+        @info = nil
+        @entries = nil
+        @tmpdir = nil
+        @size = nil
+        @pagecount = nil
+        @pdftext = nil
+        @compression = 'lzw'
+      end
+
+      # return
+      def pdfinfo
+        @info = IiifPrint::SplitPdfs::PdfImageExtractionService.new(@pdfpath) if @info.nil?
+        @info
+      end
+
+      # TODO: put this test somewhere to prevent invalid pdfs from crashing the image service.
+      def invalid_pdf?
+        return true if pdfinfo.color.include?(nil) || pdfinfo.width.nil? || pdfinfo.height.nil? || pdfinfo.entries.length.zero?
+        false
+      end
+
+      def tmpdir
+        @tmpdir = Dir.mktmpdir if @tmpdir.nil?
+        @tmpdir
+      end
+
+      def colordevice(channels, bpc)
+        bits = bpc * channels
+        # will be either 8bpc/16bpd color TIFF,
+        #   with any CMYK source transformed to 8bpc RBG
+        bits = 24 unless [24, 48].include? bits
+        "tiff#{bits}nc"
+      end
+
+      def gsdevice
+        color, channels, bpc = pdfinfo.color
+        device = nil
+        # CCITT Group 4 Black and White, if applicable:
+        if color == 'gray' && bpc == 1
+          device = 'tiffg4'
+          @compression = 'g4'
+        end
+        # 8 Bit Grayscale, if applicable:
+        device = 'tiffgray' if color == 'gray' && bpc > 1
+        # otherwise color:
+        device = colordevice(channels, bpc) if device.nil?
+        device
+      end
+
+      # TODO: this method came from newspaper gem but appears to be unused. Is it needed anywhere?
+      # def gstext
+      #   cmd = "gs -q -dNOPAUSE -dBATCH -sDEVICE=txtwrite " \
+      #         "-sOutputFile=- -f #{@pdfpath}"
+      #   Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+      #     @pdftext = stdout.read
+      #   end
+      #   @pdftext
+      # end
+
+      def pagecount
+        cmd = "pdfinfo #{@pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          output = stdout.read.split("\n")
+          # rubocop:disable Performance/Detect
+          pages_e = output.select { |e| e.start_with?('Pages:') }[0]
+          # rubocop:enable Performance/Detect
+          @pagecount = pages_e.split[-1].to_i
+        end
+        @pagecount
+      end
+
+      def looks_scanned
+        max_image_px = pdfinfo.width * pdfinfo.height
+        single_image_per_page = pdfinfo.entries.length == pagecount
+        # single 10mp+ image per page?
+        single_image_per_page && max_image_px > 1024 * 1024 * 10
+      end
+
+      def ppi
+        unless looks_scanned
+          # 400 dpi for something that does not look like scanned media:
+          return 400
+        end
+        # For scanned media, defer to detected image PPI:
+        pdfinfo.ppi
+      end
+
+      # ghostscript convert all pages to TIFF
+      def gsconvert
+        output_base = File.join(tmpdir, "#{@baseid}-page%d.tiff")
+        cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} " \
+              "-dTextAlphaBits=4 -sCompression=#{@compression} " \
+              "-sOutputFile=#{output_base} -r#{ppi} -f #{@pdfpath}"
+        Open3.popen3(cmd) do |_stdin, stdout, _stderr, _wait_thr|
+          output = stdout.read.split("\n")
+          # rubocop:disable Performance/Count
+          @size = output.select { |e| e.start_with?('Page ') }.length
+          # rubocop:enable Performance/Count
+        end
+        # Return an array of expected filenames
+        (1..@size).map { |n| File.join(tmpdir, "#{@baseid}-page#{n}.tiff") }
+      end
+
+      # entries for each page
+      def entries
+        @entries = gsconvert if @entries.nil?
+        @entries
+      end
+
+      def each
+        entries.each do |e|
+          yield(e)
+        end
+      end
+    end
+  end
+end

data/lib/iiif_print/split_pdfs/pdf_image_extraction_service.rb

@@ -0,0 +1,85 @@
+require 'open3'
+require 'mini_magick'
+
+module IiifPrint
+  module SplitPdfs
+    # Uses poppler 0.19+ pdfimages command to extract image
+    #   listing metadata from PDF files.
+    # For dpi extraction, falls back to calculating using MiniMagick,
+    #   if neccessary.
+    class PdfImageExtractionService
+      # class constant column numbers
+      COL_WIDTH = 3
+      COL_HEIGHT = 4
+      COL_COLOR = 5
+      COL_CHANNELS = 6
+      COL_BITS = 7
+      # only poppler 0.25+ has this column in output:
+      COL_XPPI = 12
+
+      def initialize(path)
+        @path = path
+        @cmd = format('pdfimages -list %<path>s', path: path)
+        @output = nil
+        @entries = nil
+      end
+
+      def process
+        # call just once
+        if @output.nil?
+          Open3.popen3(@cmd) do |_stdin, stdout, _stderr, _wait_thr|
+            @output = stdout.read.split("\n")
+          end
+        end
+        @output.slice(2, @output.size - 1)
+      end
+
+      def entries
+        if @entries.nil?
+          @entries = []
+          output = process
+          (0..output.size - 1).each do |i|
+            @entries.push(output[i].gsub(/\s+/m, ' ').strip.split(" "))
+          end
+        end
+        @entries
+      end
+
+      def selectcolumn(i, &block)
+        result = entries.map { |e| e[i] }
+        return result.map!(&block) if block_given?
+        result
+      end
+
+      def width
+        selectcolumn(COL_WIDTH, &:to_i).max
+      end
+
+      def height
+        selectcolumn(COL_HEIGHT, &:to_i).max
+      end
+
+      def color
+        # desc is either 'gray', 'cmyk', 'rgb', but 1-bit gray is black/white
+        #   so caller may want all of this information, and in case of
+        #   mixed color spaces across images, this returns maximum
+        desc = entries.any? { |e| e[COL_COLOR] != 'gray' } ? 'rgb' : 'gray'
+        channels = entries.map { |e| e[COL_CHANNELS].to_i }.max
+        bits = entries.map { |e| e[COL_BITS].to_i }.max
+        [desc, channels, bits]
+      end
+
+      def ppi
+        if entries[0].size <= 12
+          # poppler < 0.25
+          pdf = MiniMagick::Image.open(@path)
+          width_points = pdf.width
+          width_px = width
+          return (72 * width_px / width_points).to_i
+        end
+        # with poppler 0.25+, pdfimages just gives us this:
+        selectcolumn(COL_XPPI, &:to_i).max
+      end
+    end
+  end
+end

data/lib/iiif_print/text_extraction/alto_reader.rb

@@ -0,0 +1,123 @@
+require 'active_support/core_ext/module/delegation'
+require 'json'
+require 'nokogiri'
+
+module IiifPrint
+  # Module for text extraction
+  module TextExtraction
+    # Class to obtain plain text and JSON word-coordinates from ALTO source
+    class AltoReader
+      attr_accessor :source, :doc_stream
+      delegate :text, to: :doc_stream
+
+      # SAX Document Stream class to gather text and word tokens from ALTO
+      class AltoDocStream < Nokogiri::XML::SAX::Document
+        attr_accessor :text, :words
+
+        def initialize(image_width = nil)
+          super()
+          # scaling matters:
+          @image_width = image_width
+          @scaling = 1.0 # pt to px, if ALTO using points
+          # plain text buffer:
+          @text = ''
+          # list of word hash, containing word+coord:
+          @words = []
+        end
+
+        # Return coordinates from String element attribute hash
+        #
+        # @param attrs [Hash] hash containing ALTO `String` element attributes.
+        # @return [Array] Array of position x, y, width, height in px.
+        def s_coords(attrs)
+          height = scale_value((attrs['HEIGHT'] || 0).to_i)
+          width = scale_value((attrs['WIDTH'] || 0).to_i)
+          hpos = scale_value((attrs['HPOS'] || 0).to_i)
+          vpos = scale_value((attrs['VPOS'] || 0).to_i)
+          [hpos, vpos, width, height]
+        end
+
+        def compute_scaling(attrs)
+          return if @image_width.nil?
+          match = attrs.find { |e| e[0].casecmp?('WIDTH') }
+          return if match.empty?
+          page_width = match[1].to_i
+          return if @image_width == page_width
+          @scaling = page_width / @image_width.to_f
+        end
+
+        def scale_value(v)
+          (v / @scaling).to_i
+        end
+
+        # Callback for element start, implementation of which ignores
+        #   non-String elements.
+        #
+        # @param name [String] element name.
+        # @param attrs [Array] Array of key, value pair Arrays.
+        def start_element(name, attrs = [])
+          values = attrs.to_h
+          compute_scaling(attrs) if name == 'Page'
+          return if name != 'String'
+          token = values['CONTENT']
+          @text << token
+          @words << {
+            word: token,
+            coordinates: s_coords(values)
+          }
+        end
+
+        # Callback for element end, used here to manage endings of lines and
+        #   blocks.
+        #
+        # @param name [String] element name.
+        def end_element(name)
+          @text << " " if name == 'String'
+          @text << "\n" if name == 'TextBlock'
+          @text << "\n" if name == 'TextLine'
+        end
+
+        # Callback for completion of parsing ALTO, used to normalize generated
+        #   text content (strip unneeded whitespace incidental to output).
+        def end_document
+          # postprocess @text to remove trailing spaces on lines
+          @text = @text.split("\n").map(&:strip).join("\n")
+          # remove trailing whitespace at end of buffer
+          @text.strip!
+        end
+      end
+
+      # Construct with either path
+      #
+      # @param xml [String], and process document
+      def initialize(xml, image_width = nil, image_height = nil)
+        @source = isxml?(xml) ? xml : File.read(xml)
+        @image_width = image_width
+        @image_height = image_height
+        @doc_stream = AltoDocStream.new(image_width)
+        parser = Nokogiri::XML::SAX::Parser.new(doc_stream)
+        parser.parse(@source)
+      end
+
+      # Determine if source parameter is path or xml
+      #
+      # @param xml [String] either path to xml file or xml source
+      # @return [true, false] true if string appears to be XML source, not path
+      def isxml?(xml)
+        xml.lstrip.start_with?('<')
+      end
+
+      # Output JSON flattened word coordinates
+      #
+      # @return [String] JSON serialization of flattened word coordinates
+      def json
+        words = @doc_stream.words
+        IiifPrint::TextExtraction::WordCoordsBuilder.json_coordinates_for(
+          words: words,
+          width: @image_width,
+          height: @image_height
+        )
+      end
+    end
+  end
+end