derivative-rodeo 0.2.0

data/lib/derivative_rodeo/services/extract_word_coordinates_from_hocr_sgml_service.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'json'
+require 'nokogiri'
+
+module DerivativeRodeo
+  module Services
+    ##
+    # Responsible for converting an SGML string into JSON coordinates
+    class ExtractWordCoordinatesFromHocrSgmlService
+      ##
+      # @param sgml [String] The SGML (e.g. XML or HTML) text of a HOCR file.
+      # @return [String] A JSON document
+      def self.call(sgml)
+        new(sgml).to_json
+      end
+
+      ##
+      # Construct with either path or HTML [String]
+      #
+      # @param html [String] either an XML string or a path to a file.
+      def initialize(html)
+        @source = xml?(html) ? html : File.read(html)
+        @doc_stream = DocStream.new
+        parser = Nokogiri::HTML::SAX::Parser.new(@doc_stream)
+        parser.parse(@source)
+      end
+      attr_reader :doc_stream, :source
+
+      delegate :text, :width, :height, :words, to: :doc_stream
+
+      # Output JSON flattened word coordinates
+      #
+      # @return [String] JSON serialization of flattened word coordinates
+      def to_json
+        @to_json ||= WordCoordinates.to_json(
+          words: doc_stream.words,
+          width: doc_stream.width,
+          height: doc_stream.height
+        )
+      end
+      alias json to_json
+
+      private
+
+      def xml?(xml)
+        xml.lstrip.start_with?('<')
+      end
+
+      # SAX Document Stream class to gather text and word tokens from hOCR
+      class DocStream < Nokogiri::XML::SAX::Document
+        attr_accessor :text, :words, :width, :height
+
+        def initialize
+          super()
+          # plain text buffer:
+          @text = ''
+          # list of word hash, containing word+coord:
+          @words = []
+          # page width and height to be found in hOCR for `div.ocr_page`
+          @width = nil
+          @height = nil
+          # to hold current word data state across #start_element, #characters,
+          #   and #end_element methods (to associate word with coordinates).
+          @current = nil
+          # to preserve element classname from start to use by #end_element
+          @element_class_name = nil
+        end
+
+        # Return coordinates from `span.ocrx_word` element attribute hash
+        #
+        # @param attrs [Hash] hash with hOCR `span.ocrx_word` element attributes
+        # @return [Array] Array of position x, y, width, height in px.
+        def s_coords(attrs)
+          element_title = attrs['title']
+          bbox = element_title.split(';')[0].split('bbox ')[-1]
+          x1, y1, x2, y2 = bbox.split(' ').map(&:to_i)
+          height = y2 - y1
+          width = x2 - x1
+          hpos = x1
+          vpos = y1
+          [hpos, vpos, width, height]
+        end
+
+        # Consider element for processing?
+        #   - `div.ocr_page` — to get page width/height
+        #   - `span.ocr_line` — to help make plain text readable
+        #   - `span.ocrx_word` — for word-coordinate JSON and plain text word
+        # @param name [String] Element name
+        # @param class_name [String] HTML class name
+        # @return [Boolean] true if element should be processed; otherwise false
+        def consider?(name, class_name)
+          selector = "#{name}.#{class_name}"
+          ['div.ocr_page', 'span.ocr_line', 'span.ocrx_word'].include?(selector)
+        end
+
+        def start_word(attrs)
+          @current = {}
+          # will be replaced during #characters method call:
+          @current[:word] = nil
+          @current[:coordinates] = s_coords(attrs)
+        end
+
+        def start_page(attrs)
+          title = attrs['title']
+          fields = title.split(';')
+          bbox = fields[1].split('bbox ')[-1].split(' ').map(&:to_i)
+          # width and height:
+          @width = bbox[2]
+          @height = bbox[3]
+        end
+
+        def word_complete?
+          return false if @current.nil?
+          coords = @current[:coordinates]
+          @current[:word].present? && coords.size == 4
+        end
+
+        def end_word
+          # add trailing space to plaintext buffer for between words:
+          @text += ' '
+          @words.push(@current) if word_complete?
+        end
+
+        def end_line
+          # strip trailing whitespace
+          @text.strip!
+          # then insert a line break
+          @text += "\n"
+        end
+
+        # Callback for element start, ignores elements except for:
+        #   - `div.ocr_page` — to get page width/height
+        #   - `span.ocr_line` — to help make plain text readable
+        #   - `span.ocrx_word` — for word-coordinate JSON and plain text word
+        #
+        # @param name [String] element name.
+        # @param attrs [Array] Array of key, value pair Arrays.
+        def start_element(name, attrs = [])
+          attributes = attrs.to_h
+          @element_class_name = attributes['class']
+          return unless consider?(name, @element_class_name)
+          start_word(attributes) if @element_class_name == 'ocrx_word'
+          start_page(attributes) if @element_class_name == 'ocr_page'
+        end
+
+        def characters(value)
+          return if @current.nil?
+          return if @current[:coordinates].nil?
+          @current[:word] ||= ''
+          @current[:word] += value
+          @text += value
+        end
+
+        # Callback for element end; at this time, flush word coordinate state
+        #   for current word, and append line endings to plain text:
+        #
+        # @param _name [String] element name.
+        def end_element(_name)
+          end_line if @element_class_name == 'ocr_line'
+          end_word if @element_class_name == 'ocrx_word'
+        end
+
+        # Callback for completion of parsing hOCR, used to normalize generated
+        #   text content (strip unneeded whitespace incidental to output).
+        def end_document
+          # postprocess @text to remove trailing spaces on lines
+          @text = @text.split("\n").map(&:strip).join("\n")
+          # remove excess line break
+          @text.gsub!(/\n+/, "\n")
+          @text.delete("\r")
+          # remove trailing whitespace at end of buffer
+          @text.strip!
+        end
+      end
+
+      class WordCoordinates
+        ##
+        # @api public
+        #
+        # @param words [Array<Hash>] an array of hash objects that have the keys `:word` and `:coordinates`.
+        # @param width [Integer] the width of the "canvas" on which the words appear.
+        # @param height [Integer] the height of the "canvas" on which the words appear.
+        #
+        # @return [String] a JSON encoded string.
+        def self.to_json(words:, width: nil, height: nil)
+          new(words: words, width: width, height: height).to_json
+        end
+
+        def initialize(words:, width:, height:)
+          @words = words
+          @width = width
+          @height = height
+        end
+        attr_reader :words, :width, :height
+
+        # Output JSON flattened word coordinates
+        #
+        # @return [String] JSON serialization of flattened word coordinates
+        def to_json
+          coordinates = {}
+          words.each do |word|
+            word_chars = word[:word]
+            word_coords = word[:coordinates]
+            if coordinates[word_chars]
+              coordinates[word_chars] << word_coords
+            else
+              coordinates[word_chars] = [word_coords]
+            end
+          end
+          payload = { width: width, height: height, coords: coordinates }
+          JSON.generate(payload)
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/image_identify_service.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module Services
+    ##
+    # This module is responsible for extracting technical_metadata for a given path.
+    #
+    # @see .technical_metadata_for
+    class ImageIdentifyService < BaseService
+      class_attribute :identify_format_option,
+                      default: %(Geometry: %G\nDepth: %[bit-depth]\nColorspace: %[colorspace]\nAlpha: %A\nMIME Step: %m\n) # rubocop:disable Layout/LineLength
+
+      ##
+      # @api public
+      # @param path [String]
+      # @return [Derivative::Rodeo::TechnicalMetadata]
+      def self.technical_metadata_for(path:)
+        new(path).technical_metadata
+      end
+
+      def initialize(path)
+        super()
+        @path = path
+        # The first 23 characters of a file contains the magic.
+        @initial_file_contents = File.read(@path, 23, 0)
+      end
+      attr_reader :path
+
+      # Return metadata by means of imagemagick identify
+      def technical_metadata
+        technical_metadata = TechnicalMetadata.new
+        lines = im_identify
+        width, height = im_identify_geometry(lines)
+        technical_metadata.width = width
+        technical_metadata.height = height
+        technical_metadata.content_type = im_mime(lines)
+        populate_im_color!(lines, technical_metadata)
+        technical_metadata
+      end
+
+      private
+
+      # @return [Array<String>] lines of output from imagemagick `identify`
+      def im_identify
+        return @im_identify if defined?(@im_identify)
+
+        # Instead of relying on all of the properties, we're requesting on the specific properties
+        cmd = "identify -format '#{identify_format_option}' #{path}"
+        # cmd = "identify -verbose #{path}"
+        @im_identify = `#{cmd}`.lines
+      end
+
+      # @return [Array(Integer, Integer)] width, height in Integer px units
+      def im_identify_geometry(lines)
+        img_geo = im_line_select(lines, 'geometry').split('+')[0]
+        img_geo.split('x').map(&:to_i)
+      end
+
+      def im_mime(lines)
+        return 'application/pdf' if pdf? # workaround older imagemagick bug
+
+        im_line_select(lines, 'mime step')
+      end
+
+      def pdf?
+        @initial_file_contents.start_with?('%PDF-')
+      end
+
+      def populate_im_color!(lines, technical_metadata)
+        bpc = im_line_select(lines, 'depth').split('-')[0].to_i # '1-bit' -> 1
+        colorspace = im_line_select(lines, 'colorspace')
+        color = colorspace == 'Gray' ? 'gray' : 'color'
+        has_alpha = !im_line_select(lines, 'alpha') == 'Undefined'
+        technical_metadata.num_components = (color == 'gray' ? 1 : 3) + (has_alpha ? 1 : 0)
+        technical_metadata.color = bpc == 1 ? 'monochrome' : color
+        technical_metadata.bits_per_component = bpc
+      end
+
+      def im_line_select(lines, key)
+        line = lines.find { |l| l.scrub.downcase.strip.start_with?(key.downcase) }
+        # Given "key: value" line, return the value as String stripped of
+        #   leading and trailing whitespace
+        return line if line.nil?
+
+        line.strip.split(':')[-1].strip
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/image_jp2_service.rb

@@ -0,0 +1,112 @@
+# rubocop:disable Style/FrozenStringLiteralComment
+# TODO freeze them literals
+
+module DerivativeRodeo
+  module Services
+    ##
+    # A utility class for extracting technical metadata from a JP2.
+    #
+    # @see .technical_metadata_for
+    class ImageJp2Service < BaseService
+      TOKEN_MARKER_START = "\xFF".force_encoding('BINARY')
+      TOKEN_MARKER_SIZ = "\x51".force_encoding('BINARY')
+      TOKEN_IHDR = 'ihdr'.freeze
+
+      ##
+      # @api public
+      #
+      # @param path [String] path to jp2, for reading
+      #
+      # @return [Derivative::Rodeo::TechnicalMetadata]
+      def self.technical_metadata_for(path:)
+        new(path).technical_metadata
+      end
+
+      attr_reader :path
+
+      def initialize(path)
+        super()
+        @path = path
+      end
+
+      # rubocop:disable Metrics/MethodLength
+      def technical_metadata
+        io = File.open(path, 'rb')
+        io.seek(0, IO::SEEK_SET)
+        validate_jp2(io)
+        x_siz, y_siz = extract_jp2_dim(io)
+        nc, bpc = extract_jp2_components(io)
+        color = nc >= 3 ? 'color' : 'gray'
+        TechnicalMetadata.new(
+          color: bpc == 1 ? 'monochrome' : color,
+          num_components: nc,
+          bits_per_component: bpc,
+          width: x_siz,
+          height: y_siz,
+          content_type: 'image/jp2'
+        )
+      ensure
+        io.close
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      private
+
+      # @param io [IO] IO stream opened in binary mode, for reading
+      # @return [Array(Integer, Integer)] X size, Y size, in Integer-stepd px
+      # rubocop:disable Metrics/MethodLength
+      def extract_jp2_dim(io)
+        raise IOError, 'file not open in binary mode' unless io.binmode?
+
+        buffer = ''
+        siz_found = false
+        # Informed by ISO/IEC 15444-1:2000, pp. 26-27
+        #   via:
+        #   http://hosting.astro.cornell.edu/~carcich/LRO/jp2/ISO_JPEG200_Standard/INCITS+ISO+IEC+15444-1-2000.pdf
+        #
+        # first 23 bytes are file-magic, we can skip
+        io.seek(23, IO::SEEK_SET)
+        while !siz_found && !buffer.nil?
+          # read one byte at a time, until we hit marker start 0xFF
+          buffer = io.read(1) while buffer != TOKEN_MARKER_START
+          # - on 0xFF read subsequent byte; if value != 0x51, continue
+          buffer = io.read(1)
+          next if buffer != TOKEN_MARKER_SIZ
+
+          # - on 0x51, read next 12 bytes
+          buffer = io.read(12)
+          siz_found = true
+        end
+        # discard first 4 bytes; next 4 bytes are XSiz; last 4 bytes are YSiz
+        x_siz = buffer.byteslice(4, 4).unpack1('N')
+        y_siz = buffer.byteslice(8, 4).unpack1('N')
+        [x_siz, y_siz]
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      # @param io [IO] IO stream opened in binary mode, for reading
+      # @return [Array(Integer, Integer)] number components, bits-per-component
+      def extract_jp2_components(io)
+        raise IOError, 'file not open in binary mode' unless io.binmode?
+
+        io.seek(0, IO::SEEK_SET)
+        # IHDR should be in first 64 bytes
+        buffer = io.read(64)
+        ihdr_data = buffer.split(TOKEN_IHDR)[-1]
+        raise IOError if ihdr_data.nil?
+
+        num_components = ihdr_data.byteslice(8, 2).unpack1('n')
+        # stored as "bit depth of the components in the codestream, minus 1", so add 1
+        bits_per_component = ihdr_data.byteslice(10, 1).unpack1('c') + 1
+        [num_components, bits_per_component]
+      end
+
+      def validate_jp2(io)
+        # verify file is jp2
+        magic = io.read(23)
+        raise IOError, 'Not JP2 file' unless magic.end_with?('ftypjp2')
+      end
+    end
+  end
+end
+# rubocop:enable Style/FrozenStringLiteralComment

data/lib/derivative_rodeo/services/image_service.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
+module DerivativeRodeo
+  module Services
+    ##
+    # @api private
+    #
+    # @see .technical_metadata
+    # @see .convert
+    class ImageService < BaseService
+      attr_accessor :path
+
+      def initialize(path)
+        super()
+        @path = path
+        # The first 23 characters of a file contains the magic.
+        @initial_file_contents = File.read(@path, 23, 0)
+      end
+
+      def jp2?
+        @initial_file_contents.end_with?('ftypjp2')
+      end
+
+      # @return [Derivative::Rodeo::TechnicalMetadata]
+      def technical_metadata
+        return @technical_metadata if defined?(@technical_metadata)
+
+        @technical_metadata = if jp2?
+                                ImageJp2Service.technical_metadata_for(path: path)
+                              else
+                                ImageIdentifyService.technical_metadata_for(path: path)
+                              end
+      end
+      alias metadata technical_metadata
+
+      extend Forwardable
+      def_delegator :technical_metadata, :monochrome?
+
+      # Convert source image to image at destination path, inferring file type from destination
+      # file extension.  In case of JP2 files, create intermediate file using OpenJPEG 2000 that
+      # ImageMagick can use.  Only outputs monochrome output if monochrome is true, destination
+      # format is TIFF.
+      #
+      # @param destination [String] Path to output / destination file
+      # @param monochrome [Boolean] true if monochrome output, otherwise false
+      def convert(destination:, monochrome: false)
+        raise 'JP2 output not yet supported' if destination.end_with?('jp2')
+
+        source = jp2? ? jp2_to_tiff(path) : path
+        convert_image(source: source, destination: destination, monochrome: monochrome)
+      end
+
+      private
+
+      def convert_image(source:, destination:, monochrome:)
+        monochrome &&= destination.slice(-4, 4).index('tif')
+        mono_opts = '-depth 1 -monochrome -compress Group4 -type bilevel '
+        opts = monochrome ? mono_opts : ''
+        cmd = "convert #{source} #{opts}#{destination}"
+        `#{cmd}`
+      end
+
+      def jp2_to_tiff(source)
+        intermediate_path = File.join(Dir.mktmpdir, 'intermediate.tif')
+        jp2_cmd = "opj_decompress -i #{source} -o #{intermediate_path}"
+        `#{jp2_cmd}`
+        intermediate_path
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter/base.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'securerandom'
+require 'tmpdir'
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitter
+      ##
+      # @param name [String]
+      # @return [PdfSplitter::Base]
+      def self.for(name)
+        klass_name = "#{name.to_s.classify}_page".classify
+        "DerivativeRodeo::Services::PdfSplitter::#{klass_name}".constantize
+      end
+
+      ##
+      # @abstract
+      #
+      # The purpose of this class is to split the PDF into constituent image files.
+      #
+      # @see #each
+      class Base
+        class_attribute :image_extension
+        class_attribute :default_dpi, default: 400
+        # Should we perform compression logic on the images?
+        class_attribute :compression, default: nil
+        # What is the image quality we're using?
+        class_attribute :quality, default: nil
+
+        class_attribute :gsdevice, instance_accessor: false
+        class_attribute :page_count_regexp, instance_accessor: true, default: /^Pages: +(\d+)$/
+        ##
+        # @api public
+        #
+        # @param path [String] The path the the PDF
+        #
+        # @return [Enumerable, Utilities::PdfSplitter::Base]
+        def self.call(path, baseid: SureRandom.uuid, tmpdir: Dir.mktmpdir)
+          new(path, baseid: baseid, tmpdir: tmpdir)
+        end
+
+        ##
+        # @param path [String] the path to the source PDF that we're processing.
+        # @param baseid [String] used for creating a unique identifier
+        # @param tmpdir [String] place to perform the "work" of splitting the PDF.
+        # @param pdf_pages_summary [Derivative::Rodeo::PdfPagesSummary] by default we'll
+        #        extract this from the given path, but for testing purposes, you might want to
+        #        provide a specific summary.
+        # @param logger [Logger, #error]
+        def initialize(path,
+                       baseid: SecureRandom.uuid,
+                       # TODO: Do we need to provide the :tmpdir for the application?
+                       tmpdir: Dir.mktmpdir,
+                       pdf_pages_summary: PagesSummary.extract_from(path: path),
+                       logger: DerivativeRodeo.config.logger)
+          @baseid = baseid
+          @pdfpath = path
+          @pdf_pages_summary = pdf_pages_summary
+          @tmpdir = tmpdir
+          @logger = logger
+        end
+
+        attr_reader :logger
+
+        # In creating {#each} we get many of the methods of array operation (e.g. #to_a).
+        include Enumerable
+
+        ##
+        # @api public
+        #
+        # @yieldparam [String] the path to the page's tiff.
+        def each(&block)
+          entries.each(&block)
+        end
+
+        # @api private
+        def invalid_pdf?
+          !pdf_pages_summary.valid?
+        end
+
+        attr_reader :pdf_pages_summary, :tmpdir, :baseid, :pdfpath
+        private :pdf_pages_summary, :tmpdir, :baseid, :pdfpath
+
+        # @api private
+        def gsdevice
+          return self.class.gsdevice if self.class.gsdevice
+
+          raise NotImplementedError, "#{self.class}#gsdevice"
+        end
+
+        private
+
+        # entries for each page
+        def entries
+          return @entries if defined? @entries
+
+          @entries = Array.wrap(gsconvert)
+        end
+
+        def output_base
+          @output_base ||= File.join(tmpdir, "#{baseid}-page%d.#{image_extension}")
+        end
+
+        def gsconvert
+          # NOTE: you must call gsdevice before compression, as compression is
+          # updated during the gsdevice call.
+          file_names = []
+
+          Open3.popen3(gsconvert_cmd(output_base)) do |_stdin, stdout, stderr, _wait_thr|
+            err = stderr.read
+            logger.error "#{self.class}#gsconvert encountered the following error with `gs': #{err}" if err.present?
+
+            page_number = 1
+            stdout.read.split("\n").each do |line|
+              next unless line.start_with?('Page ')
+
+              file_names << format(output_base, page_number)
+              page_number += 1
+            end
+          end
+
+          file_names
+        end
+
+        def create_file_name(line:, page_number:); end
+
+        def gsconvert_cmd(output_base)
+          @gsconvert_cmd ||= begin
+                               cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} -dTextAlphaBits=4"
+                               cmd += " -sCompression=#{compression}" if compression?
+                               cmd += " -dJPEGQ=#{quality}" if quality?
+                               cmd += " -sOutputFile=#{output_base} -r#{ppi} -f #{pdfpath}"
+                               cmd
+                             end
+        end
+
+        def pagecount
+          return @pagecount if defined? @pagecount
+
+          cmd = "pdfinfo #{pdfpath}"
+          Open3.popen3(cmd) do |_stdin, stdout, stderr, _wait_thr|
+            err = stderr.read
+            logger.error "#{self.class}#pagecount encountered the following error with `pdfinfo': #{err}" if err.present?
+            output = stdout.read
+            raise "pdfinfo failed to return output for #{pdfpath} - #{err}" if output.blank?
+            match = page_count_regexp.match(output)
+
+            @pagecount = match[1].to_i
+          end
+          @pagecount
+        end
+
+        def ppi
+          if looks_scanned?
+            # For scanned media, defer to detected image PPI:
+            pdf_pages_summary.ppi
+          else
+            # 400 dpi for something that does not look like scanned media:
+            default_dpi
+          end
+        end
+
+        def looks_scanned?
+          max_image_px = pdf_pages_summary.width * pdf_pages_summary.height
+          # single 10mp+ image per page?
+          single_image_per_page? && max_image_px > 1024 * 1024 * 10
+        end
+
+        def single_image_per_page?
+          pdf_pages_summary.page_count == pagecount
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter/jpg_page.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitter
+      # The purpose of this class is to split the PDF into constituent jpg files.
+      class JpgPage < PdfSplitter::Base
+        self.image_extension = 'jpg'
+        self.quality = '50'
+        self.gsdevice = 'jpeg'
+      end
+    end
+  end
+end