derivative-rodeo 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20e512d7162170875d60f90ff48bba694237ac7d31f38d806d2b87f570536c1c
-  data.tar.gz: a1311ea39a3994b4d24ffdbdbade62b7fbb15d2c326a3b510607f315fb4dd865
+  metadata.gz: dc2eed3e32c7a4558d55e9d530b6790a5b876dcdfc4ced421cfa4894aa977d44
+  data.tar.gz: 6e16e4bd7b9d38a1a19b1768a5cdb021c6aa946287f430c6a6c62fa26a215ca6
 SHA512:
-  metadata.gz: 157a9e276c6cefe739137fbe17e783557d0317dcee531cd353ad86987bda33ad55c2ada8179e254b306a467d01f8f759a6e89fc91b8cbcf6e968cf5a28a9037b
-  data.tar.gz: 5bd45db467194cf1e8af7f7e1ed625c2b3898d011f20a581a9a55a2ccbb7be56ca8c276752b7bef41e2c1d5efa4737d1291f83211dd69cd18e4f7caeed25fef2
+  metadata.gz: 0ac19d20f92490eed508949b18df66ce61d0850a22a2b8b1e514673ddd447afb578e8090d4234dc0a179b85c25a145e44bce6a1e71cfe2f67d2e3b438cb4b9ff
+  data.tar.gz: 6f503dd265243982bc9163b7fb6da42211eca3eb647b1ee9491fcbc06b373c6822222ee6d72c190f2e1bbd7ca63c8126102acd841b1e8f0240434a1af3a69a4f

data/lib/derivative_rodeo/generators/alto_generator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../services/extract_word_coordinates_from_hocr_sgml_service'
+
+module DerivativeRodeo
+  module Generators
+    ##
+    # Generate the Alto XML from the given input_uris.
+    #
+    # @note Assumes that we're receiving a HOCR file (generated via {HocrGenerator}).
+    class AltoGenerator < BaseGenerator
+      self.output_extension = "alto.xml"
+
+      class_attribute :service, default: Services::ExtractWordCoordinatesFromHocrSgmlService
+
+      ##
+      # @param output_location [StorageLocations::BaseLocation]
+      # @param input_tmp_file_path [String] the location of the file that we can use for processing.
+      #
+      # @return [StorageLocations::BaseLocation]
+      #
+      # @see #requisite_files
+      def build_step(output_location:, input_tmp_file_path:, **)
+        output_location.with_new_tmp_path do |output_tmp_file_path|
+          convert_to_coordinates(path_to_hocr: input_tmp_file_path, path_to_alto: output_tmp_file_path)
+        end
+      end
+
+      private
+
+      ##
+      # @param path_to_hocr [String]
+      # @param path_to_alto [String]
+      def convert_to_coordinates(path_to_hocr:, path_to_alto:)
+        hocr_html = File.read(path_to_hocr)
+        File.open(path_to_alto, "w+") do |file|
+          file.puts service.call(hocr_html).to_alto
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -27,7 +27,6 @@ module DerivativeRodeo
       # @!endgroup Class Attributes
 
       attr_reader :input_uris,
-                  :logger,
                   :output_location_template,
                   :preprocessed_location_template
 
@@ -39,23 +38,25 @@ module DerivativeRodeo
       #        to find preprocessed uris by transforming the :input_uris via
       #        {Services::ConvertUriViaTemplateService} with the given
       #        :preprocessed_location_template.
-      # @param logger [Logger]
-      def initialize(input_uris:, output_location_template:, preprocessed_location_template: nil, logger: DerivativeRodeo.config.logger)
+      def initialize(input_uris:, output_location_template:, preprocessed_location_template: nil)
+        # NOTE: Are we using this preprocessed_location_template?  Wondering?
         @input_uris = Array.wrap(input_uris)
         @output_location_template = output_location_template
         @preprocessed_location_template = preprocessed_location_template
-        @logger = logger
 
         return if valid_instantiation?
 
         raise Errors::ExtensionMissingError.new(klass: self.class)
       end
 
+      delegate :logger, to: DerivativeRodeo
+
       ##
       # @api private
       #
       # @return [Boolean]
       def valid_instantiation?
+        # TODO: Does this even make sense.
         # When we have a BaseGenerator and not one of it's children or when we've assigned the
         # output_extension.  instance_of? is more specific than is_a?
         instance_of?(DerivativeRodeo::Generators::BaseGenerator) || output_extension
@@ -83,6 +84,7 @@ module DerivativeRodeo
       # @see #build_step
       # @see #with_each_requisite_location_and_tmp_file_path
       def generated_files
+        # TODO: Examples please
         return @generated_files if defined?(@generated_files)
 
         # As much as I would like to use map or returned values; given the implementations it's
@@ -92,6 +94,9 @@ module DerivativeRodeo
         # helps ease subclass implementations of the #with_each_requisite_location_and_tmp_file_path or
         # #build_step
         @generated_files = []
+
+        # BaseLocation is like the Ruby `File` (Pathname) "File.exist?(path) :: location.exist?"
+        # "file:///Users/jfriesen/.profile"
         with_each_requisite_location_and_tmp_file_path do |input_location, input_tmp_file_path|
           generated_file = destination(input_location)
           @generated_files << if generated_file.exist?
@@ -170,7 +175,7 @@ module DerivativeRodeo
         return output_location unless preprocessed_location_template
 
         preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template)
-        # We only want
+        # We only want the location if it exists
         return preprocessed_location if preprocessed_location&.exist?
 
         # NOTE: The file does not exist at the output_location; but we pass this information along so

data/lib/derivative_rodeo/generators/pdf_split_generator.rb

@@ -17,19 +17,66 @@ module DerivativeRodeo
       include CopyFileConcern
 
       ##
-      # @param name [#to_s] Convert the given name into the resulting {Services::PdfSplitter::Base}.
+      # A helper method for downstream implementations to ask if this file is perhaps split from a
+      # PDF.
       #
-      # @return [#call, Services::PdfSplitter::Base]
-      def pdf_splitter(name: pdf_splitter_name)
-        @pdf_splitter ||= Services::PdfSplitter.for(name)
+      # @param filename [String]
+      # @param extension [String] the extension (either with or without the leading period); if none
+      #        is provided use the extension of the given :filename.
+      # @return [TrueClass] when the file name likely represents a file split from a PDF.
+      # @return [FalseClass] when the file name does not, by convention, represent a file split from
+      #         a PDF.
+      #
+      # @see #image_file_basename_template
+      def self.filename_for_a_derived_page_from_a_pdf?(filename:, extension: nil)
+        extension ||= File.extname(filename)
+
+        # Strip the leading period from the extension.
+        extension = extension[1..-1] if extension.start_with?('.')
+        regexp = %r{--page-\d+\.#{extension}$}
+        !!regexp.match(filename)
       end
 
       ##
-      # @return [Symbol]
+      # @param basename [String] The given PDF file's base name (e.g. "hello.pdf" would have a base name of
+      #        "hello").
+      #
+      # @return [String] A template for the filenames of the images produced by Ghostscript.
+      #
+      # @note This must include "%d" in the returning value, as that is how Ghostscript will assign
+      # the page number.
+      #
+      # @note I have extracted this function to make it abundantly clear the expected location
+      # each split image.  Further there is an interaction in this
       #
-      # @see .output_extension
-      def pdf_splitter_name
-        output_extension.to_s.split(".").last.to_sym
+      # @see #existing_page_locations
+      # @see .filename_for_a_derived_page_from_a_pdf?
+      def image_file_basename_template(basename:)
+        "#{basename}/pages/#{basename}--page-%d.#{output_extension}"
+      end
+
+      ##
+      # We want to check the output location and pre-processed location for the existence of already
+      # split pages.  This method checks both places.
+      #
+      # @param input_location [StorageLocations::BaseLocation]
+      #
+      # @return [Enumerable<StorageLocations::BaseLocation>] the files at the given :input_location
+      #         with :tail_glob.
+      #
+      # @note There is relation to {Generators::BaseGenerator#destination} and this method.
+      #
+      # @note The tail_glob is in relation to the {#image_file_basename_template}
+      def existing_page_locations(input_location:)
+        # See image_file_basename_template
+        tail_glob = "#{input_location.file_basename}/pages/*.#{output_extension}"
+
+        output_locations = input_location.derived_file_from(template: output_location_template).globbed_tail_locations(tail_glob: tail_glob)
+        return output_locations if output_locations.count.positive?
+
+        return [] if preprocessed_location_template.blank?
+
+        input_location.derived_file_from(template: preprocessed_location_template).globbed_tail_loations(tail_glob: tail_glob)
       end
 
       ##
@@ -44,18 +91,35 @@ module DerivativeRodeo
       # @yieldparam image_location [StorageLocations::FileLocation] the file and adapter logic.
       # @yieldparam image_path [String] where to find this file in the tmp space
       #
+      # @note This function makes a concession; namely that if it encounters any
+      # {#existing_page_locations} it will use all of that result as the entire number of pages.
+      # We could make this smarter but at the moment we're deferring on that.
+      #
       # @see BaseGenerator#with_each_requisite_location_and_tmp_file_path for further discussion
+      #
+      # rubocop:disable Metrics/MethodLength
       def with_each_requisite_location_and_tmp_file_path
         input_files.each do |input_location|
           input_location.with_existing_tmp_path do |input_tmp_file_path|
-            image_paths = pdf_splitter.call(input_tmp_file_path, baseid: input_location.file_basename, tmpdir: File.dirname(input_tmp_file_path))
-            image_paths.each do |image_path|
+            ## We want a single call for a directory listing of the image_file_basename_template
+            generated_files = existing_page_locations(input_location: input_location)
+
+            if generated_files.count.zero?
+              generated_files = Services::PdfSplitter.call(
+                input_tmp_file_path,
+                image_extension: output_extension,
+                image_file_basename_template: image_file_basename_template(basename: input_location.file_basename)
+              )
+            end
+
+            generated_files.each do |image_path|
               image_location = StorageLocations::FileLocation.new("file://#{image_path}")
               yield(image_location, image_path)
             end
           end
         end
       end
+      # rubocop:enable Metrics/MethodLength
     end
   end
 end

data/lib/derivative_rodeo/generators/plain_text_generator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../services/extract_word_coordinates_from_hocr_sgml_service'
+
+module DerivativeRodeo
+  module Generators
+    ##
+    # Generate the word coordinates (as JSON) from the given input_uris.
+    #
+    # @note Assumes that we're receiving a HOCR file (generated via {HocrGenerator}).
+    class PlainTextGenerator < BaseGenerator
+      self.output_extension = "plain_text.txt"
+
+      class_attribute :service, default: Services::ExtractWordCoordinatesFromHocrSgmlService
+
+      ##
+      # @param output_location [StorageLocations::BaseLocation]
+      # @param input_tmp_file_path [String] the location of the file that we can use for processing.
+      #
+      # @return [StorageLocations::BaseLocation]
+      #
+      # @see #requisite_files
+      def build_step(output_location:, input_tmp_file_path:, **)
+        output_location.with_new_tmp_path do |output_tmp_file_path|
+          convert_to_coordinates(path_to_hocr: input_tmp_file_path, path_to_plain_text: output_tmp_file_path)
+        end
+      end
+
+      private
+
+      ##
+      # @param path_to_hocr [String]
+      # @param path_to_plain_text [String]
+      def convert_to_coordinates(path_to_hocr:, path_to_plain_text:)
+        hocr_html = File.read(path_to_hocr)
+        File.open(path_to_plain_text, "w+") do |file|
+          file.puts service.call(hocr_html).to_text
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/generators/thumbnail_generator.rb

@@ -6,12 +6,33 @@ module DerivativeRodeo
     # This generator is responsible for converting a given binary into a thumbnail.  As of
     # <2023-05-22 Mon>, we're needing to generate thumbnails for PDFs and images.
     class ThumbnailGenerator < BaseGenerator
+      ##
+      # @!group Class Attributes
+
       ##
       # We want to mirror the same file "last" extension as described in Hyrax.
       #
       # @see https://github.com/samvera/hyrax/blob/426575a9065a5dd3b30f458f5589a0a705ad7be2/app/services/hyrax/file_set_derivatives_service.rb
       self.output_extension = 'thumbnail.jpeg'
 
+      ##
+      # @!attribute dimensions_by_type
+      #
+      #   @return [Hash<Symbol,String>] the "types" (as categorized by
+      #           Hyrax::FileSetDerivativeService).  These aren't mime-types per se but a conceptual
+      #           distillation of that.
+      #
+      #   @see https://github.com/samvera/hyrax/blob/815e0abaacf9f331a5640c5d6129661d01eadf75/app/services/hyrax/file_set_derivatives_service.rb
+      class_attribute :dimensions_by_type, default: { pdf: "338x493" }
+
+      ##
+      # @!attribute dimensions_fallback
+      #
+      #   @return [String] when there's no match for {.dimensions_by_type} use this value.
+      class_attribute :dimensions_fallback, default: "200x150>"
+      # @!endgroup Class Attributes
+      ##
+
       ##
       # @param output_location [StorageLocations::BaseLocation]
       # @param input_tmp_file_path [String] the location of the file that we can use for processing.
@@ -23,6 +44,20 @@ module DerivativeRodeo
         end
       end
 
+      ##
+      # @param filename [String]
+      # @return [String]
+      #
+      # @see .dimensions_by_type
+      # @see .dimensions_fallback
+      def self.dimensions_for(filename:)
+        type = DerivativeRodeo::Services::MimeTypeService.hyrax_type(filename: filename)
+        dimensions_by_type.fetch(type, dimensions_fallback)
+      end
+
+      # Want to expose the dimensions_for as an instance method
+      delegate :dimensions_for, to: :class
+
       ##
       # Convert the file found at :path_to_input into a thumbnail, writing it to the
       # :path_for_thumbnail_output
@@ -30,8 +65,8 @@ module DerivativeRodeo
       # @param path_of_file_to_create_thumbnail_from [String]
       # @param path_for_thumbnail_output [String]
       def thumbnify(path_of_file_to_create_thumbnail_from:, path_for_thumbnail_output:)
-        # @todo the dimensions might not be always 200x150, figure out a way to make it dynamic
-        `convert #{path_of_file_to_create_thumbnail_from} -thumbnail '200x150>' -flatten #{path_for_thumbnail_output}`
+        dimensions = dimensions_for(filename: path_of_file_to_create_thumbnail_from)
+        `convert #{path_of_file_to_create_thumbnail_from} -thumbnail '#{dimensions}' -flatten #{path_for_thumbnail_output}`
       end
     end
   end

data/lib/derivative_rodeo/generators/word_coordinates_generator.rb

@@ -31,7 +31,7 @@ module DerivativeRodeo
       def convert_to_coordinates(path_to_hocr:, path_to_coordinate:, service: Services::ExtractWordCoordinatesFromHocrSgmlService)
         hocr_html = File.read(path_to_hocr)
         File.open(path_to_coordinate, "w+") do |file|
-          file.puts service.call(hocr_html)
+          file.puts service.call(hocr_html).to_json
         end
       end
     end

data/lib/derivative_rodeo/services/extract_word_coordinates_from_hocr_sgml_service.rb

@@ -13,7 +13,7 @@ module DerivativeRodeo
       # @param sgml [String] The SGML (e.g. XML or HTML) text of a HOCR file.
       # @return [String] A JSON document
       def self.call(sgml)
-        new(sgml).to_json
+        new(sgml)
       end
 
       ##
@@ -42,6 +42,21 @@ module DerivativeRodeo
       end
       alias json to_json
 
+      # Output plain text, keeping the method calls consistent with so calling this #to_text
+      #
+      # @return [String] plain text of OCR'd document
+      def to_text
+        @to_text ||= doc_stream.text
+      end
+
+      def to_alto
+        @to_alto ||= AltoXml.to_alto(
+          words: doc_stream.words,
+          width: doc_stream.width,
+          height: doc_stream.height
+        )
+      end
+
       private
 
       def xml?(xml)
@@ -121,6 +136,7 @@ module DerivativeRodeo
           # add trailing space to plaintext buffer for between words:
           @text += ' '
           @words.push(@current) if word_complete?
+          @current = nil # clear the current word
         end
 
         def end_line
@@ -156,10 +172,13 @@ module DerivativeRodeo
         # Callback for element end; at this time, flush word coordinate state
         #   for current word, and append line endings to plain text:
         #
-        # @param _name [String] element name.
-        def end_element(_name)
-          end_line if @element_class_name == 'ocr_line'
-          end_word if @element_class_name == 'ocrx_word'
+        # @param name [String] element name.
+        def end_element(name)
+          if name == 'span'
+            end_word if @element_class_name == 'ocrx_word'
+            @text += "\n" if @element_class_name.nil?
+          end
+          @element_class_name = nil
         end
 
         # Callback for completion of parsing hOCR, used to normalize generated
@@ -213,6 +232,102 @@ module DerivativeRodeo
           JSON.generate(payload)
         end
       end
+
+      class AltoXml
+        ##
+        # @api public
+        #
+        # @param words [Array<Hash>] an array of hash objects that have the keys `:word` and `:coordinates`.
+        # @param width [Integer, nil] the width of the "canvas" on which the words appear.
+        # @param height [Integer, nil] the height of the "canvas" on which the words appear.
+        #
+        # @return [String] the ALTO XML representation of the given words and their coordinates.
+        def self.to_alto(words:, width: nil, height: nil)
+          new(words: words, width: width, height: height).to_alto
+        end
+
+        def initialize(words:, width:, height:, scaling: 1.0)
+          @words = words
+          @height = height.to_i
+          @width = width.to_i
+          @scaling = scaling
+        end
+
+        attr_reader :words, :width, :height, :scaling
+
+        # Output ALTO XML of word coordinates
+        #
+        # @return [String] ALTO XML representation of the words and their coordinates
+        def to_alto
+          page = alto_page(width, height) do |xml|
+            words.each do |word|
+              xml.String(
+                CONTENT: word[:word],
+                WIDTH: scale_point(word[:coordinates][2]).to_s,
+                HEIGHT: scale_point(word[:coordinates][3]).to_s,
+                HPOS: scale_point(word[:coordinates][0]).to_s,
+                VPOS: scale_point(word[:coordinates][1]).to_s
+              ) { xml.text '' }
+            end
+          end
+          page.to_xml
+        end
+
+        private
+
+        # given block to manage word generation, wrap with page/block/line
+        def alto_page(pixel_width, pixel_height, &block)
+          builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+            xml.alto(xmlns: 'http://www.loc.gov/standards/alto/ns-v2#') do
+              xml.Description do
+                xml.MeasurementUnit 'pixel'
+              end
+              alto_layout(xml, pixel_width, pixel_height, &block)
+            end
+          end
+          builder
+        end
+
+        def scale_point(value)
+          # NOTE: presuming non-fractional, even though ALTO 2.1
+          #   specifies coordinates are xsd:float, not xsd:int,
+          #   simplify to integer value for output:
+          (value * scaling).to_i
+        end
+
+        # return layout for page
+        def alto_layout(xml, pixel_width, pixel_height, &block)
+          xml.Layout do
+            xml.Page(ID: 'ID1',
+                     PHYSICAL_IMG_NR: '1',
+                     HEIGHT: pixel_height,
+                     WIDTH: pixel_width) do
+              xml.PrintSpace(HEIGHT: pixel_height,
+                             WIDTH: pixel_width,
+                             HPOS: '0',
+                             VPOS: '0') do
+                alto_blockline(xml, pixel_width, pixel_height, &block)
+              end
+            end
+          end
+        end
+
+        # make block line and call word-block
+        def alto_blockline(xml, pixel_width, pixel_height)
+          xml.TextBlock(ID: 'ID1a',
+                        HEIGHT: pixel_height,
+                        WIDTH: pixel_width,
+                        HPOS: '0',
+                        VPOS: '0') do
+            xml.TextLine(HEIGHT: pixel_height,
+                         WIDTH: pixel_width,
+                         HPOS: '0',
+                         VPOS: '0') do
+              yield(xml)
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/derivative_rodeo/services/mime_type_service.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+require 'marcel'
+
+module DerivativeRodeo
+  module Services
+    ##
+    # This module provides an interface for determining a mime-type.
+    module MimeTypeService
+      ##
+      # Hyrax has it's own compression of mime_types into conceptual types (as defined in
+      # Hyrax::FileSetDerivativesService).  This provides a somewhat conceptual overlap with that,
+      # while also being more generalized.
+      #
+      # @param filename [String]
+      # @return [Symbol]
+      def self.hyrax_type(filename:)
+        mime = mime_type(filename: filename)
+        media_type, sub_type = mime.split("/")
+        case media_type
+        when "image", "audio", "text", "video"
+          media_type.to_sym
+        when "application" # The wild woolly weird world of all the things.
+          # TODO: Do we need to worry about office documents?
+          sub_type.to_sym
+        else
+          sub_type.to_sym
+        end
+      end
+
+      ##
+      # Given a local :filename (e.g. downloaded and available on the server this is running),
+      # return the mime_type of the file.
+      #
+      # @param filename [String]
+      # @return [String] (e.g. "application/pdf", "text/plain")
+      def self.mime_type(filename:)
+        ##
+        # TODO: Does this attempt to read the whole file?  That may create memory constraints.  By
+        # using Pathname (instead of File.read), we're letting Marcel do it's best mime magic.
+        pathname = Pathname.new(filename)
+        extension = filename.split(".")&.last&.downcase
+        if extension
+          # By including a possible extension, we can help nudge Marcel into making a more
+          # Without extension, we will get a lot of "application/octet-stream" results.
+          ::Marcel::MimeType.for(pathname, extension: extension)
+        else
+          ::Marcel::MimeType.for(pathname)
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter/base.rb

@@ -2,17 +2,32 @@
 
 require 'open3'
 require 'securerandom'
-require 'tmpdir'
-
 module DerivativeRodeo
   module Services
+    ##
+    # A service module for splitting PDFs into one image per page.
+    #
+    # @see .call
     module PdfSplitter
       ##
-      # @param name [String]
-      # @return [PdfSplitter::Base]
-      def self.for(name)
-        klass_name = "#{name.to_s.classify}_page".classify
-        "DerivativeRodeo::Services::PdfSplitter::#{klass_name}".constantize
+      # @api public
+      #
+      # Split the file found at the given :path
+      #
+      # @param path [String] the path to the source PDF that we're processing.
+      # @param image_extension [String] used to determine the splitting service we use; there is an
+      #        implicit relationship between image_extension and image_file_basename_template
+      #        (though filenames do not necessarily reflect mime types)
+      # @param image_file_basename_template [String] use this string to generate the unique filename
+      #        for an image "split" from the given PDF.  It must include "%d" as part of the
+      #        declaration.  For example if the template is "hello-world-%d.png" then the first
+      #        split page will be "hello-world-1.png".
+      #
+      # @return [Enumerable, Utilities::PdfSplitter::Base, #each] see {Base#each}
+      def self.call(path, image_extension:, image_file_basename_template:)
+        klass_name = "#{image_extension.to_s.classify}_page".classify
+        klass = "DerivativeRodeo::Services::PdfSplitter::#{klass_name}".constantize
+        klass.new(path, image_file_basename_template: image_file_basename_template)
       end
 
       ##
@@ -31,38 +46,23 @@ module DerivativeRodeo
 
         class_attribute :gsdevice, instance_accessor: false
         class_attribute :page_count_regexp, instance_accessor: true, default: /^Pages: +(\d+)$/
-        ##
-        # @api public
-        #
-        # @param path [String] The path the the PDF
-        #
-        # @return [Enumerable, Utilities::PdfSplitter::Base]
-        def self.call(path, baseid: SureRandom.uuid, tmpdir: Dir.mktmpdir)
-          new(path, baseid: baseid, tmpdir: tmpdir)
-        end
 
-        ##
-        # @param path [String] the path to the source PDF that we're processing.
-        # @param baseid [String] used for creating a unique identifier
-        # @param tmpdir [String] place to perform the "work" of splitting the PDF.
-        # @param pdf_pages_summary [Derivative::Rodeo::PdfPagesSummary] by default we'll
-        #        extract this from the given path, but for testing purposes, you might want to
-        #        provide a specific summary.
-        # @param logger [Logger, #error]
         def initialize(path,
-                       baseid: SecureRandom.uuid,
-                       # TODO: Do we need to provide the :tmpdir for the application?
-                       tmpdir: Dir.mktmpdir,
-                       pdf_pages_summary: PagesSummary.extract_from(path: path),
-                       logger: DerivativeRodeo.config.logger)
-          @baseid = baseid
+                       image_file_basename_template:,
+                       pdf_pages_summary: PagesSummary.extract_from(path: path))
           @pdfpath = path
           @pdf_pages_summary = pdf_pages_summary
-          @tmpdir = tmpdir
-          @logger = logger
+          @ghost_script_output_file_template = File.join(File.dirname(path), image_file_basename_template)
+
+          # We need to ensure that this temporary directory exists so we can write the files to it.
+          # Fortunately, because this file space must be "local" tmp dir, we don't need to work
+          # through any of the location antics of {StorageLocations::BaseLocation}.
+          FileUtils.mkdir_p(File.dirname(@ghost_script_output_file_template))
         end
 
-        attr_reader :logger
+        attr_reader :ghost_script_output_file_template
+
+        delegate :logger, to: DerivativeRodeo
 
         # In creating {#each} we get many of the methods of array operation (e.g. #to_a).
         include Enumerable
@@ -80,8 +80,8 @@ module DerivativeRodeo
           !pdf_pages_summary.valid?
         end
 
-        attr_reader :pdf_pages_summary, :tmpdir, :baseid, :pdfpath
-        private :pdf_pages_summary, :tmpdir, :baseid, :pdfpath
+        attr_reader :pdf_pages_summary, :basename, :pdfpath
+        private :pdf_pages_summary, :basename, :pdfpath
 
         # @api private
         def gsdevice
@@ -99,16 +99,12 @@ module DerivativeRodeo
           @entries = Array.wrap(gsconvert)
         end
 
-        def output_base
-          @output_base ||= File.join(tmpdir, "#{baseid}-page%d.#{image_extension}")
-        end
-
         def gsconvert
           # NOTE: you must call gsdevice before compression, as compression is
           # updated during the gsdevice call.
           file_names = []
 
-          Open3.popen3(gsconvert_cmd(output_base)) do |_stdin, stdout, stderr, _wait_thr|
+          Open3.popen3(gsconvert_cmd(ghost_script_output_file_template)) do |_stdin, stdout, stderr, _wait_thr|
             err = stderr.read
             logger.error "#{self.class}#gsconvert encountered the following error with `gs': #{err}" if err.present?
 
@@ -116,7 +112,7 @@ module DerivativeRodeo
             stdout.read.split("\n").each do |line|
               next unless line.start_with?('Page ')
 
-              file_names << format(output_base, page_number)
+              file_names << format(ghost_script_output_file_template, page_number)
               page_number += 1
             end
           end
@@ -126,12 +122,12 @@ module DerivativeRodeo
 
         def create_file_name(line:, page_number:); end
 
-        def gsconvert_cmd(output_base)
+        def gsconvert_cmd(ghost_script_output_file_template)
           @gsconvert_cmd ||= begin
                                cmd = "gs -dNOPAUSE -dBATCH -sDEVICE=#{gsdevice} -dTextAlphaBits=4"
                                cmd += " -sCompression=#{compression}" if compression?
                                cmd += " -dJPEGQ=#{quality}" if quality?
-                               cmd += " -sOutputFile=#{output_base} -r#{ppi} -f #{pdfpath}"
+                               cmd += " -sOutputFile=#{ghost_script_output_file_template} -r#{ppi} -f #{pdfpath}"
                                cmd
                              end
         end

data/lib/derivative_rodeo/services/url_service.rb

@@ -21,7 +21,7 @@ module DerivativeRodeo
       def self.read(url)
         HTTParty.get(url, logger: DerivativeRodeo.config.logger).body
       rescue StandardError => e
-        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        DerivativeRodeo.config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
         raise e
       end
 

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -42,6 +42,8 @@ module DerivativeRodeo
 
       class << self
         alias scheme location_name
+
+        delegate :config, to: DerivativeRodeo
       end
 
       ##
@@ -206,6 +208,22 @@ module DerivativeRodeo
         klass.build(from_uri: file_path, template: template)
       end
 
+      ##
+      # When you have a known location and want to check for files that are within that location,
+      # use the {#globbed_tail_locations} method.  In the case of {Generators::PdfSplitGenerator} we
+      # need to know the path to all of the image files we "split" off of the given PDF.
+      #
+      # We can use the :file_path as the prefix the given :tail_glob as the suffix for a "fully
+      # qualified" Dir.glob type search.
+      #
+      # @param tail_glob [String]
+      #
+      # @return [Enumerable<StorageLocations::BaseLocation>] the locations of the files; an empty
+      #         array when there are none.
+      def globbed_tail_locations(tail_glob:)
+        raise NotImplementedError, "#{self.class}#globbed_locations"
+      end
+
       ##
       # @param extension [String, StorageLocations::SAME]
       # @return [String] the path for the new extension; when given {StorageLocations::SAME} re-use

data/lib/derivative_rodeo/storage_locations/concerns/download_concern.rb

@@ -43,7 +43,7 @@ module DerivativeRodeo
       # @param url [String]
       #
       # @return [String]
-      def read(url)
+      def get(url)
         HTTParty.get(url, logger: config.logger)
       rescue => e
         config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
@@ -51,13 +51,11 @@ module DerivativeRodeo
       end
 
       ##
-      # @param url [String]
-      #
       # @return [URI] when the URL resolves successfully
       # @return [FalseClass] when the URL's head request is not successful or we've exhausted our
       #         remaining redirects.
-      def exists?(url)
-        HTTParty.head(url, logger: config.logger)
+      def exist?
+        HTTParty.head(file_uri, logger: config.logger)
       rescue => e
         config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
         false

data/lib/derivative_rodeo/storage_locations/file_location.rb

@@ -34,6 +34,10 @@ module DerivativeRodeo
         FileUtils.cp_r(tmp_file_path, file_path)
         file_uri
       end
+
+      def globbed_tail_locations(tail_glob:)
+        Dir.glob(File.join(file_dir, tail_glob))
+      end
     end
   end
 end

data/lib/derivative_rodeo/storage_locations/s3_location.rb

@@ -8,7 +8,15 @@ module DerivativeRodeo
     # Location to download and upload files to S3
     #
     class S3Location < BaseLocation
-      attr_writer :bucket
+      ##
+      # @!group Class Attributes
+      # @!attribute use_actual_s3_bucket
+      #
+      # When true , we are going to use a live S3 bucket.  When false, we'll use a fake local bucket.
+      class_attribute :use_actual_s3_bucket, default: true
+      # @!endgroup Class Attributes
+      ##
+
       ##
       # Create a new uri of the classes type. Parts argument should have a default in
       # implementing classes. Must support a number or the symbol :all
@@ -24,10 +32,11 @@ module DerivativeRodeo
       end
 
       ##
-      # @param config [DerivativeRodeo::Configuration]
+      # @param bucket_name [String, NilClass] when given, use this as the bucket, otherwise, def
+      #
       # @return [String]
-      def self.adapter_prefix(config: DerivativeRodeo.config)
-        "#{scheme}://#{config.aws_s3_bucket}.s3.#{config.aws_s3_region}.amazonaws.com"
+      def self.adapter_prefix(bucket_name: config.aws_s3_bucket)
+        "#{scheme}://#{bucket_name}.s3.#{config.aws_s3_region}.amazonaws.com"
       end
 
       ##
@@ -53,6 +62,38 @@ module DerivativeRodeo
         bucket.objects(prefix: file_path).count.positive?
       end
 
+      ##
+      # @return [Enumerable<DerivativeRodeo::StorageLocations::S3Location>]
+      #
+      # @note S3 allows searching on a prefix but does not allow for "wildcard" searches.  We can
+      #       use the components of the file_path to fake that behavior.
+      #
+      # @see Generators::PdfSplitGenerator#image_file_basename_template
+      def globbed_tail_locations(tail_glob:)
+        # file_path = "s3://blah/1234/hello-world/pages/*.tiff"
+        #
+        # NOTE: Should we be storing our files as such?  The pattern we need is
+        # :parent_identifier/:file_set_identifier/files There are probably cases where a work has
+        # more than one PDF (that we intend to split); we don't want to trample on those split files
+        # and miscolate two PDFs.
+        #
+        # file_path = "s3://blah/1234/hello-world/hello-world.pdf
+        globname = File.join(file_dir, tail_glob)
+        regexp = %r{#{File.extname(globname)}$}
+
+        # NOTE: We're making some informed guesses, needing to include the fully qualified template
+        # based on both the key of the item in the bucket as well as the bucket's host.
+        uri = URI.parse(file_uri)
+        scheme_and_host = "#{uri.scheme}://#{uri.host}"
+
+        bucket.objects(prefix: File.dirname(globname)).flat_map do |object|
+          if object.key.match(regexp)
+            template = File.join(scheme_and_host, object.key)
+            derived_file_from(template: template)
+          end
+        end
+      end
+
       ##
       # @api public
       # write the tmp file to the file_uri
@@ -71,6 +112,9 @@ module DerivativeRodeo
       #
       # @return [Aws::S3::Resource]
       def resource
+        # TODO: Are there instantiation considerations when running in Lambda?  In tests
+        # initializing a resource is very slow (e.g. 3 seconds or so).  Should this be a class
+        # method?  Can it be given the SpaceStone constraints?
         @resource ||= if DerivativeRodeo.config.aws_s3_access_key_id
                         Aws::S3::Resource.new(region: DerivativeRodeo.config.aws_s3_region,
                                               credentials: Aws::Credentials.new(
@@ -91,13 +135,28 @@ module DerivativeRodeo
         raise Errors::BucketMissingError
       end
 
+      # @see .use_actual_s3_bucket
       def bucket
-        @bucket ||= resource.bucket(bucket_name)
+        @bucket ||= if use_actual_s3_bucket?
+                      resource.bucket(bucket_name)
+                    else
+                      self.class.faux_bucket
+                    end
       end
 
       def file_path
         @file_path ||= @file_uri.sub(%r{.+://.+?/}, '')
       end
+
+      ##
+      # A fake constructed fake bucket that confroms to the narrow S3 interface that we use.
+      #
+      # @see .use_actual_s3_bucket
+      # @return [AwsS3FauxBucket]
+      def self.faux_bucket
+        # We are not requiring this file; except in the spec context.
+        @faux_bucket ||= AwsS3FauxBucket.new
+      end
     end
   end
 end

data/lib/derivative_rodeo/storage_locations/sqs_location.rb

@@ -14,9 +14,20 @@ module DerivativeRodeo
     # It uploads a file_uri to the queue, not the contents of that file
     # reading from the queue is not currently implemented
     class SqsLocation < BaseLocation
+      ##
+      # @!group Class Attributes
+      #
+      # @!attribute batch_size
+      #   @return [Integer]
       class_attribute :batch_size, default: 10
 
-      attr_writer :client
+      # @!attribute use_real_sqs
+      #   When true, use the real SQS; else when false use a fake one.  You probably don't want to
+      #   use the fake one in your production.  But it's exposed in this manner to ease testing of
+      #   downstream dependencies.
+      class_attribute :use_real_sqs, default: true
+      # @!endgroup Class Attributes
+      ##
 
       ##
       # Create a new uri of the classes type. Parts argument should have a default in
@@ -82,19 +93,26 @@ module DerivativeRodeo
         file_uri
       end
 
+      # rubocop:disable Metrics/MethodLength
       def client
-        @client ||= if config.aws_sqs_access_key_id && config.aws_sqs_secret_access_key
-                      Aws::SQS::Client.new(
-                        region: config.aws_sqs_region,
-                        credentials: Aws::Credentials.new(
-                          config.aws_sqs_access_key_id,
-                          config.aws_sqs_secret_access_key
+        @client ||= if use_real_sqs?
+                      if config.aws_sqs_access_key_id && config.aws_sqs_secret_access_key
+                        Aws::SQS::Client.new(
+                          region: config.aws_sqs_region,
+                          credentials: Aws::Credentials.new(
+                            config.aws_sqs_access_key_id,
+                            config.aws_sqs_secret_access_key
+                          )
                         )
-                      )
+                      else
+                        Aws::SQS::Client.new(region: config.aws_sqs_region)
+                      end
                     else
-                      Aws::SQS::Client.new(region: config.aws_sqs_region)
+                      # We are not requiring this file; except in the spec context.
+                      AwsSqsFauxClient.new
                     end
       end
+      # rubocop:enable Metrics/MethodLength
 
       def add(message:)
         client.send_message({

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/derivative_rodeo.rb

@@ -33,4 +33,8 @@ module DerivativeRodeo
     yield(@config) if block_given?
     @config
   end
+
+  class << self
+    delegate :logger, to: :config
+  end
 end

data/lib/spec_support/aws_s3_faux_bucket.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+##
+# This class is very rudimentary implementation of a bucket.  It conforms to the necessary
+# interface for downloading and uploading and filter on prefix.
+#
+# It is provided as a lib/spec/support so that downstream implementations can leverage a fake S3
+# bucket.
+#
+# @see [DerivativeRodeo::StorageLocations::S3Location]
+class AwsS3FauxBucket
+  def initialize
+    @storage = {}
+  end
+  attr_reader :storage
+  def object(path)
+    # Yup, we've got nested buckets
+    @storage[path] ||= Storage.new(key: path)
+  end
+
+  def objects(prefix:)
+    @storage.each_with_object([]) do |(path, obj), accumulator|
+      accumulator << obj if path.start_with?(prefix)
+    end
+  end
+
+  class Storage
+    # Because we're now coping with the glob tail finder, we need to account for the bucket entry's
+    # key.
+    def initialize(key:)
+      @key = key
+      @storage = {}
+    end
+    attr_reader :storage, :key
+
+    def upload_file(path)
+      @storage[:upload] = File.read(path)
+    end
+
+    def download_file(path)
+      return false unless @storage.key?(:upload)
+      content = @storage.fetch(:upload)
+      File.open(path, 'wb') do |f|
+        f.puts(content)
+      end
+    end
+  end
+end

data/lib/spec_support/aws_sqs_faux_client.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+require 'ostruct'
+##
+# This class is very rudimentary implementation of an SQS client.  It conforms to the necessary
+# interface for sending messages and reading messages
+#
+# @see [DerivativeRodeo::StorageAdapters::SqsAdapter]
+class AwsSqsFauxClient
+  def initialize(queue_url: nil)
+    @queue_url = queue_url || 'https://sqs.us-west-2.amazonaws.com/5555555555/fake'
+    @storage = {}
+  end
+  attr_reader :storage
+
+  def send_message(arg_hash)
+    @storage[arg_hash[:queue_url]] ||= []
+    @storage[arg_hash[:queue_url]] << arg_hash[:message_body]
+  end
+
+  def send_message_batch(arg_hash)
+    @storage[arg_hash[:queue_url]] ||= []
+    @storage[arg_hash[:queue_url]] += arg_hash[:entries]
+  end
+
+  def receive_message(arg_hash)
+    output = []
+    args_hash[:mx_number_of_messages].times do
+      value = @storage[arg_hash[:queue_url]]&.pop
+      output << value if value
+    end
+  end
+
+  def get_queue_url(*)
+    OpenStruct.new(queue_url: @queue_url)
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-25 00:00:00.000000000 Z
+date: 2023-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -281,12 +281,14 @@ files:
 - lib/derivative_rodeo.rb
 - lib/derivative_rodeo/configuration.rb
 - lib/derivative_rodeo/errors.rb
+- lib/derivative_rodeo/generators/alto_generator.rb
 - lib/derivative_rodeo/generators/base_generator.rb
 - lib/derivative_rodeo/generators/concerns/copy_file_concern.rb
 - lib/derivative_rodeo/generators/copy_generator.rb
 - lib/derivative_rodeo/generators/hocr_generator.rb
 - lib/derivative_rodeo/generators/monochrome_generator.rb
 - lib/derivative_rodeo/generators/pdf_split_generator.rb
+- lib/derivative_rodeo/generators/plain_text_generator.rb
 - lib/derivative_rodeo/generators/thumbnail_generator.rb
 - lib/derivative_rodeo/generators/word_coordinates_generator.rb
 - lib/derivative_rodeo/services/base_service.rb
@@ -295,6 +297,7 @@ files:
 - lib/derivative_rodeo/services/image_identify_service.rb
 - lib/derivative_rodeo/services/image_jp2_service.rb
 - lib/derivative_rodeo/services/image_service.rb
+- lib/derivative_rodeo/services/mime_type_service.rb
 - lib/derivative_rodeo/services/pdf_splitter/base.rb
 - lib/derivative_rodeo/services/pdf_splitter/jpg_page.rb
 - lib/derivative_rodeo/services/pdf_splitter/pages_summary.rb
@@ -311,6 +314,8 @@ files:
 - lib/derivative_rodeo/storage_locations/sqs_location.rb
 - lib/derivative_rodeo/technical_metadata.rb
 - lib/derivative_rodeo/version.rb
+- lib/spec_support/aws_s3_faux_bucket.rb
+- lib/spec_support/aws_sqs_faux_client.rb
 homepage: https://github.com/scientist-softserv/derivative_rodeo
 licenses:
 - APACHE-2.0