derivative-rodeo 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bee0908ac5d045db1948b062d3a4e1569fff41b4a5a0f1521f31b620c15c53a6
-  data.tar.gz: 328de6f1cde3bcdadca31a361821b2c573425d15bfddbea5bf85d9475a8a3d0e
+  metadata.gz: f5ea42b2912b6b1eb6981b8a72a3d39e4d6f466e07ad60381ba4880431214b18
+  data.tar.gz: 4e4ca5cdd6ba61898ba13970fbd95c8836c33492c377eb6f258255c8ebf79e67
 SHA512:
-  metadata.gz: 2d8ba019ef30666d9633b73e90fe599b53d5f6698de0400b43a25c37a06470542962db8f61d74ed6a444f1ecc5ffbd604d2c0891dffcf5fa86d9dce87ace1bb5
-  data.tar.gz: b682377ce08e4379f1323bfdb6c6b8f46c97d45ebb392bb1dc7f5bc27f5b01701731880dc08f96cb5ef2b0da8ad39349b4534a63b4039e78616331b93bd9516c
+  metadata.gz: 6ebaacf09b1459e8bb8527ddd3b45b6e84106d335e91cc506a6b9b31aae6c74931c755dc5e084e9a39a6ddb0ca8f1b0f85ee21e461cac445edd3eea45384b1b5
+  data.tar.gz: e11195b3b7169a1f7e2f5df821c681512af93123fc272d3b1dd35d9c971f15ad58a86b4ab1cabf38b463c77d5531114af7ca36eb9197275e46549788a4ec6c1f

data/lib/derivative_rodeo/errors.rb

@@ -35,7 +35,7 @@ module DerivativeRodeo
     ##
     # Raised when AWS bucket does not exist or is not accessible by current permissions
     class BucketMissingError < Error
-      def initialize
+      def initialize(file_uri:)
         super("Bucket part missing #{file_uri}")
       end
     end

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -11,11 +11,21 @@ module DerivativeRodeo
     ##
     # The Base Generator defines the interface and common methods.
     #
+    # Fundamentally, they are about ensuring the files end up at the specified location, based on
+    # the given:
+    #
+    # - {#input_uris}
+    # - {#output_location_template}
+    # - {#preprocessed_location_template}
+    #
     # In extending a BaseGenerator you:
     #
     # - must assign an {.output_extension}
     # - must impliment a {#build_step} method
     # - may override {#with_each_requisite_location_and_tmp_file_path}
+    #
+    # {#generated_files} is "where the magic happens"
+    # rubocop:disable Metrics/ClassLength
     class BaseGenerator
       ##
       # @!group Class Attributes
@@ -26,9 +36,27 @@ module DerivativeRodeo
       class_attribute :output_extension
       # @!endgroup Class Attributes
 
-      attr_reader :input_uris,
-                  :output_location_template,
-                  :preprocessed_location_template
+      ##
+      # @!group Attributes
+      #
+      # The "original" files that we'll be processing (via {#generated_files})
+      # @return [Array<String>]
+      attr_reader :input_uris
+
+      ##
+      # The template that defines where we'll be writing the {#input_uris} (via {#generated_files})
+      # @return [String]
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
+      attr_reader :output_location_template
+
+      ##
+      # The template that defines where we might find existing processed files for the given
+      # {#input_uris} (via {#generated_files})
+      #
+      # @return [String, NilClass]
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
+      attr_reader :preprocessed_location_template
+      # @!endgroup Attributes
 
       ##
       # @param input_uris [Array<String>]
@@ -78,14 +106,28 @@ module DerivativeRodeo
       ##
       # @api public
       #
+      # Based on the {#input_uris} ensure that we have files at the given output location (as
+      # derived from the {#output_location_template}).  We ensure that by:
+      #
+      # - Checking if a file already exists at the output location
+      # - Copying a preprocessed file to the output location if a preprocessed file exists
+      # - Generating the file based on the input location
+      #
+      # @note This is the method where the magic happens!
+      #
       # @return [Array<StorageLocations::BaseLocation>]
       #
       # @see #build_step
       # @see #with_each_requisite_location_and_tmp_file_path
+      # rubocop:disable Metrics/MethodLength
       def generated_files
         # TODO: Examples please
         return @generated_files if defined?(@generated_files)
 
+        logger.info("Starting #{self.class}#generated_files with " \
+                    "input_uris: #{input_uris.inspect}, " \
+                    "output_location_template: #{output_location_template.inspect}, and " \
+                    "preprocessed_location_template: #{preprocessed_location_template.inspect}.")
         # As much as I would like to use map or returned values; given the implementations it's
         # better to explicitly require that; reducing downstream implementation headaches.
         #
@@ -97,15 +139,20 @@ module DerivativeRodeo
         # BaseLocation is like the Ruby `File` (Pathname) "File.exist?(path) :: location.exist?"
         # "file:///Users/jfriesen/.profile"
         with_each_requisite_location_and_tmp_file_path do |input_location, input_tmp_file_path|
-          generated_file = destination(input_location)
-          @generated_files << if generated_file.exist?
-                                generated_file
+          output_location = destination(input_location)
+          @generated_files << if output_location.exist?
+                                output_location
                               else
-                                build_step(input_location: input_location, output_location: generated_file, input_tmp_file_path: input_tmp_file_path)
+                                log_message = "#{self.class}#generated_files :: " \
+                                          "input_location file_uri #{input_location.file_uri} :: " \
+                                          "Generating output_location file_uri #{output_location.file_uri} via build_step."
+                                logger.info(log_message)
+                                build_step(input_location: input_location, output_location: output_location, input_tmp_file_path: input_tmp_file_path)
                               end
         end
         @generated_files
       end
+      # rubocop:enable Metrics/MethodLength
 
       ##
       # @return [Array<String>]
@@ -157,9 +204,13 @@ module DerivativeRodeo
       end
 
       ##
-      # Returns the location destination for the given :input_file.  The file at the location
-      # destination might exist or might not.  In the case of non-existence, then the {#build_step}
-      # will create the file.
+      # Returns the output location for the given :input_location.  The file at the location
+      # destination might exist or might not.  In the case where we have a
+      # {#preprocessed_location_template}, we'll also check the preprocessed location for the file,
+      # and if it exists there copy it to the target output location.
+      #
+      # In the case of non-existence, then the {#build_step} will create
+      # the file.
       #
       # @param input_location [StorageLocations::BaseLocation]
       #
@@ -167,20 +218,80 @@ module DerivativeRodeo
       #         {#output_location_template} or {#preprocessed_location_template}.
       #
       # @see [StorageLocations::BaseLocation#exist?]
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
       def destination(input_location)
-        output_location = input_location.derived_file_from(template: output_location_template)
+        output_location = input_location.derived_file_from(template: output_location_template, extension: output_extension)
+
+        if output_location.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found output_location file_uri #{output_location.file_uri}."
+          logger.info(log_message)
 
-        return output_location if output_location.exist?
-        return output_location unless preprocessed_location_template
+          return output_location
+        end
+
+        unless preprocessed_location_template
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "No preprocessed_location_template provided " \
+                        "nor does a file exist at output_location file_uri #{output_location.file_uri}; " \
+                        "moving on to generation via #{self.class}#build_step."
+          logger.info(log_message)
+
+          return output_location
+        end
+
+        template = derive_preprocessed_template_from(input_location: input_location, preprocessed_location_template: preprocessed_location_template)
 
-        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template)
+        preprocessed_location = input_location.derived_file_from(template: template, extension: output_extension)
         # We only want the location if it exists
-        return preprocessed_location if preprocessed_location&.exist?
+        if preprocessed_location.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found preprocessed_location file_uri #{preprocessed_location.file_uri}."
+          logger.info(log_message)
+
+          # Let's make sure we reap the fruits of the pre-processing; and don't worry that generator
+          # will also write some logs.
+          output_location = CopyGenerator.new(
+            input_uris: [preprocessed_location.file_uri],
+            output_location_template: output_location.file_uri
+          ).generated_files.first
+
+          return output_location
+        end
+
+        log_message = "#{self.class}#destination :: " \
+                      "input_location file_uri #{input_location.file_uri} :: " \
+                      "No file exists at preprocessed_location file_uri #{preprocessed_location.file_uri} " \
+                      "nor output_location file_uri #{output_location.file_uri}; " \
+                      "moving on to generation via #{self.class}#build_step."
+        logger.info(log_message)
 
         # NOTE: The file does not exist at the output_location; but we pass this information along so
         # that the #build_step knows where to write the file.
         output_location
       end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+
+      ##
+      # Some generators (e.g. {PdfSplitGenerator}) need to cooerce the location template based on
+      # the input location.  Most often, however, the given :preprocessed_location_template is
+      # adequate and would be the typical returned value.
+      #
+      # @param input_location [StorageLocations::BaseLocation]
+      # @param preprocessed_location_template [String]
+      #
+      # @return [String]
+      #
+      # rubocop:disable Lint/UnusedMethodArgument
+      def derive_preprocessed_template_from(input_location:, preprocessed_location_template:)
+        preprocessed_location_template
+      end
+      # rubocop:enable Lint/UnusedMethodArgument
 
       ##
       # A bit of indirection to create a common interface for running a shell command.
@@ -196,6 +307,7 @@ module DerivativeRodeo
         result
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end
 

data/lib/derivative_rodeo/generators/monochrome_generator.rb

@@ -5,7 +5,8 @@ module DerivativeRodeo
     ##
     # Take images an ensures that we have a monochrome derivative of those images.
     class MonochromeGenerator < BaseGenerator
-      # TODO: Can we assume a tiff?
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService for the interaction of the
+      #      magic ".mono" suffix
       self.output_extension = 'mono.tiff'
 
       ##

data/lib/derivative_rodeo/generators/pdf_split_generator.rb

@@ -66,7 +66,7 @@ module DerivativeRodeo
       #
       # @note There is relation to {Generators::BaseGenerator#destination} and this method.
       #
-      # @note The tail_glob is in relation to the {#image_file_basename_template}
+      # @note The tail_regexp is in relation to the {#image_file_basename_template}
       def existing_page_locations(input_location:)
         # See image_file_basename_template
         tail_regexp = %r{#{input_location.file_basename}--page-\d+\.#{output_extension}$}
@@ -76,12 +76,14 @@ module DerivativeRodeo
 
         return [] if preprocessed_location_template.blank?
 
-        input_location.derived_file_from(template: preprocessed_location_template).globbed_tail_loations(tail_regexp: tail_regexp)
+        input_location.derived_file_from(template: preprocessed_location_template).matching_locations_in_file_dir(tail_regexp: tail_regexp)
       end
 
       ##
       # @api public
       #
+      # @param splitter [#call]
+      #
       # Take the given PDF(s) and into one image per page.  Remember that the URL should account for
       # the page number.
       #
@@ -98,22 +100,27 @@ module DerivativeRodeo
       # @see BaseGenerator#with_each_requisite_location_and_tmp_file_path for further discussion
       #
       # rubocop:disable Metrics/MethodLength
-      def with_each_requisite_location_and_tmp_file_path
+      # rubocop:disable Metrics/AbcSize
+      def with_each_requisite_location_and_tmp_file_path(splitter: Services::PdfSplitter)
         input_files.each do |input_location|
           input_location.with_existing_tmp_path do |input_tmp_file_path|
             existing_locations = existing_page_locations(input_location: input_location)
 
             if existing_locations.count.positive?
-              existing_locations.each do |location|
+              logger.info("#{self.class}##{__method__} found #{existing_locations.count} file(s) at existing split location for #{input_location.file_uri.inspect}.")
+              existing_locations.each_with_index do |location, index|
+                logger.info("#{self.class}##{__method__} found ##{index} split file #{location.file_path.inspect} for #{input_location.file_uri.inspect}.")
                 yield(location, location.file_path)
               end
             else
+              logger.info("#{self.class}##{__method__} did not find at existing location split files for #{input_location.file_uri.inspect}.  Proceeding with #{splitter}.call")
               # We're going to need to create the files and "cast" them to locations.
-              Services::PdfSplitter.call(
+              splitter.call(
                 input_tmp_file_path,
                 image_extension: output_extension,
                 image_file_basename_template: image_file_basename_template(basename: input_location.file_basename)
-              ).each do |image_path|
+              ).each_with_index do |image_path, index|
+                logger.info("#{self.class}##{__method__} generated (via #{splitter}.call) ##{index} split file #{image_path.inspect} for #{input_location.file_uri.inspect}.")
                 image_location = StorageLocations::FileLocation.new("file://#{image_path}")
                 yield(image_location, image_path)
               end
@@ -121,7 +128,18 @@ module DerivativeRodeo
           end
         end
       end
+      # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/MethodLength
+
+      ##
+      # We're working with an input location with a filename basename of "123.ARCHIVAL--page-1.tiff"
+      # The :preprocessed_location_template, due to constraints, likely ends with the original PDF's
+      # filename (e.g. "123.ARCHIVAL.pdf")
+      #
+      # And since the template doesn't have a concept of page number, we introduce this kludge.
+      def derive_preprocessed_template_from(input_location:, preprocessed_location_template:)
+        File.join(File.dirname(preprocessed_location_template), input_location.file_name)
+      end
     end
   end
 end

data/lib/derivative_rodeo/generators/word_coordinates_generator.rb

@@ -33,6 +33,11 @@ module DerivativeRodeo
         File.open(path_to_coordinate, "w+") do |file|
           file.puts service.call(hocr_html).to_json
         end
+      rescue => e
+        message = "#{self.class}##{__method__} encountered `#{e.class}' error “#{e}” for path_to_hocr: #{path_to_hocr.inspect} and path_to_coordinate: #{path_to_coordinate.inspect}"
+        exception = RuntimeError.new(message)
+        exception.set_backtrace(e.backtrace)
+        raise exception
       end
     end
   end

data/lib/derivative_rodeo/services/convert_uri_via_template_service.rb

@@ -46,11 +46,12 @@ module DerivativeRodeo
       #     from_uris: ["file:///path1/A/file.pdf", "aws:///path2/B/file.pdf"],
       #     template: "file:///dest1/{{dir_parts[-1..-1]}}/{{ filename }}")
       #   => ["file:///dest1/A/file.pdf", "aws:///dest1/B/file.pdf"]
-      def self.call(from_uri:, template:, adapter: nil, separator: "/")
-        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator).call
+      def self.call(from_uri:, template:, adapter: nil, separator: "/", **options)
+        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator, **options).call
       end
 
-      def initialize(from_uri:, template:, adapter: nil, separator: "/")
+      # rubocop:disable Metrics/MethodLength
+      def initialize(from_uri:, template:, adapter: nil, separator: "/", **options)
         @from_uri = from_uri
         @template = template
         @adapter = adapter
@@ -60,12 +61,23 @@ module DerivativeRodeo
         @from_scheme, @path = uri.split("://")
         @parts = @path.split(separator)
         @dir_parts = @parts[0..-2]
-        @filename = @parts[-1]
-        @basename = File.basename(@filename, ".*")
-        @extension = File.extname(@filename)
+        @filename = options[:filename] || @parts[-1]
+        @basename = options[:basename] || File.basename(@filename, ".*")
+
+        ##
+        # HACK: Because the HocrGenerator has `.mono.tiff` and we are not interested in carrying
+        # forward the `.mono` suffix as that makes it hard to find the preprocessed word
+        # coordinates, alto, and plain text.  This ensures files derived from the .mono are findable
+        # in IIIF Print.
+        @basename = @basename.sub(/\.mono\z/, '')
+        @extension = options[:extension] || File.extname(@filename)
+        # When a generator specifies "same" we want to use the given file's extension
+        @extension = File.extname(@filename) if @extension == DerivativeRodeo::StorageLocations::SAME
+        @extension = ".#{@extension}" unless @extension.start_with?(".")
 
         @template_without_query, @template_query = template.split("?")
       end
+      # rubocop:enable Metrics/MethodLength
 
       def call
         to_uri = template_without_query.gsub(DIR_PARTS_REPLACEMENT_REGEXP) do |text|

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -46,6 +46,8 @@ module DerivativeRodeo
         delegate :config, to: DerivativeRodeo
       end
 
+      delegate :logger, to: DerivativeRodeo
+
       ##
       # @param location_name [String]
       #
@@ -101,10 +103,10 @@ module DerivativeRodeo
       # @param service [#call, Module<DerivativeRodeo::Services::ConvertUriViaTemplateService>]
       #
       # @return [StorageLocations::BaseLocation]
-      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService)
+      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService, **options)
         # HACK: Ensuring that we have the correct scheme.  Maybe this is a hack?
         from_uri = "#{scheme}://#{from_uri}" unless from_uri.start_with?("#{scheme}://")
-        to_uri = service.call(from_uri: from_uri, template: template, adapter: self)
+        to_uri = service.call(from_uri: from_uri, template: template, adapter: self, **options)
         new(to_uri)
       end
 
@@ -203,9 +205,9 @@ module DerivativeRodeo
       # @return [StorageLocations::BaseLocation]
       #
       # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
-      def derived_file_from(template:)
+      def derived_file_from(template:, **options)
         klass = DerivativeRodeo::StorageLocations::BaseLocation.load_location(template)
-        klass.build(from_uri: file_path, template: template)
+        klass.build(from_uri: file_path, template: template, **options)
       end
 
       ##
@@ -231,6 +233,7 @@ module DerivativeRodeo
       def with_new_extension(extension)
         return file_path if extension == StorageLocations::SAME
 
+        # NOTE: May need to revisit this
         "#{file_path.split('.')[0]}.#{extension}"
       end
 

data/lib/derivative_rodeo/storage_locations/file_location.rb

@@ -42,6 +42,9 @@ module DerivativeRodeo
       #
       # @param tail_regexp [Regexp]
       def matching_locations_in_file_dir(tail_regexp:)
+        logger.debug("#{self.class}##{__method__} searching for matching files in " \
+                    "file_dir: #{file_dir.inspect} " \
+                    "with tail_regexp: #{tail_regexp.inspect}.")
         Dir.glob(File.join(file_dir, "*")).each_with_object([]) do |filename, accumulator|
           accumulator << derived_file_from(template: "file://#{filename}") if tail_regexp.match(filename)
         end

data/lib/derivative_rodeo/storage_locations/s3_location.rb

@@ -73,7 +73,10 @@ module DerivativeRodeo
       def matching_locations_in_file_dir(tail_regexp:)
         uri = URI.parse(file_uri)
         scheme_and_host = "#{uri.scheme}://#{uri.host}"
-
+        logger.debug("#{self.class}##{__method__} searching for matching files for " \
+                    "scheme_and_host: #{scheme_and_host.inspect} " \
+                    "file_dir: #{file_dir.inspect} " \
+                    "with tail_regexp: #{tail_regexp.inspect}.")
         bucket.objects(prefix: file_dir).each_with_object([]) do |object, accumulator|
           if tail_regexp.match(object.key)
             template = File.join(scheme_and_host, object.key)
@@ -120,7 +123,7 @@ module DerivativeRodeo
       def bucket_name
         @bucket_name ||= file_uri.match(%r{s3://(.+)\.s3})&.[](1)
       rescue StandardError
-        raise Errors::BucketMissingError
+        raise Errors::BucketMissingError.new(file_uri: file_uri)
       end
 
       # @see .use_actual_s3_bucket

data/lib/derivative_rodeo/storage_locations/sqs_location.rb

@@ -13,6 +13,8 @@ module DerivativeRodeo
     # Location to download and upload files to Sqs
     # It uploads a file_uri to the queue, not the contents of that file
     # reading from the queue is not currently implemented
+    #
+    # rubocop:disable Metrics/ClassLength
     class SqsLocation < BaseLocation
       ##
       # @!group Class Attributes
@@ -85,11 +87,14 @@ module DerivativeRodeo
         batch = []
         Dir.glob("#{File.dirname(tmp_file_path)}/**/**").each.with_index do |fp, i|
           batch << { id: SecureRandom.uuid, message_body: output_json("file://#{fp}") }
-          if (i % batch_size).zero?
+          if (i + 1 % batch_size).zero?
             add_batch(messages: batch)
             batch = []
           end
         end
+
+        # Ensure we're flushing the batched up queue as part of completing the write.
+        add_batch(messages: batch) if batch.present?
         file_uri
       end
 
@@ -181,6 +186,7 @@ module DerivativeRodeo
       end
 
       def output_json(uri)
+        # TODO: Add ability to handle a pre-process-template given to an SQS, and pass that along to the generator when applicable.
         key = DerivativeRodeo::Services::ConvertUriViaTemplateService.call(from_uri: uri, template: template, adapter: self)
         { key => [template] }.to_json
       end
@@ -201,5 +207,6 @@ module DerivativeRodeo
         @file_uri_parts
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -337,7 +337,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: An ETL Ecosystem for Derivative Processing.