derivative-rodeo 0.4.2 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62872d16bfd5d73940f87d5c09f61f2a88ee67414f51905ce503f411b9b2fb37
-  data.tar.gz: 742d63ca02418b3453824655738e25b47d3cca918f030e2fb5db4c997d52e945
+  metadata.gz: d2dc233be659f66709737ebdebc8ac6a69802f74bce6ac450e5eabb383ea4e54
+  data.tar.gz: 7a72ce66e69f827374b6f8e72767aca12638c71a9e5b2c63be09bbca08c12640
 SHA512:
-  metadata.gz: e43b94745f35474edf4b463cd11b8c7d7bb29391f443c7ef2a9e84966d969aa6cf7c205a92c32a03452ba28c490c790ccd05d6569fd9db59d8c119b7e38f1dde
-  data.tar.gz: '07962c3175aed6d77295e473ad8462d0ae634c931a2f3f5bc75be1195977dded3cec013448c9d2fe2653fa83b9d178dd3513aaa1eda8eeafe9da539e3dbf06b0'
+  metadata.gz: de5ff8fc29943f5dc52b49c7741ae4a742793439dcca97ea94259d044cc15870dd19ced0911d9305901757e838a8ecb401eb81744c4f0529eef9607534d5dcb8
+  data.tar.gz: bec393ace49d0e6689a07d335017987fa6e6edf9aae632159b87150dc2d4a0720441d64f6c6ce48ceb7c4498df3b6ada40b010cd0d439cb76cb071bd65c2c19d

data/lib/derivative_rodeo/errors.rb

@@ -35,7 +35,7 @@ module DerivativeRodeo
     ##
     # Raised when AWS bucket does not exist or is not accessible by current permissions
     class BucketMissingError < Error
-      def initialize
+      def initialize(file_uri:)
         super("Bucket part missing #{file_uri}")
       end
     end
@@ -43,6 +43,9 @@ module DerivativeRodeo
     ##
     # Raised when trying to write a tmp file that does not exist
     class FileMissingError < Error
+      def self.with_info(**info)
+        new(info.inspect)
+      end
     end
 
     ##

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -11,11 +11,21 @@ module DerivativeRodeo
     ##
     # The Base Generator defines the interface and common methods.
     #
+    # Fundamentally, they are about ensuring the files end up at the specified location, based on
+    # the given:
+    #
+    # - {#input_uris}
+    # - {#output_location_template}
+    # - {#preprocessed_location_template}
+    #
     # In extending a BaseGenerator you:
     #
     # - must assign an {.output_extension}
     # - must impliment a {#build_step} method
     # - may override {#with_each_requisite_location_and_tmp_file_path}
+    #
+    # {#generated_files} is "where the magic happens"
+    # rubocop:disable Metrics/ClassLength
     class BaseGenerator
       ##
       # @!group Class Attributes
@@ -26,9 +36,27 @@ module DerivativeRodeo
       class_attribute :output_extension
       # @!endgroup Class Attributes
 
-      attr_reader :input_uris,
-                  :output_location_template,
-                  :preprocessed_location_template
+      ##
+      # @!group Attributes
+      #
+      # The "original" files that we'll be processing (via {#generated_files})
+      # @return [Array<String>]
+      attr_reader :input_uris
+
+      ##
+      # The template that defines where we'll be writing the {#input_uris} (via {#generated_files})
+      # @return [String]
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
+      attr_reader :output_location_template
+
+      ##
+      # The template that defines where we might find existing processed files for the given
+      # {#input_uris} (via {#generated_files})
+      #
+      # @return [String, NilClass]
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
+      attr_reader :preprocessed_location_template
+      # @!endgroup Attributes
 
       ##
       # @param input_uris [Array<String>]
@@ -78,6 +106,15 @@ module DerivativeRodeo
       ##
       # @api public
       #
+      # Based on the {#input_uris} ensure that we have files at the given output location (as
+      # derived from the {#output_location_template}).  We ensure that by:
+      #
+      # - Checking if a file already exists at the output location
+      # - Copying a preprocessed file to the output location if a preprocessed file exists
+      # - Generating the file based on the input location
+      #
+      # @note This is the method where the magic happens!
+      #
       # @return [Array<StorageLocations::BaseLocation>]
       #
       # @see #build_step
@@ -87,6 +124,10 @@ module DerivativeRodeo
         # TODO: Examples please
         return @generated_files if defined?(@generated_files)
 
+        logger.info("Starting #{self.class}#generated_files with " \
+                    "input_uris: #{input_uris.inspect}, " \
+                    "output_location_template: #{output_location_template.inspect}, and " \
+                    "preprocessed_location_template: #{preprocessed_location_template.inspect}.")
         # As much as I would like to use map or returned values; given the implementations it's
         # better to explicitly require that; reducing downstream implementation headaches.
         #
@@ -98,15 +139,15 @@ module DerivativeRodeo
         # BaseLocation is like the Ruby `File` (Pathname) "File.exist?(path) :: location.exist?"
         # "file:///Users/jfriesen/.profile"
         with_each_requisite_location_and_tmp_file_path do |input_location, input_tmp_file_path|
-          generated_file = destination(input_location)
-          @generated_files << if generated_file.exist?
-                                generated_file
+          output_location = destination(input_location)
+          @generated_files << if output_location.exist?
+                                output_location
                               else
                                 log_message = "#{self.class}#generated_files :: " \
                                           "input_location file_uri #{input_location.file_uri} :: " \
-                                          "Generating output_location file_uri #{generated_file.file_uri} via build_step."
+                                          "Generating output_location file_uri #{output_location.file_uri} via build_step."
                                 logger.info(log_message)
-                                build_step(input_location: input_location, output_location: generated_file, input_tmp_file_path: input_tmp_file_path)
+                                build_step(input_location: input_location, output_location: output_location, input_tmp_file_path: input_tmp_file_path)
                               end
         end
         @generated_files
@@ -163,9 +204,13 @@ module DerivativeRodeo
       end
 
       ##
-      # Returns the location destination for the given :input_file.  The file at the location
-      # destination might exist or might not.  In the case of non-existence, then the {#build_step}
-      # will create the file.
+      # Returns the output location for the given :input_location.  The file at the location
+      # destination might exist or might not.  In the case where we have a
+      # {#preprocessed_location_template}, we'll also check the preprocessed location for the file,
+      # and if it exists there copy it to the target output location.
+      #
+      # In the case of non-existence, then the {#build_step} will create
+      # the file.
       #
       # @param input_location [StorageLocations::BaseLocation]
       #
@@ -191,22 +236,31 @@ module DerivativeRodeo
           log_message = "#{self.class}#destination :: " \
                         "input_location file_uri #{input_location.file_uri} :: " \
                         "No preprocessed_location_template provided " \
-                        "nor does a file exist at output_location file_uri #{output_location.file_uri};" \
-                        " moving on to generation via #{self.class}#build_step."
+                        "nor does a file exist at output_location file_uri #{output_location.file_uri}; " \
+                        "moving on to generation via #{self.class}#build_step."
           logger.info(log_message)
 
           return output_location
         end
 
-        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template, extension: output_extension)
+        template = derive_preprocessed_template_from(input_location: input_location, preprocessed_location_template: preprocessed_location_template)
+
+        preprocessed_location = input_location.derived_file_from(template: template, extension: output_extension)
         # We only want the location if it exists
-        if preprocessed_location&.exist?
+        if preprocessed_location.exist?
           log_message = "#{self.class}#destination :: " \
                         "input_location file_uri #{input_location.file_uri} :: " \
-                        "Found preprocessed_location file_uri #{output_location.file_uri}."
+                        "Found preprocessed_location file_uri #{preprocessed_location.file_uri}."
           logger.info(log_message)
 
-          return preprocessed_location
+          # Let's make sure we reap the fruits of the pre-processing; and don't worry that generator
+          # will also write some logs.
+          output_location = CopyGenerator.new(
+            input_uris: [preprocessed_location.file_uri],
+            output_location_template: output_location.file_uri
+          ).generated_files.first
+
+          return output_location
         end
 
         log_message = "#{self.class}#destination :: " \
@@ -223,6 +277,22 @@ module DerivativeRodeo
       # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/MethodLength
 
+      ##
+      # Some generators (e.g. {PdfSplitGenerator}) need to cooerce the location template based on
+      # the input location.  Most often, however, the given :preprocessed_location_template is
+      # adequate and would be the typical returned value.
+      #
+      # @param input_location [StorageLocations::BaseLocation]
+      # @param preprocessed_location_template [String]
+      #
+      # @return [String]
+      #
+      # rubocop:disable Lint/UnusedMethodArgument
+      def derive_preprocessed_template_from(input_location:, preprocessed_location_template:)
+        preprocessed_location_template
+      end
+      # rubocop:enable Lint/UnusedMethodArgument
+
       ##
       # A bit of indirection to create a common interface for running a shell command.
       #
@@ -237,6 +307,7 @@ module DerivativeRodeo
         result
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end
 

data/lib/derivative_rodeo/generators/pdf_split_generator.rb

@@ -66,7 +66,7 @@ module DerivativeRodeo
       #
       # @note There is relation to {Generators::BaseGenerator#destination} and this method.
       #
-      # @note The tail_glob is in relation to the {#image_file_basename_template}
+      # @note The tail_regexp is in relation to the {#image_file_basename_template}
       def existing_page_locations(input_location:)
         # See image_file_basename_template
         tail_regexp = %r{#{input_location.file_basename}--page-\d+\.#{output_extension}$}
@@ -76,12 +76,14 @@ module DerivativeRodeo
 
         return [] if preprocessed_location_template.blank?
 
-        input_location.derived_file_from(template: preprocessed_location_template).globbed_tail_loations(tail_regexp: tail_regexp)
+        input_location.derived_file_from(template: preprocessed_location_template).matching_locations_in_file_dir(tail_regexp: tail_regexp)
       end
 
       ##
       # @api public
       #
+      # @param splitter [#call]
+      #
       # Take the given PDF(s) and into one image per page.  Remember that the URL should account for
       # the page number.
       #
@@ -98,22 +100,27 @@ module DerivativeRodeo
       # @see BaseGenerator#with_each_requisite_location_and_tmp_file_path for further discussion
       #
       # rubocop:disable Metrics/MethodLength
-      def with_each_requisite_location_and_tmp_file_path
+      # rubocop:disable Metrics/AbcSize
+      def with_each_requisite_location_and_tmp_file_path(splitter: Services::PdfSplitter)
         input_files.each do |input_location|
           input_location.with_existing_tmp_path do |input_tmp_file_path|
             existing_locations = existing_page_locations(input_location: input_location)
 
             if existing_locations.count.positive?
-              existing_locations.each do |location|
+              logger.info("#{self.class}##{__method__} found #{existing_locations.count} file(s) at existing split location for #{input_location.file_uri.inspect}.")
+              existing_locations.each_with_index do |location, index|
+                logger.info("#{self.class}##{__method__} found ##{index} split file #{location.file_path.inspect} for #{input_location.file_uri.inspect}.")
                 yield(location, location.file_path)
               end
             else
+              logger.info("#{self.class}##{__method__} did not find at existing location split files for #{input_location.file_uri.inspect}.  Proceeding with #{splitter}.call")
               # We're going to need to create the files and "cast" them to locations.
-              Services::PdfSplitter.call(
+              splitter.call(
                 input_tmp_file_path,
                 image_extension: output_extension,
                 image_file_basename_template: image_file_basename_template(basename: input_location.file_basename)
-              ).each do |image_path|
+              ).each_with_index do |image_path, index|
+                logger.info("#{self.class}##{__method__} generated (via #{splitter}.call) ##{index} split file #{image_path.inspect} for #{input_location.file_uri.inspect}.")
                 image_location = StorageLocations::FileLocation.new("file://#{image_path}")
                 yield(image_location, image_path)
               end
@@ -121,7 +128,18 @@ module DerivativeRodeo
           end
         end
       end
+      # rubocop:enable Metrics/AbcSize
       # rubocop:enable Metrics/MethodLength
+
+      ##
+      # We're working with an input location with a filename basename of "123.ARCHIVAL--page-1.tiff"
+      # The :preprocessed_location_template, due to constraints, likely ends with the original PDF's
+      # filename (e.g. "123.ARCHIVAL.pdf")
+      #
+      # And since the template doesn't have a concept of page number, we introduce this kludge.
+      def derive_preprocessed_template_from(input_location:, preprocessed_location_template:)
+        File.join(File.dirname(preprocessed_location_template), input_location.file_name)
+      end
     end
   end
 end

data/lib/derivative_rodeo/generators/word_coordinates_generator.rb

@@ -33,6 +33,11 @@ module DerivativeRodeo
         File.open(path_to_coordinate, "w+") do |file|
           file.puts service.call(hocr_html).to_json
         end
+      rescue => e
+        message = "#{self.class}##{__method__} encountered `#{e.class}' error “#{e}” for path_to_hocr: #{path_to_hocr.inspect} and path_to_coordinate: #{path_to_coordinate.inspect}"
+        exception = RuntimeError.new(message)
+        exception.set_backtrace(e.backtrace)
+        raise exception
       end
     end
   end

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -46,6 +46,8 @@ module DerivativeRodeo
         delegate :config, to: DerivativeRodeo
       end
 
+      delegate :logger, to: DerivativeRodeo
+
       ##
       # @param location_name [String]
       #

data/lib/derivative_rodeo/storage_locations/concerns/download_concern.rb

@@ -23,8 +23,8 @@ module DerivativeRodeo
       delegate :config, to: DerivativeRodeo
 
       def with_existing_tmp_path(&block)
-        with_tmp_path(lambda { |_file_path, tmp_file_path, exist|
-          raise Errors::FileMissingError unless exist
+        with_tmp_path(lambda { |file_path, tmp_file_path, exist|
+          raise Errors::FileMissingError.with_info(method: __method__, context: self, file_path: file_path, tmp_file_path: tmp_file_path) unless exist
 
           response = get(file_uri)
           File.open(tmp_file_path, 'wb') { |fp| fp.write(response.body) }

data/lib/derivative_rodeo/storage_locations/file_location.rb

@@ -16,7 +16,7 @@ module DerivativeRodeo
 
       def with_existing_tmp_path(&block)
         with_tmp_path(lambda { |file_path, tmp_file_path, exist|
-          raise Errors::FileMissingError unless exist
+          raise Errors::FileMissingError.with_info(method: __method__, context: self, file_path: file_path, tmp_file_path: tmp_file_path) unless exist
 
           FileUtils.cp(file_path, tmp_file_path)
         }, &block)
@@ -42,6 +42,9 @@ module DerivativeRodeo
       #
       # @param tail_regexp [Regexp]
       def matching_locations_in_file_dir(tail_regexp:)
+        logger.debug("#{self.class}##{__method__} searching for matching files in " \
+                    "file_dir: #{file_dir.inspect} " \
+                    "with tail_regexp: #{tail_regexp.inspect}.")
         Dir.glob(File.join(file_dir, "*")).each_with_object([]) do |filename, accumulator|
           accumulator << derived_file_from(template: "file://#{filename}") if tail_regexp.match(filename)
         end

data/lib/derivative_rodeo/storage_locations/s3_location.rb

@@ -47,7 +47,7 @@ module DerivativeRodeo
       # @return [String] the path to the tmp file
       def with_existing_tmp_path(&block)
         with_tmp_path(lambda { |file_path, tmp_file_path, exist|
-                        raise Errors::FileMissingError unless exist
+                        raise Errors::FileMissingError.with_info(method: __method__, context: self, file_path: file_path, tmp_file_path: tmp_file_path) unless exist
                         obj = bucket.object(file_path)
                         obj.download_file(tmp_file_path)
                       }, &block)
@@ -73,7 +73,10 @@ module DerivativeRodeo
       def matching_locations_in_file_dir(tail_regexp:)
         uri = URI.parse(file_uri)
         scheme_and_host = "#{uri.scheme}://#{uri.host}"
-
+        logger.debug("#{self.class}##{__method__} searching for matching files for " \
+                    "scheme_and_host: #{scheme_and_host.inspect} " \
+                    "file_dir: #{file_dir.inspect} " \
+                    "with tail_regexp: #{tail_regexp.inspect}.")
         bucket.objects(prefix: file_dir).each_with_object([]) do |object, accumulator|
           if tail_regexp.match(object.key)
             template = File.join(scheme_and_host, object.key)
@@ -120,7 +123,7 @@ module DerivativeRodeo
       def bucket_name
         @bucket_name ||= file_uri.match(%r{s3://(.+)\.s3})&.[](1)
       rescue StandardError
-        raise Errors::BucketMissingError
+        raise Errors::BucketMissingError.new(file_uri: file_uri)
       end
 
       # @see .use_actual_s3_bucket

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.4.2'
+  VERSION = '0.5.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.1
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-10 00:00:00.000000000 Z
+date: 2023-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport