derivative-rodeo 0.5.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d2dc233be659f66709737ebdebc8ac6a69802f74bce6ac450e5eabb383ea4e54
-  data.tar.gz: 7a72ce66e69f827374b6f8e72767aca12638c71a9e5b2c63be09bbca08c12640
+  metadata.gz: 21db0b8b76cb0eadec2dab8848e7a7ac6240895eee96b5a247ed627fc682ac35
+  data.tar.gz: ab68faec3b4c8775f9243ae53090bb89191e4ea57c6ef2e8e8f40c1d4fa73f16
 SHA512:
-  metadata.gz: de5ff8fc29943f5dc52b49c7741ae4a742793439dcca97ea94259d044cc15870dd19ced0911d9305901757e838a8ecb401eb81744c4f0529eef9607534d5dcb8
-  data.tar.gz: bec393ace49d0e6689a07d335017987fa6e6edf9aae632159b87150dc2d4a0720441d64f6c6ce48ceb7c4498df3b6ada40b010cd0d439cb76cb071bd65c2c19d
+  metadata.gz: 6605936719a33bd7f8bbb6d945b1aa1d87e6b08a53206890af462668a975f519f5a29235e963404d1c159d8486af2db1800015da30be93a9493938e4d69c8c09
+  data.tar.gz: 3ad8cfa0dd71d4167439277c958ccca3682dc87c095afcbcbeb45afe2cf399c70b19d887fec2faaf763fc54a1d2d643957c72124768db8efad76f67c63624449

data/README.md

@@ -17,6 +17,7 @@
       - [Registered Generators](#registered-generators)
     - [Storage Locations](#storage-locations)
       - [Supported Storage Locations](#supported-storage-locations)
+      - [Templates](#templates)
   - [Development](#development)
     - [Logging in Test Environment](#logging-in-test-environment)
   - [Contributing](#contributing)
@@ -34,9 +35,11 @@ The `DerivativeRodeo` "moves" files from one storage location (e.g. *input*) to
 
 ## Process Life Cycle
 
-In the case of a *input* storage location, we expect that the underlying file pointed at by the *input* storage location exists.  After all we can't move what we don't have.
+In the case of a *input* storage location (e.g. `input_location`), we expect that the underlying file pointed at by the *input* storage location exists.  After all we can't move what we don't have.
 
-In the case of a *output* storage location, we expect that the underlying file will exist after the generator has completed.  The *output* storage location *could* already exist or we might need to generate the file for the *output* location.
+In the case of a *output* storage location (e.g. `output_location`), we expect that the underlying file will exist after the generator has completed.  The *output* storage location *could* already exist or we might need to generate the file for the *output* location.
+
+There is also the concept of the *pre\_processed* storage location; when the *pre\_processed* storage location exists for the given input, copy that *pre\_processed* file to the *output* location.  And skip running the derivative generator on the *input* storage location.  In other words, if we've already done the derivation elsewhere, use that.
 
 During the generator's process, we need to have a working copy of both the *input* and *output* file.  This is done by creating a temporary file.
 
@@ -224,6 +227,24 @@ Storage locations follow a [URI pattern](https://en.wikipedia.org/wiki/Uniform_R
 - `s3://` :: <abbr title="Amazon Web Service">AWS</abbr>’s <abbr title="Simple Storage Service">S3</abbr> storage system
 - `sqs://` :: <abbr title="Amazon Web Service">AWS</abbr>’s <abbr title="Simple Queue Service">SQS</abbr>
 
+#### Templates
+
+Throughout the code you'll see reference to the following concepts:
+
+- `input_location_template`
+- `output_location_template`
+- `preprocessed_location_template`
+
+In [Process Life Cycle](#process-life-cycle) we discussed the `input_location`, `output_location`, and `preprocessed_location`.  The concept of the template provides a flexibility in mapping a location to another location
+
+Examples of mapping one file path to another are:
+
+- I want to copy `https://hello.com/world/GUID/file.jpg` to `file:///tmp/GUID/file.jpg`.
+- I want to transform `file:///tmp/GUID/file.jpg` to `file:///tmp/GUID/file.hocr`; that is run OCR on an image and write a `.hocr` file.
+- I want to use the `file:///tmp/GUID/file.hocr` to generate a `file:///tmp/GUID/file.coordinates.json`; that is convert the HOCR file to a coordinates.json file.
+
+See [DerivativeRodeo::Service::ConvertUriViaTemplateService](./lib/derivative_rodeo/services/convert_uri_via_template_service.rb) for more details.
+
 ## Development
 
 - Checkout the repository: `git clone https://github.com/scientist-softserv/derivative_rodeo`

data/lib/derivative_rodeo/generators/alto_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../services/extract_word_coordinates_from_hocr_sgml_service'
+require_relative 'hocr_generator'
 
 module DerivativeRodeo
   module Generators
@@ -13,6 +14,8 @@ module DerivativeRodeo
 
       class_attribute :service, default: Services::ExtractWordCoordinatesFromHocrSgmlService
 
+      include HocrGenerator::RequiresExistingFile
+
       ##
       # @param output_location [StorageLocations::BaseLocation]
       # @param input_tmp_file_path [String] the location of the file that we can use for processing.

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -187,6 +187,7 @@ module DerivativeRodeo
       #
       # @see Generators::HocrGenerator
       # @see Generators::PdfSplitGenerator
+      # @see Services::ConvertUriViaTemplateService.coerce_pre_requisite_template_from
       def with_each_requisite_location_and_tmp_file_path
         input_files.each do |input_location|
           input_location.with_existing_tmp_path do |tmp_file_path|

data/lib/derivative_rodeo/generators/hocr_generator.rb

@@ -65,7 +65,8 @@ module DerivativeRodeo
       #
       # @see BaseGenerator#with_each_requisite_location_and_tmp_file_path for further discussion
       def with_each_requisite_location_and_tmp_file_path(builder: MonochromeGenerator)
-        mono_location_template = output_location_template.gsub(self.class.output_extension, builder.output_extension)
+        mono_location_template = Services::ConvertUriViaTemplateService.coerce_pre_requisite_template_from(template: output_location_template)
+
         requisite_files ||= builder.new(input_uris: input_uris, output_location_template: mono_location_template).generated_files
         requisite_files.each do |input_location|
           input_location.with_existing_tmp_path do |tmp_file_path|
@@ -107,6 +108,32 @@ module DerivativeRodeo
         # TODO: capture output in case of exceptions; perhaps delegate that to the #run method.
         run(cmd)
       end
+
+      ##
+      # A mixin for generators that rely on hocr files.
+      #
+      # @see #with_each_requisite_location_and_tmp_file_path
+      module RequiresExistingFile
+        ##
+        # @param builder [Class, #generated_files]
+        #
+        # When a generator depends on a hocr file, this method will ensure that we have the requisite
+        # hocr file.
+        #
+        # @yieldparam file [StorageLocations::BaseLocation]
+        # @yieldparam tmp_path [String]
+        #
+        # @see BaseGenerator#with_each_requisite_location_and_tmp_file_path for further discussion
+        def with_each_requisite_location_and_tmp_file_path(builder: HocrGenerator)
+          prereq_output_location_template = Services::ConvertUriViaTemplateService.coerce_pre_requisite_template_from(template: output_location_template)
+          requisite_files ||= builder.new(input_uris: input_uris, output_location_template: prereq_output_location_template).generated_files
+          requisite_files.each do |input_location|
+            input_location.with_existing_tmp_path do |tmp_file_path|
+              yield(input_location, tmp_file_path)
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/derivative_rodeo/generators/plain_text_generator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../services/extract_word_coordinates_from_hocr_sgml_service'
+require_relative 'hocr_generator'
 
 module DerivativeRodeo
   module Generators
@@ -13,6 +14,8 @@ module DerivativeRodeo
 
       class_attribute :service, default: Services::ExtractWordCoordinatesFromHocrSgmlService
 
+      include HocrGenerator::RequiresExistingFile
+
       ##
       # @param output_location [StorageLocations::BaseLocation]
       # @param input_tmp_file_path [String] the location of the file that we can use for processing.

data/lib/derivative_rodeo/generators/word_coordinates_generator.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative 'hocr_generator'
 module DerivativeRodeo
   module Generators
     ##
@@ -9,6 +10,8 @@ module DerivativeRodeo
     class WordCoordinatesGenerator < BaseGenerator
       self.output_extension = "coordinates.json"
 
+      include HocrGenerator::RequiresExistingFile
+
       ##
       # @param output_location [StorageLocations::BaseLocation]
       # @param input_tmp_file_path [String] the location of the file that we can use for processing.

data/lib/derivative_rodeo/services/convert_uri_via_template_service.rb

@@ -7,6 +7,7 @@ module DerivativeRodeo
     # A service to convert an array of :from_uris to :to_uris via a :template.
     #
     # @see .call
+    # @see .coerce_pre_requisite_template_from
     class ConvertUriViaTemplateService
       DIR_PARTS_REPLACEMENT_REGEXP = %r{\{\{\s*dir_parts\[(?<left>\-?\d+)\.\.(?<right>\-?\d+)\]\s*\}\}}.freeze
       FILENAME_REPLACEMENT_REGEXP = %r{\{\{\s*filename\s*\}\}}.freeze
@@ -16,6 +17,15 @@ module DerivativeRodeo
       SCHEME_FOR_URI_REGEXP = %r{^(?<from_scheme>[^:]+)://}.freeze
       attr_accessor :from_uri, :template, :adapter, :separator, :uri, :from_scheme, :path, :parts, :dir_parts, :filename, :basename, :extension, :template_without_query, :template_query
 
+      ##
+      # @!group Class Attributes
+      #
+      # @!attribute separator [r|w]
+      #   @return [String] the directory seperator character; default: "/"
+      class_attribute :separator, default: '/', instance_accessor: false
+      # @!endgroup Class Attributes
+      ##
+
       ##
       # Convert the given :from_uris to a different list of uris based on the given :template.
       #
@@ -32,7 +42,7 @@ module DerivativeRodeo
       # @param from_uri [String] Of the form "scheme://dir/parts/basename.extension"
       # @param template [String] Another URI that may contain path_parts or scheme template values.
       # @param adapter [StorageLocations::Location]
-      # @param separator [String]
+      # @param options [Hash<Symbol, Object>]
       #
       # @return [String]
       #
@@ -46,16 +56,31 @@ module DerivativeRodeo
       #     from_uris: ["file:///path1/A/file.pdf", "aws:///path2/B/file.pdf"],
       #     template: "file:///dest1/{{dir_parts[-1..-1]}}/{{ filename }}")
       #   => ["file:///dest1/A/file.pdf", "aws:///dest1/B/file.pdf"]
-      def self.call(from_uri:, template:, adapter: nil, separator: "/", **options)
-        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator, **options).call
+      def self.call(from_uri:, template:, adapter: nil, **options)
+        new(from_uri: from_uri, template: template, adapter: adapter, **options).call
+      end
+
+      ##
+      # There are generators that have requisite files necessary for processing.
+      #
+      # For example, before we run `tesseract` on an image, we would like to make sure it is
+      # monochrome.  Hence the interplay between the {Generators::HocrGenerator} and the
+      # {Generators::MonochromeGenerator}.
+      #
+      # @param template [String]
+      # @return [String]
+      #
+      # @see Generators::BaseGenerator#with_each_requisite_location_and_tmp_file_path
+      def self.coerce_pre_requisite_template_from(template:)
+        template.split(separator)[0..-2].join(separator) + "#{separator}{{ basename }}{{ extension }}"
       end
 
       # rubocop:disable Metrics/MethodLength
-      def initialize(from_uri:, template:, adapter: nil, separator: "/", **options)
+      def initialize(from_uri:, template:, adapter: nil, **options)
         @from_uri = from_uri
         @template = template
         @adapter = adapter
-        @separator = separator
+        @separator = options.fetch(:separator) { self.class.separator }
 
         @uri, _query = from_uri.split("?")
         @from_scheme, @path = uri.split("://")

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -46,7 +46,7 @@ module DerivativeRodeo
         delegate :config, to: DerivativeRodeo
       end
 
-      delegate :logger, to: DerivativeRodeo
+      delegate :config, :logger, to: DerivativeRodeo
 
       ##
       # @param location_name [String]
@@ -123,16 +123,14 @@ module DerivativeRodeo
       ##
       # @param file_uri [String] a URI to the file's location; this is **not** a templated URI (as
       #        described in {DerivativeRodeo::Services::ConvertUriViaTemplateService}
-      # @param config [DerivativeRodeo::Configuration]
-      def initialize(file_uri, config: DerivativeRodeo.config)
+      def initialize(file_uri)
         @file_uri = file_uri
-        @config = config
       end
 
       attr_accessor :tmp_file_path
       private :tmp_file_path=, :tmp_file_path
 
-      attr_reader :config, :file_uri
+      attr_reader :file_uri
 
       ##
       # @param auto_write_file [Boolean] Provided as a testing helper method.

data/lib/derivative_rodeo/storage_locations/concerns/download_concern.rb

@@ -20,7 +20,7 @@ module DerivativeRodeo
         end
       end
 
-      delegate :config, to: DerivativeRodeo
+      delegate :logger, to: DerivativeRodeo
 
       def with_existing_tmp_path(&block)
         with_tmp_path(lambda { |file_path, tmp_file_path, exist|
@@ -44,9 +44,9 @@ module DerivativeRodeo
       #
       # @return [String]
       def get(url)
-        HTTParty.get(url, logger: config.logger)
+        HTTParty.get(url, logger: logger)
       rescue => e
-        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
         raise e
       end
 
@@ -55,11 +55,24 @@ module DerivativeRodeo
       # @return [FalseClass] when the URL's head request is not successful or we've exhausted our
       #         remaining redirects.
       def exist?
-        HTTParty.head(file_uri, logger: config.logger)
+        HTTParty.head(file_uri, logger: logger)
       rescue => e
-        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
         false
       end
+
+      ##
+      # @param tail_regexp [Regex] the file pattern that we're looking to find; but due to the
+      #        nature of this location adapter, it won't matter.
+      # @return [Array] always returns an empty array.
+      #
+      # @see S3Location#matching_locations_in_file_dir
+      # @see FileLocation#matching_locations_in_file_dir
+      def matching_locations_in_file_dir(tail_regexp:)
+        logger.info("#{self.class}##{__method__} for file_uri: #{file_uri.inspect}, tail_regexp: #{tail_regexp} will always return an empty array.  This is the nature of the #{self.class} location.")
+
+        []
+      end
     end
   end
 end

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.5.1'
+  VERSION = '0.5.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-06 00:00:00.000000000 Z
+date: 2023-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport