derivative-rodeo 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc2eed3e32c7a4558d55e9d530b6790a5b876dcdfc4ced421cfa4894aa977d44
-  data.tar.gz: 6e16e4bd7b9d38a1a19b1768a5cdb021c6aa946287f430c6a6c62fa26a215ca6
+  metadata.gz: 62872d16bfd5d73940f87d5c09f61f2a88ee67414f51905ce503f411b9b2fb37
+  data.tar.gz: 742d63ca02418b3453824655738e25b47d3cca918f030e2fb5db4c997d52e945
 SHA512:
-  metadata.gz: 0ac19d20f92490eed508949b18df66ce61d0850a22a2b8b1e514673ddd447afb578e8090d4234dc0a179b85c25a145e44bce6a1e71cfe2f67d2e3b438cb4b9ff
-  data.tar.gz: 6f503dd265243982bc9163b7fb6da42211eca3eb647b1ee9491fcbc06b373c6822222ee6d72c190f2e1bbd7ca63c8126102acd841b1e8f0240434a1af3a69a4f
+  metadata.gz: e43b94745f35474edf4b463cd11b8c7d7bb29391f443c7ef2a9e84966d969aa6cf7c205a92c32a03452ba28c490c790ccd05d6569fd9db59d8c119b7e38f1dde
+  data.tar.gz: '07962c3175aed6d77295e473ad8462d0ae634c931a2f3f5bc75be1195977dded3cec013448c9d2fe2653fa83b9d178dd3513aaa1eda8eeafe9da539e3dbf06b0'

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -39,7 +39,6 @@ module DerivativeRodeo
       #        {Services::ConvertUriViaTemplateService} with the given
       #        :preprocessed_location_template.
       def initialize(input_uris:, output_location_template:, preprocessed_location_template: nil)
-        # NOTE: Are we using this preprocessed_location_template?  Wondering?
         @input_uris = Array.wrap(input_uris)
         @output_location_template = output_location_template
         @preprocessed_location_template = preprocessed_location_template
@@ -83,6 +82,7 @@ module DerivativeRodeo
       #
       # @see #build_step
       # @see #with_each_requisite_location_and_tmp_file_path
+      # rubocop:disable Metrics/MethodLength
       def generated_files
         # TODO: Examples please
         return @generated_files if defined?(@generated_files)
@@ -102,11 +102,16 @@ module DerivativeRodeo
           @generated_files << if generated_file.exist?
                                 generated_file
                               else
+                                log_message = "#{self.class}#generated_files :: " \
+                                          "input_location file_uri #{input_location.file_uri} :: " \
+                                          "Generating output_location file_uri #{generated_file.file_uri} via build_step."
+                                logger.info(log_message)
                                 build_step(input_location: input_location, output_location: generated_file, input_tmp_file_path: input_tmp_file_path)
                               end
         end
         @generated_files
       end
+      # rubocop:enable Metrics/MethodLength
 
       ##
       # @return [Array<String>]
@@ -168,20 +173,55 @@ module DerivativeRodeo
       #         {#output_location_template} or {#preprocessed_location_template}.
       #
       # @see [StorageLocations::BaseLocation#exist?]
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
       def destination(input_location)
-        output_location = input_location.derived_file_from(template: output_location_template)
+        output_location = input_location.derived_file_from(template: output_location_template, extension: output_extension)
 
-        return output_location if output_location.exist?
-        return output_location unless preprocessed_location_template
+        if output_location.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found output_location file_uri #{output_location.file_uri}."
+          logger.info(log_message)
 
-        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template)
+          return output_location
+        end
+
+        unless preprocessed_location_template
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "No preprocessed_location_template provided " \
+                        "nor does a file exist at output_location file_uri #{output_location.file_uri};" \
+                        " moving on to generation via #{self.class}#build_step."
+          logger.info(log_message)
+
+          return output_location
+        end
+
+        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template, extension: output_extension)
         # We only want the location if it exists
-        return preprocessed_location if preprocessed_location&.exist?
+        if preprocessed_location&.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found preprocessed_location file_uri #{output_location.file_uri}."
+          logger.info(log_message)
+
+          return preprocessed_location
+        end
+
+        log_message = "#{self.class}#destination :: " \
+                      "input_location file_uri #{input_location.file_uri} :: " \
+                      "No file exists at preprocessed_location file_uri #{preprocessed_location.file_uri} " \
+                      "nor output_location file_uri #{output_location.file_uri}; " \
+                      "moving on to generation via #{self.class}#build_step."
+        logger.info(log_message)
 
         # NOTE: The file does not exist at the output_location; but we pass this information along so
         # that the #build_step knows where to write the file.
         output_location
       end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
 
       ##
       # A bit of indirection to create a common interface for running a shell command.

data/lib/derivative_rodeo/generators/monochrome_generator.rb

@@ -5,7 +5,8 @@ module DerivativeRodeo
     ##
     # Take images an ensures that we have a monochrome derivative of those images.
     class MonochromeGenerator < BaseGenerator
-      # TODO: Can we assume a tiff?
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService for the interaction of the
+      #      magic ".mono" suffix
       self.output_extension = 'mono.tiff'
 
       ##

data/lib/derivative_rodeo/generators/pdf_split_generator.rb

@@ -52,7 +52,7 @@ module DerivativeRodeo
       # @see #existing_page_locations
       # @see .filename_for_a_derived_page_from_a_pdf?
       def image_file_basename_template(basename:)
-        "#{basename}/pages/#{basename}--page-%d.#{output_extension}"
+        "#{basename}--page-%d.#{output_extension}"
       end
 
       ##
@@ -62,21 +62,21 @@ module DerivativeRodeo
       # @param input_location [StorageLocations::BaseLocation]
       #
       # @return [Enumerable<StorageLocations::BaseLocation>] the files at the given :input_location
-      #         with :tail_glob.
+      #         with :tail_regexp.
       #
       # @note There is relation to {Generators::BaseGenerator#destination} and this method.
       #
       # @note The tail_glob is in relation to the {#image_file_basename_template}
       def existing_page_locations(input_location:)
         # See image_file_basename_template
-        tail_glob = "#{input_location.file_basename}/pages/*.#{output_extension}"
+        tail_regexp = %r{#{input_location.file_basename}--page-\d+\.#{output_extension}$}
 
-        output_locations = input_location.derived_file_from(template: output_location_template).globbed_tail_locations(tail_glob: tail_glob)
+        output_locations = input_location.derived_file_from(template: output_location_template).matching_locations_in_file_dir(tail_regexp: tail_regexp)
         return output_locations if output_locations.count.positive?
 
         return [] if preprocessed_location_template.blank?
 
-        input_location.derived_file_from(template: preprocessed_location_template).globbed_tail_loations(tail_glob: tail_glob)
+        input_location.derived_file_from(template: preprocessed_location_template).globbed_tail_loations(tail_regexp: tail_regexp)
       end
 
       ##
@@ -101,20 +101,22 @@ module DerivativeRodeo
       def with_each_requisite_location_and_tmp_file_path
         input_files.each do |input_location|
           input_location.with_existing_tmp_path do |input_tmp_file_path|
-            ## We want a single call for a directory listing of the image_file_basename_template
-            generated_files = existing_page_locations(input_location: input_location)
+            existing_locations = existing_page_locations(input_location: input_location)
 
-            if generated_files.count.zero?
-              generated_files = Services::PdfSplitter.call(
+            if existing_locations.count.positive?
+              existing_locations.each do |location|
+                yield(location, location.file_path)
+              end
+            else
+              # We're going to need to create the files and "cast" them to locations.
+              Services::PdfSplitter.call(
                 input_tmp_file_path,
                 image_extension: output_extension,
                 image_file_basename_template: image_file_basename_template(basename: input_location.file_basename)
-              )
-            end
-
-            generated_files.each do |image_path|
-              image_location = StorageLocations::FileLocation.new("file://#{image_path}")
-              yield(image_location, image_path)
+              ).each do |image_path|
+                image_location = StorageLocations::FileLocation.new("file://#{image_path}")
+                yield(image_location, image_path)
+              end
             end
           end
         end

data/lib/derivative_rodeo/services/convert_uri_via_template_service.rb

@@ -46,11 +46,12 @@ module DerivativeRodeo
       #     from_uris: ["file:///path1/A/file.pdf", "aws:///path2/B/file.pdf"],
       #     template: "file:///dest1/{{dir_parts[-1..-1]}}/{{ filename }}")
       #   => ["file:///dest1/A/file.pdf", "aws:///dest1/B/file.pdf"]
-      def self.call(from_uri:, template:, adapter: nil, separator: "/")
-        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator).call
+      def self.call(from_uri:, template:, adapter: nil, separator: "/", **options)
+        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator, **options).call
       end
 
-      def initialize(from_uri:, template:, adapter: nil, separator: "/")
+      # rubocop:disable Metrics/MethodLength
+      def initialize(from_uri:, template:, adapter: nil, separator: "/", **options)
         @from_uri = from_uri
         @template = template
         @adapter = adapter
@@ -60,12 +61,23 @@ module DerivativeRodeo
         @from_scheme, @path = uri.split("://")
         @parts = @path.split(separator)
         @dir_parts = @parts[0..-2]
-        @filename = @parts[-1]
-        @basename = File.basename(@filename, ".*")
-        @extension = File.extname(@filename)
+        @filename = options[:filename] || @parts[-1]
+        @basename = options[:basename] || File.basename(@filename, ".*")
+
+        ##
+        # HACK: Because the HocrGenerator has `.mono.tiff` and we are not interested in carrying
+        # forward the `.mono` suffix as that makes it hard to find the preprocessed word
+        # coordinates, alto, and plain text.  This ensures files derived from the .mono are findable
+        # in IIIF Print.
+        @basename = @basename.sub(/\.mono\z/, '')
+        @extension = options[:extension] || File.extname(@filename)
+        # When a generator specifies "same" we want to use the given file's extension
+        @extension = File.extname(@filename) if @extension == DerivativeRodeo::StorageLocations::SAME
+        @extension = ".#{@extension}" unless @extension.start_with?(".")
 
         @template_without_query, @template_query = template.split("?")
       end
+      # rubocop:enable Metrics/MethodLength
 
       def call
         to_uri = template_without_query.gsub(DIR_PARTS_REPLACEMENT_REGEXP) do |text|

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -101,10 +101,10 @@ module DerivativeRodeo
       # @param service [#call, Module<DerivativeRodeo::Services::ConvertUriViaTemplateService>]
       #
       # @return [StorageLocations::BaseLocation]
-      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService)
+      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService, **options)
         # HACK: Ensuring that we have the correct scheme.  Maybe this is a hack?
         from_uri = "#{scheme}://#{from_uri}" unless from_uri.start_with?("#{scheme}://")
-        to_uri = service.call(from_uri: from_uri, template: template, adapter: self)
+        to_uri = service.call(from_uri: from_uri, template: template, adapter: self, **options)
         new(to_uri)
       end
 
@@ -203,25 +203,25 @@ module DerivativeRodeo
       # @return [StorageLocations::BaseLocation]
       #
       # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
-      def derived_file_from(template:)
+      def derived_file_from(template:, **options)
         klass = DerivativeRodeo::StorageLocations::BaseLocation.load_location(template)
-        klass.build(from_uri: file_path, template: template)
+        klass.build(from_uri: file_path, template: template, **options)
       end
 
       ##
       # When you have a known location and want to check for files that are within that location,
-      # use the {#globbed_tail_locations} method.  In the case of {Generators::PdfSplitGenerator} we
+      # use the {#matching_locations_in_file_dir} method.  In the case of {Generators::PdfSplitGenerator} we
       # need to know the path to all of the image files we "split" off of the given PDF.
       #
       # We can use the :file_path as the prefix the given :tail_glob as the suffix for a "fully
       # qualified" Dir.glob type search.
       #
-      # @param tail_glob [String]
+      # @param tail_regexp [Regexp]
       #
       # @return [Enumerable<StorageLocations::BaseLocation>] the locations of the files; an empty
       #         array when there are none.
-      def globbed_tail_locations(tail_glob:)
-        raise NotImplementedError, "#{self.class}#globbed_locations"
+      def matching_locations_in_file_dir(tail_regexp:)
+        raise NotImplementedError, "#{self.class}#matching_locations_in_file_dir"
       end
 
       ##
@@ -231,6 +231,7 @@ module DerivativeRodeo
       def with_new_extension(extension)
         return file_path if extension == StorageLocations::SAME
 
+        # NOTE: May need to revisit this
         "#{file_path.split('.')[0]}.#{extension}"
       end
 

data/lib/derivative_rodeo/storage_locations/file_location.rb

@@ -35,8 +35,16 @@ module DerivativeRodeo
         file_uri
       end
 
-      def globbed_tail_locations(tail_glob:)
-        Dir.glob(File.join(file_dir, tail_glob))
+      ##
+      # @return [Enumerable<DerivativeRodeo::StorageLocations::FileLocation>]
+      #
+      # @see Generators::PdfSplitGenerator#image_file_basename_template
+      #
+      # @param tail_regexp [Regexp]
+      def matching_locations_in_file_dir(tail_regexp:)
+        Dir.glob(File.join(file_dir, "*")).each_with_object([]) do |filename, accumulator|
+          accumulator << derived_file_from(template: "file://#{filename}") if tail_regexp.match(filename)
+        end
       end
     end
   end

data/lib/derivative_rodeo/storage_locations/s3_location.rb

@@ -65,31 +65,19 @@ module DerivativeRodeo
       ##
       # @return [Enumerable<DerivativeRodeo::StorageLocations::S3Location>]
       #
-      # @note S3 allows searching on a prefix but does not allow for "wildcard" searches.  We can
-      #       use the components of the file_path to fake that behavior.
+      # @note S3 allows searching on a prefix but does not allow for "wildcard" searches.
       #
       # @see Generators::PdfSplitGenerator#image_file_basename_template
-      def globbed_tail_locations(tail_glob:)
-        # file_path = "s3://blah/1234/hello-world/pages/*.tiff"
-        #
-        # NOTE: Should we be storing our files as such?  The pattern we need is
-        # :parent_identifier/:file_set_identifier/files There are probably cases where a work has
-        # more than one PDF (that we intend to split); we don't want to trample on those split files
-        # and miscolate two PDFs.
-        #
-        # file_path = "s3://blah/1234/hello-world/hello-world.pdf
-        globname = File.join(file_dir, tail_glob)
-        regexp = %r{#{File.extname(globname)}$}
-
-        # NOTE: We're making some informed guesses, needing to include the fully qualified template
-        # based on both the key of the item in the bucket as well as the bucket's host.
+      #
+      # @param tail_regexp [Regexp]
+      def matching_locations_in_file_dir(tail_regexp:)
         uri = URI.parse(file_uri)
         scheme_and_host = "#{uri.scheme}://#{uri.host}"
 
-        bucket.objects(prefix: File.dirname(globname)).flat_map do |object|
-          if object.key.match(regexp)
+        bucket.objects(prefix: file_dir).each_with_object([]) do |object, accumulator|
+          if tail_regexp.match(object.key)
             template = File.join(scheme_and_host, object.key)
-            derived_file_from(template: template)
+            accumulator << derived_file_from(template: template)
           end
         end
       end

data/lib/derivative_rodeo/storage_locations/sqs_location.rb

@@ -13,6 +13,8 @@ module DerivativeRodeo
     # Location to download and upload files to Sqs
     # It uploads a file_uri to the queue, not the contents of that file
     # reading from the queue is not currently implemented
+    #
+    # rubocop:disable Metrics/ClassLength
     class SqsLocation < BaseLocation
       ##
       # @!group Class Attributes
@@ -85,11 +87,14 @@ module DerivativeRodeo
         batch = []
         Dir.glob("#{File.dirname(tmp_file_path)}/**/**").each.with_index do |fp, i|
           batch << { id: SecureRandom.uuid, message_body: output_json("file://#{fp}") }
-          if (i % batch_size).zero?
+          if (i + 1 % batch_size).zero?
             add_batch(messages: batch)
             batch = []
           end
         end
+
+        # Ensure we're flushing the batched up queue as part of completing the write.
+        add_batch(messages: batch) if batch.present?
         file_uri
       end
 
@@ -181,6 +186,7 @@ module DerivativeRodeo
       end
 
       def output_json(uri)
+        # TODO: Add ability to handle a pre-process-template given to an SQS, and pass that along to the generator when applicable.
         key = DerivativeRodeo::Services::ConvertUriViaTemplateService.call(from_uri: uri, template: template, adapter: self)
         { key => [template] }.to_json
       end
@@ -201,5 +207,6 @@ module DerivativeRodeo
         @file_uri_parts
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.3.0'
+  VERSION = '0.4.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-05 00:00:00.000000000 Z
+date: 2023-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport