derivative-rodeo 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bee0908ac5d045db1948b062d3a4e1569fff41b4a5a0f1521f31b620c15c53a6
-  data.tar.gz: 328de6f1cde3bcdadca31a361821b2c573425d15bfddbea5bf85d9475a8a3d0e
+  metadata.gz: 62872d16bfd5d73940f87d5c09f61f2a88ee67414f51905ce503f411b9b2fb37
+  data.tar.gz: 742d63ca02418b3453824655738e25b47d3cca918f030e2fb5db4c997d52e945
 SHA512:
-  metadata.gz: 2d8ba019ef30666d9633b73e90fe599b53d5f6698de0400b43a25c37a06470542962db8f61d74ed6a444f1ecc5ffbd604d2c0891dffcf5fa86d9dce87ace1bb5
-  data.tar.gz: b682377ce08e4379f1323bfdb6c6b8f46c97d45ebb392bb1dc7f5bc27f5b01701731880dc08f96cb5ef2b0da8ad39349b4534a63b4039e78616331b93bd9516c
+  metadata.gz: e43b94745f35474edf4b463cd11b8c7d7bb29391f443c7ef2a9e84966d969aa6cf7c205a92c32a03452ba28c490c790ccd05d6569fd9db59d8c119b7e38f1dde
+  data.tar.gz: '07962c3175aed6d77295e473ad8462d0ae634c931a2f3f5bc75be1195977dded3cec013448c9d2fe2653fa83b9d178dd3513aaa1eda8eeafe9da539e3dbf06b0'

data/lib/derivative_rodeo/generators/base_generator.rb

@@ -82,6 +82,7 @@ module DerivativeRodeo
       #
       # @see #build_step
       # @see #with_each_requisite_location_and_tmp_file_path
+      # rubocop:disable Metrics/MethodLength
       def generated_files
         # TODO: Examples please
         return @generated_files if defined?(@generated_files)
@@ -101,11 +102,16 @@ module DerivativeRodeo
           @generated_files << if generated_file.exist?
                                 generated_file
                               else
+                                log_message = "#{self.class}#generated_files :: " \
+                                          "input_location file_uri #{input_location.file_uri} :: " \
+                                          "Generating output_location file_uri #{generated_file.file_uri} via build_step."
+                                logger.info(log_message)
                                 build_step(input_location: input_location, output_location: generated_file, input_tmp_file_path: input_tmp_file_path)
                               end
         end
         @generated_files
       end
+      # rubocop:enable Metrics/MethodLength
 
       ##
       # @return [Array<String>]
@@ -167,20 +173,55 @@ module DerivativeRodeo
       #         {#output_location_template} or {#preprocessed_location_template}.
       #
       # @see [StorageLocations::BaseLocation#exist?]
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize
       def destination(input_location)
-        output_location = input_location.derived_file_from(template: output_location_template)
+        output_location = input_location.derived_file_from(template: output_location_template, extension: output_extension)
 
-        return output_location if output_location.exist?
-        return output_location unless preprocessed_location_template
+        if output_location.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found output_location file_uri #{output_location.file_uri}."
+          logger.info(log_message)
 
-        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template)
+          return output_location
+        end
+
+        unless preprocessed_location_template
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "No preprocessed_location_template provided " \
+                        "nor does a file exist at output_location file_uri #{output_location.file_uri};" \
+                        " moving on to generation via #{self.class}#build_step."
+          logger.info(log_message)
+
+          return output_location
+        end
+
+        preprocessed_location = input_location.derived_file_from(template: preprocessed_location_template, extension: output_extension)
         # We only want the location if it exists
-        return preprocessed_location if preprocessed_location&.exist?
+        if preprocessed_location&.exist?
+          log_message = "#{self.class}#destination :: " \
+                        "input_location file_uri #{input_location.file_uri} :: " \
+                        "Found preprocessed_location file_uri #{output_location.file_uri}."
+          logger.info(log_message)
+
+          return preprocessed_location
+        end
+
+        log_message = "#{self.class}#destination :: " \
+                      "input_location file_uri #{input_location.file_uri} :: " \
+                      "No file exists at preprocessed_location file_uri #{preprocessed_location.file_uri} " \
+                      "nor output_location file_uri #{output_location.file_uri}; " \
+                      "moving on to generation via #{self.class}#build_step."
+        logger.info(log_message)
 
         # NOTE: The file does not exist at the output_location; but we pass this information along so
         # that the #build_step knows where to write the file.
         output_location
       end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
 
       ##
       # A bit of indirection to create a common interface for running a shell command.

data/lib/derivative_rodeo/generators/monochrome_generator.rb

@@ -5,7 +5,8 @@ module DerivativeRodeo
     ##
     # Take images an ensures that we have a monochrome derivative of those images.
     class MonochromeGenerator < BaseGenerator
-      # TODO: Can we assume a tiff?
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService for the interaction of the
+      #      magic ".mono" suffix
       self.output_extension = 'mono.tiff'
 
       ##

data/lib/derivative_rodeo/services/convert_uri_via_template_service.rb

@@ -46,11 +46,12 @@ module DerivativeRodeo
       #     from_uris: ["file:///path1/A/file.pdf", "aws:///path2/B/file.pdf"],
       #     template: "file:///dest1/{{dir_parts[-1..-1]}}/{{ filename }}")
       #   => ["file:///dest1/A/file.pdf", "aws:///dest1/B/file.pdf"]
-      def self.call(from_uri:, template:, adapter: nil, separator: "/")
-        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator).call
+      def self.call(from_uri:, template:, adapter: nil, separator: "/", **options)
+        new(from_uri: from_uri, template: template, adapter: adapter, separator: separator, **options).call
       end
 
-      def initialize(from_uri:, template:, adapter: nil, separator: "/")
+      # rubocop:disable Metrics/MethodLength
+      def initialize(from_uri:, template:, adapter: nil, separator: "/", **options)
         @from_uri = from_uri
         @template = template
         @adapter = adapter
@@ -60,12 +61,23 @@ module DerivativeRodeo
         @from_scheme, @path = uri.split("://")
         @parts = @path.split(separator)
         @dir_parts = @parts[0..-2]
-        @filename = @parts[-1]
-        @basename = File.basename(@filename, ".*")
-        @extension = File.extname(@filename)
+        @filename = options[:filename] || @parts[-1]
+        @basename = options[:basename] || File.basename(@filename, ".*")
+
+        ##
+        # HACK: Because the HocrGenerator has `.mono.tiff` and we are not interested in carrying
+        # forward the `.mono` suffix as that makes it hard to find the preprocessed word
+        # coordinates, alto, and plain text.  This ensures files derived from the .mono are findable
+        # in IIIF Print.
+        @basename = @basename.sub(/\.mono\z/, '')
+        @extension = options[:extension] || File.extname(@filename)
+        # When a generator specifies "same" we want to use the given file's extension
+        @extension = File.extname(@filename) if @extension == DerivativeRodeo::StorageLocations::SAME
+        @extension = ".#{@extension}" unless @extension.start_with?(".")
 
         @template_without_query, @template_query = template.split("?")
       end
+      # rubocop:enable Metrics/MethodLength
 
       def call
         to_uri = template_without_query.gsub(DIR_PARTS_REPLACEMENT_REGEXP) do |text|

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -101,10 +101,10 @@ module DerivativeRodeo
       # @param service [#call, Module<DerivativeRodeo::Services::ConvertUriViaTemplateService>]
       #
       # @return [StorageLocations::BaseLocation]
-      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService)
+      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService, **options)
         # HACK: Ensuring that we have the correct scheme.  Maybe this is a hack?
         from_uri = "#{scheme}://#{from_uri}" unless from_uri.start_with?("#{scheme}://")
-        to_uri = service.call(from_uri: from_uri, template: template, adapter: self)
+        to_uri = service.call(from_uri: from_uri, template: template, adapter: self, **options)
         new(to_uri)
       end
 
@@ -203,9 +203,9 @@ module DerivativeRodeo
       # @return [StorageLocations::BaseLocation]
       #
       # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
-      def derived_file_from(template:)
+      def derived_file_from(template:, **options)
         klass = DerivativeRodeo::StorageLocations::BaseLocation.load_location(template)
-        klass.build(from_uri: file_path, template: template)
+        klass.build(from_uri: file_path, template: template, **options)
       end
 
       ##
@@ -231,6 +231,7 @@ module DerivativeRodeo
       def with_new_extension(extension)
         return file_path if extension == StorageLocations::SAME
 
+        # NOTE: May need to revisit this
         "#{file_path.split('.')[0]}.#{extension}"
       end
 

data/lib/derivative_rodeo/storage_locations/sqs_location.rb

@@ -13,6 +13,8 @@ module DerivativeRodeo
     # Location to download and upload files to Sqs
     # It uploads a file_uri to the queue, not the contents of that file
     # reading from the queue is not currently implemented
+    #
+    # rubocop:disable Metrics/ClassLength
     class SqsLocation < BaseLocation
       ##
       # @!group Class Attributes
@@ -85,11 +87,14 @@ module DerivativeRodeo
         batch = []
         Dir.glob("#{File.dirname(tmp_file_path)}/**/**").each.with_index do |fp, i|
           batch << { id: SecureRandom.uuid, message_body: output_json("file://#{fp}") }
-          if (i % batch_size).zero?
+          if (i + 1 % batch_size).zero?
             add_batch(messages: batch)
             batch = []
           end
         end
+
+        # Ensure we're flushing the batched up queue as part of completing the write.
+        add_batch(messages: batch) if batch.present?
         file_uri
       end
 
@@ -181,6 +186,7 @@ module DerivativeRodeo
       end
 
       def output_json(uri)
+        # TODO: Add ability to handle a pre-process-template given to an SQS, and pass that along to the generator when applicable.
         key = DerivativeRodeo::Services::ConvertUriViaTemplateService.call(from_uri: uri, template: template, adapter: self)
         { key => [template] }.to_json
       end
@@ -201,5 +207,6 @@ module DerivativeRodeo
         @file_uri_parts
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/derivative_rodeo/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerivativeRodeo
-  VERSION = '0.4.0'
+  VERSION = '0.4.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: derivative-rodeo
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Rob Kaufman
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-06 00:00:00.000000000 Z
+date: 2023-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -337,7 +337,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: An ETL Ecosystem for Derivative Processing.