derivative-rodeo 0.2.0

data/lib/derivative_rodeo/services/pdf_splitter/pages_summary.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+require 'open3'
+require 'mini_magick'
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitter
+      # A simple data structure that summarizes the image properties of the given path.
+      PagesSummary = Struct.new(
+        :path, :page_count, :width,
+        :height, :pixels_per_inch, :color_description,
+        :channels, :bits_per_channel, keyword_init: true
+      ) do
+        # class constant column numbers
+        COL_WIDTH = 3
+        COL_HEIGHT = 4
+        COL_COLOR_DESC = 5
+        COL_CHANNELS = 6
+        COL_BITS = 7
+        # only poppler 0.25+ has this column in output:
+        COL_XPPI = 12
+
+        # @return [Array<String, Integer, Integer>]
+        def color
+          [color_description, channels, bits_per_channel]
+        end
+        alias_method :ppi, :pixels_per_inch
+        alias_method :bits, :bits_per_channel
+
+        # If the underlying extraction couldn't set the various properties, we likely have an
+        # invalid_pdf.
+        def valid?
+          return false if pdf_pages_summary.color_description.nil?
+          return false if pdf_pages_summary.channels.nil?
+          return false if pdf_pages_summary.bits_per_channel.nil?
+          return false if pdf_pages_summary.height.nil?
+          return false if pdf_pages_summary.page_count.to_i.zero?
+
+          true
+        end
+      end
+
+      ##
+      # @api public
+      #
+      # @param path [String]
+      # @return [DerivativeRodeo::PdfSplitter::PagesSummary]
+      #
+      # Responsible for determining the image properties of the PDF.
+      #
+      # @note
+      #
+      #   Uses poppler 0.19+ pdfimages command to extract image listing metadata from PDF files.
+      #   Though we are optimizing for 0.25 or later for poppler.
+      #
+      # @note
+      #
+      #   For dpi extraction, falls back to calculating using MiniMagick, if neccessary.
+      #
+      # The first two lines are tabular header information:
+      #
+      # @example Output from PDF Images
+      #
+      #   bash-5.1$ pdfimages -list fmc_color.pdf  | head -5
+      #   page   num  step   width height color comp bpc  enc interp  object ID x-ppi y-ppi size ratio
+      #   --------------------------------------------------------------------------------------------
+      #   1     0 image    2475   413  rgb     3   8  jpeg   no        10  0   300   300 21.8K 0.7%
+      # rubocop:disable Metrics/AbcSize - Because this helps us process the results in one loop.
+      # rubocop:disable Metrics/MethodLength - Again, to help speed up the processing loop.
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      def PagesSummary.extract_from(path:)
+        # NOTE: https://github.com/scientist-softserv/iiif_print/pull/223/files for piping warnings
+        # to /dev/null
+        command = format('pdfimages -list %<path>s 2>/dev/null', path: path)
+
+        page_count = 0
+        color_description = 'gray'
+        width = 0
+        height = 0
+        channels = 0
+        bits_per_channel = 0
+        pixels_per_inch = 0
+        Open3.popen3(command) do |_stdin, stdout, _stderr, _wait_thr|
+          stdout.read.split("\n").each_with_index do |line, index|
+            # Skip the two header lines (see the above example)
+            next if index <= 1
+
+            page_count += 1
+            cells = line.gsub(/\s+/m, ' ').strip.split(' ')
+
+            color_description = 'rgb' if cells[COL_COLOR_DESC] != 'gray'
+            width = cells[COL_WIDTH].to_i if cells[COL_WIDTH].to_i > width
+            height = cells[COL_HEIGHT].to_i if cells[COL_HEIGHT].to_i > height
+            channels = cells[COL_CHANNELS].to_i if cells[COL_CHANNELS].to_i > channels
+            bits_per_channel = cells[COL_BITS].to_i if cells[COL_BITS].to_i > bits_per_channel
+
+            # In the case of poppler version < 0.25, we will have no more than 12 columns.  As such,
+            # we need to do some alternative magic to calculate this.
+            if page_count == 1 && cells.size <= 12
+              pdf = MiniMagick::Image.open(path)
+              width_points = pdf.width
+              width_px = width
+              pixels_per_inch = (72 * width_px / width_points).to_i
+            elsif cells[COL_XPPI].to_i > pixels_per_inch
+              pixels_per_inch = cells[COL_XPPI].to_i
+            end
+            # By the magic of nil#to_i if we don't have more than 12 columns, we've already set
+            # the pixels_per_inch and this line won't due much of anything.
+          end
+        end
+
+        new(
+          path: path,
+          page_count: page_count,
+          pixels_per_inch: pixels_per_inch,
+          width: width,
+          height: height,
+          color_description: color_description,
+          channels: channels,
+          bits_per_channel: bits_per_channel
+        )
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter/png_page.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitter
+      # The purpose of this class is to split the PDF into constituent png files.
+      class PngPage < PdfSplitter::Base
+        self.image_extension = 'png'
+
+        def gsdevice
+          return @gsdevice if defined?(@gsdevice)
+
+          color = pdf_pages_summary.color_description
+          bits_per_channel = pdf_pages_summary.bits_per_channel
+          if color == 'gray'
+            # 1 Bit Grayscale, if applicable:
+            return @gsdevice = 'pngmonod' if bits_per_channel == 1
+            return @gsdevice = 'pnggray' if bits_per_channel > 1
+          end
+
+          @gsdevice = 'png16m'
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter/tiff_page.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitter
+      ##
+      # The purpose of this class is to split the PDF into constituent tiff files.
+      class TiffPage < PdfSplitter::Base
+        self.image_extension = 'tiff'
+        self.compression = 'lzw'
+
+        ##
+        # @api private
+        #
+        # @return [String]
+        def gsdevice
+          return @gsdevice if defined?(@gsdevice)
+
+          color = pdf_pages_summary.color_description
+          channels = pdf_pages_summary.channels
+          bpc = pdf_pages_summary.bits_per_channel
+
+          @gsdevice = color_bpc(color, bpc)
+
+          # otherwise color:
+          @gsdevice ||= colordevice(channels, bpc)
+        end
+
+        def color_bpc(color, bpc)
+          return unless color == 'gray'
+
+          # CCITT Group 4 Black and White, if applicable:
+          if bpc == 1
+            self.compression = 'g4'
+            'tiffg4'
+          elsif bpc > 1
+            # 8 Bit Grayscale, if applicable:
+            'tiffgray'
+          end
+        end
+
+        def colordevice(channels, bpc)
+          bits = bpc * channels
+          # will be either 8bpc/16bpd color TIFF,
+          #   with any CMYK source transformed to 8bpc RBG
+          bits = 24 unless [24, 48].include? bits
+          "tiff#{bits}nc"
+        end
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/pdf_splitter_service.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module Services
+    module PdfSplitterService
+      ##
+      # @api public
+      #
+      # Find the {PdfSplitter::Base} with the given name.
+      #
+      # @param name [#to_s]
+      # @return [PdfSplitter::Base]
+      def self.for(name)
+        klass_name = "#{name.to_s.classify}_page".classify
+        "DerivativeRodeo::Services::PdfSplitter::#{klass_name}".constantize
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/services/url_service.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'httparty'
+
+module DerivativeRodeo
+  module Services
+    ##
+    # A utility class for handling general URLs.  Provided as a means of easing the implementation
+    # logic of those that use this class.
+    #
+    # @note
+    #   It is a good design idea to wrap a library (in this case HTTParty).  The goal is to expose
+    #   the smallest interface and make it something that would be easy to swap out.
+    #
+    # @see https://rubygems.org/gems/httparty
+    module UrlService
+      ##
+      # @param url [String]
+      #
+      # @return [String]
+      def self.read(url)
+        HTTParty.get(url, logger: DerivativeRodeo.config.logger).body
+      rescue StandardError => e
+        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        raise e
+      end
+
+      ##
+      # @param url [String]
+      #
+      # @return [URI] when the URL resolves successfully
+      # @return [FalseClass] when the URL's head request is not successful or we've exhausted our
+      #         remaining redirects.
+      def self.exists?(url)
+        HTTParty.head(url, logger: DerivativeRodeo.config.logger)
+      rescue StandardError => e
+        DerivativeRodeo.config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        false
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/storage_locations/base_location.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # When the output location is the same type of location as "this" location, we indicate that via
+    # the SAME constant.
+    SAME = :same
+
+    ##
+    # The base location for storing files.
+    #
+    # - dir :: is the directory path
+    # - path :: is the full file path
+    # - uri :: is the full file path plus the uri prefix parts
+    #
+    # A location represents a pointer to a storage location.  The {#exist?} method can answer if a
+    # file exists at the path.
+    #
+    # rubocop:disable Metrics/ClassLength
+    class BaseLocation
+      @locations = []
+
+      ##
+      # @return [Array<String>]
+      def self.locations
+        @locations ||= []
+      end
+
+      def self.inherited(subclass)
+        locations << subclass.location_name
+        super
+      end
+
+      ##
+      # @return [String]
+      def self.location_name
+        to_s.demodulize.underscore.sub(/_location$/, '')
+      end
+
+      class << self
+        alias scheme location_name
+      end
+
+      ##
+      # @param location_name [String]
+      #
+      # @return [Class]
+      def self.load_location(location_name)
+        location_name = location_name.split("://").first
+        raise Errors::StorageLocationNotFoundError.new(location_name: location_name) unless locations.include?(location_name)
+        "DerivativeRodeo::StorageLocations::#{location_name.to_s.classify}Location".constantize
+      end
+
+      ##
+      # @param file_uri [String] of the form scheme://arbitrary-stuff
+      #
+      # @return [BaseLocation]
+      def self.from_uri(file_uri)
+        location_name = file_uri.split('://').first
+        raise Errors::StorageLocationMissing.new(file_uri: file_uri) if location_name.blank?
+
+        load_location(location_name).new(file_uri)
+      end
+
+      ##
+      # Registers the location with the main StorageLocation class to it can be used
+      #
+      # @param location_name [String]
+      def self.register_location(location_name)
+        return if DerivativeRodeo::StorageLocations::BaseLocation.locations.include?(location_name.to_s)
+
+        DerivativeRodeo::StorageLocations::BaseLocation.locations << location_name.to_s
+      end
+
+      ##
+      # Create a new uri of the classes type. Parts argument should have a default in
+      # implementing classes. Must support a number or the symbol :all
+      #
+      # @api public
+      #
+      # @param path [String]
+      # @param parts [Integer, :all]
+      # @return [String]
+      #
+      # @see .file_path_from_parts
+      def self.create_uri(path:, parts:)
+        raise NotImplementedError, "#{self.class}.create_uri"
+      end
+
+      ##
+      # Build a {StorageLocations::BaseLocation} by converting the :from_uri with the :template via
+      # the given :service.
+      #
+      # @param from_uri [String]
+      # @param template [String]
+      # @param service [#call, Module<DerivativeRodeo::Services::ConvertUriViaTemplateService>]
+      #
+      # @return [StorageLocations::BaseLocation]
+      def self.build(from_uri:, template:, service: DerivativeRodeo::Services::ConvertUriViaTemplateService)
+        # HACK: Ensuring that we have the correct scheme.  Maybe this is a hack?
+        from_uri = "#{scheme}://#{from_uri}" unless from_uri.start_with?("#{scheme}://")
+        to_uri = service.call(from_uri: from_uri, template: template, adapter: self)
+        new(to_uri)
+      end
+
+      ##
+      # @param path [String]
+      # @param parts [Integer, :all]
+      #
+      # @return [String]
+      def self.file_path_from_parts(path:, parts:)
+        parts = - parts unless parts == :all || parts.negative?
+        parts == :all ? path : path.split('/')[parts..-1].join('/')
+      end
+
+      ##
+      # @param file_uri [String] a URI to the file's location; this is **not** a templated URI (as
+      #        described in {DerivativeRodeo::Services::ConvertUriViaTemplateService}
+      # @param config [DerivativeRodeo::Configuration]
+      def initialize(file_uri, config: DerivativeRodeo.config)
+        @file_uri = file_uri
+        @config = config
+      end
+
+      attr_accessor :tmp_file_path
+      private :tmp_file_path=, :tmp_file_path
+
+      attr_reader :config, :file_uri
+
+      ##
+      # @param auto_write_file [Boolean] Provided as a testing helper method.
+      #
+      # @yieldparam tmp_file_path [String]
+      #
+      # @return [StorageLocations::BaseLocation]
+      # @see with_tmp_path
+      def with_new_tmp_path(auto_write_file: true, &block)
+        with_tmp_path(lambda { |_file_path, tmp_file_path, exist|
+                        FileUtils.rm_rf(tmp_file_path) if exist
+                        FileUtils.touch(tmp_file_path)
+                      }, auto_write_file: auto_write_file, &block)
+      end
+
+      ##
+      # @yieldparam tmp_file_path [String]
+      # @return [StorageLocations::BaseLocation]
+      def with_existing_tmp_path
+        raise NotImplementedError, "#{self.class}#with_existing_tmp_path"
+      end
+
+      ##
+      # @param preamble_lambda [Lambda, #call] the "function" we should call to prepare the
+      #        temporary location before we yield it's location.
+      #
+      # @param auto_write_file [Boolean] Provided as a testing helper method.  Given that we have
+      #        both {#with_new_tmp_path} and {#with_existing_tmp_path}, we want the default to not
+      #        automatically perform the write.  But this is something we can easily forget when
+      #        working with the {#with_new_tmp_path}
+      #
+      # @yieldparam tmp_file_path [String]
+      #
+      # @return [StorageLocations::BaseLocation]
+      def with_tmp_path(preamble_lambda, auto_write_file: false)
+        raise ArgumentError, 'Expected a block' unless block_given?
+
+        tmp_file_dir do |tmpdir|
+          self.tmp_file_path = File.join(tmpdir, file_dir, file_name)
+          FileUtils.mkdir_p(File.dirname(tmp_file_path))
+          preamble_lambda.call(file_path, tmp_file_path, exist?)
+          yield tmp_file_path
+          write if auto_write_file
+        end
+        # TODO: Do we need to ensure this?
+        self.tmp_file_path = nil
+
+        # In returning self we again remove the need for those calling #with_new_tmp_path,
+        # #with_tmp_path, and #with_new_tmp_path to remember to return the current Location.
+        # In other words removing the jagged edges of the code.
+        self
+      end
+
+      ##
+      # Write the tmp file to the file_uri
+      def write
+        raise NotImplementedError, "#{self.class}#write"
+      end
+
+      ##
+      # @return [TrueClass] when the file exists in this storage
+      # @return [FalseClass] when the file does not exist in this storage
+      def exist?
+        raise NotImplementedError, "#{self.class}#exist?"
+      end
+      alias exists? exist?
+
+      ##
+      # @param template [String]
+      # @return [StorageLocations::BaseLocation]
+      #
+      # @see DerivativeRodeo::Services::ConvertUriViaTemplateService
+      def derived_file_from(template:)
+        klass = DerivativeRodeo::StorageLocations::BaseLocation.load_location(template)
+        klass.build(from_uri: file_path, template: template)
+      end
+
+      ##
+      # @param extension [String, StorageLocations::SAME]
+      # @return [String] the path for the new extension; when given {StorageLocations::SAME} re-use
+      #         the file's extension.
+      def with_new_extension(extension)
+        return file_path if extension == StorageLocations::SAME
+
+        "#{file_path.split('.')[0]}.#{extension}"
+      end
+
+      def file_path
+        @file_path ||= @file_uri.sub(%r{.+://}, '')
+      end
+
+      def file_dir
+        @file_dir ||= File.dirname(file_path)
+      end
+
+      def file_name
+        @file_name ||= File.basename(file_path)
+      end
+
+      def file_extension
+        @file_extension ||= File.extname(file_path)
+      end
+
+      def file_basename
+        @file_basename ||= File.basename(file_path, file_extension)
+      end
+
+      def tmp_file_dir(&block)
+        raise ArgumentError, 'Expected a block' unless block_given?
+
+        Dir.mktmpdir(&block)
+      end
+    end
+    # rubocop:enable Metrics/ClassLength
+  end
+end
+
+Dir.glob(File.join(__dir__, '**/*')).sort.each do |location|
+  require location unless File.directory?(location) || location.match?('base_location')
+end

data/lib/derivative_rodeo/storage_locations/concerns/download_concern.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'httparty'
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # A helper module for copying files from one location to another.
+    module DownloadConcern
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def create_uri(path:, parts: :all, ssl: true)
+          file_path = file_path_from_parts(path: path, parts: parts)
+          "#{adapter_prefix(ssl: ssl)}#{file_path}"
+        end
+
+        def adapter_prefix(ssl: true)
+          ssl ? "https://" : "http://"
+        end
+      end
+
+      delegate :config, to: DerivativeRodeo
+
+      def with_existing_tmp_path(&block)
+        with_tmp_path(lambda { |_file_path, tmp_file_path, exist|
+          raise Errors::FileMissingError unless exist
+
+          response = get(file_uri)
+          File.open(tmp_file_path, 'wb') { |fp| fp.write(response.body) }
+        }, &block)
+      end
+
+      ##
+      # Implemented to complete the interface.
+      #
+      # @raise [NotImplementedError]
+      def write
+        raise "#{self.class}#write is deliberately not implemented"
+      end
+
+      ##
+      # @param url [String]
+      #
+      # @return [String]
+      def read(url)
+        HTTParty.get(url, logger: config.logger)
+      rescue => e
+        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        raise e
+      end
+
+      ##
+      # @param url [String]
+      #
+      # @return [URI] when the URL resolves successfully
+      # @return [FalseClass] when the URL's head request is not successful or we've exhausted our
+      #         remaining redirects.
+      def exists?(url)
+        HTTParty.head(url, logger: config.logger)
+      rescue => e
+        config.logger.error(%(#{e.message}\n#{e.backtrace.join("\n")}))
+        false
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/storage_locations/file_location.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # Location for files found on a local disk
+    class FileLocation < BaseLocation
+      def self.create_uri(path:, parts: :all)
+        file_path = file_path_from_parts(path: path, parts: parts)
+        "#{adapter_prefix}#{file_path}"
+      end
+
+      def self.adapter_prefix
+        "#{scheme}://"
+      end
+
+      def with_existing_tmp_path(&block)
+        with_tmp_path(lambda { |file_path, tmp_file_path, exist|
+          raise Errors::FileMissingError unless exist
+
+          FileUtils.cp(file_path, tmp_file_path)
+        }, &block)
+      end
+
+      def exist?
+        File.exist?(file_path)
+      end
+
+      # write the file to the file_uri
+      def write
+        raise Errors::FileMissingError("Use write within a with_new_tmp_path block and fille the mp file with data before writing") unless File.exist?(tmp_file_path)
+
+        FileUtils.mkdir_p(file_dir)
+        FileUtils.cp_r(tmp_file_path, file_path)
+        file_uri
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/storage_locations/http_location.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'derivative_rodeo/storage_locations/concerns/download_concern'
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # Location for files from the web. Download only, can not write!
+    class HttpLocation < BaseLocation
+      include DownloadConcern
+    end
+  end
+end

data/lib/derivative_rodeo/storage_locations/https_location.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'derivative_rodeo/storage_locations/concerns/download_concern'
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # Location for files from the web. Download only, can not write!
+    class HttpsLocation < BaseLocation
+      include DownloadConcern
+    end
+  end
+end