derivative-rodeo 0.2.0

data/lib/derivative_rodeo/storage_locations/s3_location.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # Location to download and upload files to S3
+    #
+    class S3Location < BaseLocation
+      attr_writer :bucket
+      ##
+      # Create a new uri of the classes type. Parts argument should have a default in
+      # implementing classes. Must support a number or the symbol :all
+      #
+      # @api public
+      #
+      # @param path [String]
+      # @param parts [Integer, :all], defaults to 2 for S3 which is parent_dir/file_name.ext
+      # @return [String]
+      def self.create_uri(path:, parts: 2)
+        file_path = file_path_from_parts(path: path, parts: parts)
+        File.join("#{adapter_prefix}/#{file_path}")
+      end
+
+      ##
+      # @param config [DerivativeRodeo::Configuration]
+      # @return [String]
+      def self.adapter_prefix(config: DerivativeRodeo.config)
+        "#{scheme}://#{config.aws_s3_bucket}.s3.#{config.aws_s3_region}.amazonaws.com"
+      end
+
+      ##
+      # @api public
+      # download or copy the file to a tmp path
+      # deletes the tmp file after the block is executed
+      #
+      # @return [String] the path to the tmp file
+      def with_existing_tmp_path(&block)
+        with_tmp_path(lambda { |file_path, tmp_file_path, exist|
+                        raise Errors::FileMissingError unless exist
+                        obj = bucket.object(file_path)
+                        obj.download_file(tmp_file_path)
+                      }, &block)
+      end
+
+      ##
+      # @api public
+      #
+      # Existance is futile
+      # @return [Boolean]
+      def exist?
+        bucket.objects(prefix: file_path).count.positive?
+      end
+
+      ##
+      # @api public
+      # write the tmp file to the file_uri
+      #
+      # @return [String] the file_uri
+      def write
+        raise Errors::FileMissingError("Use write within a with__new_tmp_path block and fill the tmp file with data before writing") unless File.exist?(tmp_file_path)
+
+        obj = bucket.object(file_path)
+        obj.upload_file(tmp_file_path)
+        file_uri
+      end
+
+      ##
+      # @api private
+      #
+      # @return [Aws::S3::Resource]
+      def resource
+        @resource ||= if DerivativeRodeo.config.aws_s3_access_key_id
+                        Aws::S3::Resource.new(region: DerivativeRodeo.config.aws_s3_region,
+                                              credentials: Aws::Credentials.new(
+                                                DerivativeRodeo.config.aws_s3_access_key_id,
+                                                DerivativeRodeo.config.aws_s3_secret_access_key
+                                              ))
+                      else
+                        Aws::S3::Resource.new
+                      end
+      end
+
+      ##
+      # @api private
+      # https://long-term-video-storage.s3.eu-west-1.amazonaws.com/path1/path2/file.tld
+      def bucket_name
+        @bucket_name ||= file_uri.match(%r{s3://(.+)\.s3})&.[](1)
+      rescue StandardError
+        raise Errors::BucketMissingError
+      end
+
+      def bucket
+        @bucket ||= resource.bucket(bucket_name)
+      end
+
+      def file_path
+        @file_path ||= @file_uri.sub(%r{.+://.+?/}, '')
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/storage_locations/sqs_location.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+require 'cgi'
+
+ActiveSupport::Inflector.inflections(:en) do |inflect|
+  inflect.uncountable 'sqs'
+end
+
+module DerivativeRodeo
+  module StorageLocations
+    ##
+    # Location to download and upload files to Sqs
+    # It uploads a file_uri to the queue, not the contents of that file
+    # reading from the queue is not currently implemented
+    class SqsLocation < BaseLocation
+      class_attribute :batch_size, default: 10
+
+      attr_writer :client
+
+      ##
+      # Create a new uri of the classes type. Parts argument should have a default in
+      # implementing classes. Must support a number or the symbol :all
+      #
+      # @api public
+      #
+      # @param path [String]
+      # @param parts [Integer, :all], defaults to 1 for Sqs which is file_name.ext. We use 1 because it helps with tmp files, but the actual sqs queue does not have a file path
+      # @return [String]
+      def self.create_uri(path:, parts: 1)
+        file_path = file_path_from_parts(path: path, parts: parts)
+        "#{adapter_prefix}#{file_path}"
+      end
+
+      def self.adapter_prefix(config: DerivativeRodeo.config)
+        "#{scheme}://#{config.aws_sqs_region}.amazonaws.com/#{config.aws_sqs_account_id}/#{config.aws_sqs_queue}/"
+      end
+
+      # TODO: implement read
+      # ##
+      # # @api public
+      # # download or copy the file to a tmp path
+      # # deletes the tmp file after the block is executed
+      # #
+      # # @return [String] the path to the tmp file
+      # def with_existing_tmp_path(&block)
+      #   with_tmp_path(lambda { |_file_path, tmp_file_path, exist|
+      #                   raise Errors::FileMissingError unless exist
+      #                   File.open(tmp_file_path, 'w') do |file|
+      #                     read_batch.each do |message|
+      #                       file.write(message)
+      #                     end
+      #                   end
+      #                 }, &block)
+      # end
+
+      ##
+      # @api public
+      #
+      # Existance is futile. And there's not way to check if a specific item is in an sqs queue
+      # @return [Boolean]
+      def exist?
+        false
+      end
+
+      ##
+      # @api public
+      # write the tmp file to the file_uri
+      #
+      # @return [String] the file_uri
+      def write
+        raise Errors::FileMissingError("Use write within a with__new_tmp_path block and fille the mp file with data before writing") unless File.exist?(tmp_file_path)
+        raise Errors::MaxQqueueSize(batch_size: batch_size) if batch_size > config.aws_sqs_max_batch_size
+        batch = []
+        Dir.glob("#{File.dirname(tmp_file_path)}/**/**").each.with_index do |fp, i|
+          batch << { id: SecureRandom.uuid, message_body: output_json("file://#{fp}") }
+          if (i % batch_size).zero?
+            add_batch(messages: batch)
+            batch = []
+          end
+        end
+        file_uri
+      end
+
+      def client
+        @client ||= if config.aws_sqs_access_key_id && config.aws_sqs_secret_access_key
+                      Aws::SQS::Client.new(
+                        region: config.aws_sqs_region,
+                        credentials: Aws::Credentials.new(
+                          config.aws_sqs_access_key_id,
+                          config.aws_sqs_secret_access_key
+                        )
+                      )
+                    else
+                      Aws::SQS::Client.new(region: config.aws_sqs_region)
+                    end
+      end
+
+      def add(message:)
+        client.send_message({
+                              queue_url: queue_url,
+                              message_body: message
+                            })
+      end
+
+      def add_batch(messages:)
+        client.send_message_batch({
+                                    queue_url: queue_url,
+                                    entries: messages
+                                  })
+      end
+
+      # def read_batch
+      #   raise Errors::MaxQqueueSize(batch_size: batch_size) if batch_size > DerivativeRodeo.config.aws_sqs_max_batch_size
+
+      #   response = client.receive_message({
+      #                                       queue_url: queue_url,
+      #                                       max_number_of_messages: batch_size
+      #                                     })
+      #   response.messages.map do |message|
+      #     JSON.parse(message.body)
+      #   end
+      # end
+
+      def queue_url
+        @queue_url ||= client.get_queue_url(queue_name: queue_name).queue_url
+      end
+
+      ##
+      # @api private
+      def queue_name
+        @queue_name ||= file_uri_parts[:queue_name]
+      rescue StandardError
+        raise Errors::QueueMissingError
+      end
+
+      ##
+      # @api private
+      def file_path
+        @file_path ||= [file_dir, file_name].join('/')
+      end
+
+      def file_dir
+        @file_dir ||= file_uri_parts[:file_dir]
+      end
+
+      ##
+      # @api private
+      def file_name
+        @file_name ||= file_uri_parts[:file_name]
+      end
+
+      def template
+        params&.[]('template')&.first
+      end
+
+      def scheme
+        file_uri_parts&.[](:scheme)
+      end
+
+      def params
+        @params ||= CGI.parse(file_uri_parts[:query]) if file_uri_parts[:query]
+      end
+
+      def output_json(uri)
+        key = DerivativeRodeo::Services::ConvertUriViaTemplateService.call(from_uri: uri, template: template, adapter: self)
+        { key => [template] }.to_json
+      end
+
+      def file_uri_parts
+        return @file_uri_parts if @file_uri_parts
+        uri = URI.parse(file_uri)
+        @file_uri_parts = uri&.component&.inject({}) do |hash, component|
+          hash[component] = uri.send(component)
+          hash
+        end
+        @file_uri_parts[:region] = @file_uri_parts[:host]&.split('.')&.[](0)
+        path_parts = @file_uri_parts[:path]&.split('/')
+        @file_uri_parts[:account_id] = path_parts&.[](1)
+        @file_uri_parts[:queue_name] = path_parts&.[](2)
+        @file_uri_parts[:file_name] = path_parts&.[](-1)
+        @file_uri_parts[:file_dir] = path_parts&.[](3..-2)&.join('/')
+        @file_uri_parts
+      end
+    end
+  end
+end

data/lib/derivative_rodeo/technical_metadata.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  TechnicalMetadata = Struct.new(:color, :num_components, :bits_per_component, :width, :height, :content_type,
+                                 keyword_init: true) do
+    alias_method :number_of_components, :num_components
+
+    def to_hash
+      {
+        color: color,
+        num_components: num_components,
+        bits_per_component: bits_per_component,
+        width: width,
+        height: height,
+        content_type: content_type
+      }
+    end
+
+    def monochrome?
+      color.to_s.casecmp('monochrome').zero?
+    end
+  end
+end

data/lib/derivative_rodeo/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DerivativeRodeo
+  VERSION = '0.2.0'
+end

data/lib/derivative_rodeo.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext'
+
+require 'byebug' if ENV['DEBUG']
+
+require 'derivative_rodeo/errors'
+
+require 'derivative_rodeo/configuration'
+require 'derivative_rodeo/technical_metadata'
+require 'derivative_rodeo/version'
+# Base Location loads the other adapters
+require 'derivative_rodeo/storage_locations/base_location'
+require 'derivative_rodeo/generators/base_generator'
+require 'derivative_rodeo/services/base_service'
+
+##
+# DerivativeRodeo is a gem that allows you to generate derivative files from source files
+# It is storage location agnostic, relying on {StorageLocations}. Files can be stored locally or in the cloud.
+# {Generators} are designed to be simple to create and to short circut logic if a
+# pre processed version exists
+module DerivativeRodeo
+  ##
+  # The {Configuration} that the various processes in your implementation will use.
+  #
+  # @api public
+  #
+  # @yieldparam [Derivative::Rodeo::Configuration]
+  # @return [Derivative::Rodeo::Configuration]
+  def self.config
+    @config ||= Configuration.new
+    yield(@config) if block_given?
+    @config
+  end
+end

metadata

@@ -0,0 +1,339 @@
+--- !ruby/object:Gem::Specification
+name: derivative-rodeo
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Rob Kaufman
+- Jeremy Friesen
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: marcel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mini_magick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bixby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard-activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda-matchers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: An ETL Ecosystem for Derivative Processing.
+email:
+- rob@notch8.com
+- jeremy.n.friesen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- derivative_rodeo.gemspec
+- lib/derivative-rodeo.rb
+- lib/derivative/rodeo.rb
+- lib/derivative_rodeo.rb
+- lib/derivative_rodeo/configuration.rb
+- lib/derivative_rodeo/errors.rb
+- lib/derivative_rodeo/generators/base_generator.rb
+- lib/derivative_rodeo/generators/concerns/copy_file_concern.rb
+- lib/derivative_rodeo/generators/copy_generator.rb
+- lib/derivative_rodeo/generators/hocr_generator.rb
+- lib/derivative_rodeo/generators/monochrome_generator.rb
+- lib/derivative_rodeo/generators/pdf_split_generator.rb
+- lib/derivative_rodeo/generators/thumbnail_generator.rb
+- lib/derivative_rodeo/generators/word_coordinates_generator.rb
+- lib/derivative_rodeo/services/base_service.rb
+- lib/derivative_rodeo/services/convert_uri_via_template_service.rb
+- lib/derivative_rodeo/services/extract_word_coordinates_from_hocr_sgml_service.rb
+- lib/derivative_rodeo/services/image_identify_service.rb
+- lib/derivative_rodeo/services/image_jp2_service.rb
+- lib/derivative_rodeo/services/image_service.rb
+- lib/derivative_rodeo/services/pdf_splitter/base.rb
+- lib/derivative_rodeo/services/pdf_splitter/jpg_page.rb
+- lib/derivative_rodeo/services/pdf_splitter/pages_summary.rb
+- lib/derivative_rodeo/services/pdf_splitter/png_page.rb
+- lib/derivative_rodeo/services/pdf_splitter/tiff_page.rb
+- lib/derivative_rodeo/services/pdf_splitter_service.rb
+- lib/derivative_rodeo/services/url_service.rb
+- lib/derivative_rodeo/storage_locations/base_location.rb
+- lib/derivative_rodeo/storage_locations/concerns/download_concern.rb
+- lib/derivative_rodeo/storage_locations/file_location.rb
+- lib/derivative_rodeo/storage_locations/http_location.rb
+- lib/derivative_rodeo/storage_locations/https_location.rb
+- lib/derivative_rodeo/storage_locations/s3_location.rb
+- lib/derivative_rodeo/storage_locations/sqs_location.rb
+- lib/derivative_rodeo/technical_metadata.rb
+- lib/derivative_rodeo/version.rb
+homepage: https://github.com/scientist-softserv/derivative_rodeo
+licenses:
+- APACHE-2.0
+metadata:
+  homepage_uri: https://github.com/scientist-softserv/derivative_rodeo
+  source_code_uri: https://github.com/scientist-softserv/derivative_rodeo
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: An ETL Ecosystem for Derivative Processing.
+test_files: []