jay_api 27.1.0

data/README.md

@@ -0,0 +1,61 @@
+# Jay API
+
+This gem provides a set of classes and modules to access Jay functionality
+while abstracting internal implementations.
+
+## Requirements
+
+* Ruby >= 2.7.0
+* Bundler ~> 2, < 2.5.0
+
+## Setup
+
+Clone the repository and install the dependencies by running:
+
+```shell
+bundle install
+```
+
+## Running Tests
+
+You can run the tests just by executing rspec.
+
+```shell
+bundle exec rspec
+```
+
+To generate a Coverage report:
+
+```shell
+export COVERAGE=true
+rspec
+```
+
+*The coverage report will be written to the `/coverage` path*
+
+## Generating Documentation
+
+```shell
+bundle exec yard
+```
+
+*The documentation will be generated in the `/doc` path*
+
+## Contributing
+
+* This project uses [Semantic Versioning](https://semver.org/)
+* This project uses a CHANGELOG.md to keep track of the changes.
+
+1. Add your feature.
+2. While editing your code keep an eye out for Rubocop and Reek suggestions
+   try to keep both linters happy. 😉
+3. Write unit and integration *(desirably but not required)* tests for it.
+4. Run the tests with the coverage report generation enabled (Check the *Running
+   Tests section)*.
+5. Make sure your Unit Test coverage is at least 90%
+6. Run the `yard` command to generate documentation and make sure your
+   documentation coverage is 100% (everything should be documented)
+7. Add your features to the `CHANGELOG.md` file under the *Unreleased* section.
+   (Check the `CHANGELOG.md`) file for info on how to properly add the changes
+   there.
+8. Push your changes for code review

data/jay_api.gemspec

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative 'lib/jay_api/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'jay_api'
+  spec.version       = JayAPI::VERSION
+  spec.authors       = ['Accenture-Industry X', 'ESR Labs']
+
+  spec.summary       = "A collection of classes and modules to access JAY's functionality"
+  spec.description   = "A collection of classes and modules to access JAY's functionality"
+  spec.homepage      = 'https://github.com/esrlabs/jay_api'
+  spec.license       = 'Apache-2.0'
+
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.7.0')
+
+  spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = 'https://github.com/esrlabs/jay_api/blob/master/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").select { |f| f.match(%r{^(CHANGELOG|README|lib/)}) } << File.basename(__FILE__)
+  end
+
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'activesupport', '~> 7'
+  spec.add_runtime_dependency 'concurrent-ruby', '~> 1'
+  spec.add_runtime_dependency 'elasticsearch', '~> 7', '<= 7.9.0'
+  spec.add_runtime_dependency 'git', '~> 1', '>= 1.8.0-1'
+  spec.add_runtime_dependency 'logging', '~> 2'
+  spec.add_runtime_dependency 'rspec', '~> 3.0'
+end

data/lib/jay_api/abstract/connection.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module JayAPI
+  module Abstract
+    # A class for an abstract 'Connection'. It is responsible for yielding a block
+    # for +max_attempts+ times at most, or until a specified +error+ is no longer
+    # raised. The reason the class is specifically called 'Connection', is because
+    # it contains logging that describes a connection.
+    class Connection
+      attr_reader :attempts, :max_attempts, :wait_strategy, :logger
+
+      # @param [Integer] max_attempts The maximum number of connection attempts to be made.
+      # @param [JayAPI::Elasticsearch::WaitStrategy] wait_strategy The waiting strategy for reconnections.
+      # @param [Logging::Logger] logger
+      def initialize(max_attempts:, wait_strategy:, logger:)
+        @max_attempts = max_attempts
+        @wait_strategy = wait_strategy
+        @logger = logger
+        @attempts = 0
+      end
+
+      # Yields the passed block and if the specified 'error' is raised, a new
+      # yield attempt will be made until the +max_attempts+ limit is reached.
+      # @param [Class, Array<Class>] errors Some error Class, or a list of them.
+      # @param [Array<Class>] except An array of exceptions for which no retry
+      #   should happen even if they are subclasses of the exception(s) passed
+      #   in +errors+.
+      def retry(errors:, except: [])
+        self.attempts += 1
+        yield
+      rescue *errors => e
+        raise if except.any? { |exception| e.is_a?(exception) }
+
+        logger.info("#{e} occurred")
+        if attempts < max_attempts
+          wait_strategy.wait
+          logger.info("Retrying... (There are #{max_attempts - attempts} retries left)")
+          retry
+        end
+
+        logger.info('No more attempts to connect will be made')
+        raise
+      end
+
+      private
+
+      attr_writer :attempts
+    end
+  end
+end

data/lib/jay_api/abstract/constant_wait.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative 'wait_strategy'
+
+module JayAPI
+  module Abstract
+    # A constant wait strategy implementation of the WaitStrategy abstract class.
+    # This strategy uses a fixed wait interval between retries. The wait interval does not change
+    # regardless of the number of attempts made. It is suitable for scenarios where a constant
+    # delay is preferred over an increasing delay.
+    #
+    # Inherits from WaitStrategy and overrides the wait_time method to provide a linear waiting time.
+    class ConstantWait < WaitStrategy
+      alias wait_time wait_interval
+    end
+  end
+end

data/lib/jay_api/abstract/geometric_wait.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'wait_strategy'
+
+module JayAPI
+  module Abstract
+    # A geometric wait strategy implementation of the WaitStrategy abstract class.
+    # This strategy uses a geometrically increasing wait interval between retries.
+    # The wait interval is exponentially increased based on the number of attempts made,
+    # making it suitable for scenarios where a rapidly increasing delay is preferred.
+    #
+    # Inherits from WaitStrategy and overrides the wait_time method to provide a geometrically increasing waiting time.
+    class GeometricWait < WaitStrategy
+      private
+
+      attr_writer :calls_count
+
+      # Determines the time to wait before the next retry in a geometric manner.
+      # The wait time increases exponentially with each call, calculated as wait_interval
+      # raised to the power of call number.
+      # @return [Integer] The exponentially increasing time to wait in seconds.
+      def wait_time
+        self.calls_count += 1
+        wait_interval**calls_count
+      end
+
+      # Tracks the number of calls made to the wait_time method.
+      # This count is used to calculate the geometrically increasing wait time.
+      # @return [Integer] The number of times the wait_time method has been called.
+      def calls_count
+        @calls_count ||= 0
+      end
+    end
+  end
+end

data/lib/jay_api/abstract/wait_strategy.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module JayAPI
+  module Abstract
+    # Abstract base class for implementing different waiting strategies.
+    # This class provides a framework for implementing a strategy that dictates how long to wait
+    # before retrying an operation, typically used in situations where an operation might need to be
+    # retried multiple times (like network requests, etc.)
+    #
+    # @abstract Subclass and override {#wait_time} to implement a custom WaitStrategy.
+    class WaitStrategy
+      attr_reader :wait_interval
+
+      # @param [Integer] wait_interval The initial time to wait before retrying.
+      # @param [Logging::Logger] logger The logger to be used for logging wait times, defaults to stdout.
+      def initialize(wait_interval:, logger: nil)
+        @wait_interval = wait_interval
+        @logger = logger || Logging.logger($stdout)
+      end
+
+      # Executes the wait strategy.
+      # Logs the waiting time and pauses the execution for the determined wait time.
+      def wait
+        wait_time.tap do |wait_time|
+          logger.info("Sleeping: #{format('%.2f', wait_time)} s")
+          Kernel.sleep(wait_time)
+        end
+      end
+
+      private
+
+      attr_reader :logger
+
+      # Determines the time to wait before the next retry.
+      # This method must be implemented by subclasses.
+      # @raise [NotImplementedError] if the method is not overridden in a subclass.
+      # @return [Integer] The time to wait in seconds.
+      def wait_time
+        raise(NotImplementedError, "#{self.class} must implement the #{__method__} method")
+      end
+    end
+  end
+end

data/lib/jay_api/configuration.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/hash/indifferent_access'
+
+require 'erb'
+require 'forwardable'
+require 'ostruct'
+require 'yaml'
+
+require_relative 'errors/configuration_error'
+
+module JayAPI
+  # Hold the configuration for Jay's API
+  class Configuration < OpenStruct
+    extend Forwardable
+
+    def_delegators :deep_to_h, :with_indifferent_access
+
+    # Loads the configuration from the given file.
+    # @param [String] file_name The file from which to load the configuration.
+    # @return [JayAPI::Configuration] The configuration for Jay's API.
+    # @raise [Errno::ENOENT] If the given file cannot be found.
+    # @raise [Psych::DisallowedClass] If the YAML contains a class other than
+    #   Symbol
+    def self.from_file(file_name)
+      from_string(File.read(file_name))
+    end
+
+    # Loads the configuration from the given YAML string.
+    # @param [String] yaml The YAML string containing the configuration.
+    # @return [JayAPI::Configuration] The configuration for Jay's API
+    # @raise [Psych::DisallowedClass] If the YAML contains a class other than
+    #   Symbol
+    def self.from_string(yaml)
+      yaml = ERB.new(yaml).result
+      config = YAML.safe_load(yaml, permitted_classes: [Symbol])
+
+      unless config.is_a?(Hash)
+        raise JayAPI::Errors::ConfigurationError.new(
+          "Jay's configuration should be a set of key-value pairs.", yaml
+        )
+      end
+
+      from_hash(config)
+    end
+
+    class << self
+      private
+
+      # Creates an instance of the class by parsing the given Hash.
+      # Nested hashes are recursively parsed an new instances of the class are
+      # created from them.
+      # @param [Hash] hash The hash with the data.
+      # @return [JayAPI::Configuration] An instance of the class created out of
+      #   the given Hash.
+      def from_hash(hash)
+        new.tap do |configuration|
+          hash.symbolize_keys.each do |key, value|
+            configuration[key] = parsed_value(value)
+          end
+        end
+      end
+
+      # Takes a value and parses it in accordance to its type.
+      # @param [Object] value The value to parse.
+      # @return [JayAPI::Configuration, Array, Object] The parsed value.
+      def parsed_value(value)
+        case value
+        when Hash
+          from_hash(value)
+        when Array
+          value.map { |item| parsed_value(item) }
+        else
+          value
+        end
+      end
+    end
+
+    # Recursively converts the receiver into a standard Hash
+    # @return [Hash] The result of the conversion.
+    def deep_to_h
+      to_h { |key, value| [key, value_for_h(value)] }
+    end
+
+    # @return [String] The configuration in the YAML format
+    def to_yaml
+      YAML.dump(deep_to_h.deep_stringify_keys)
+    end
+
+    private
+
+    # Takes a value and transforms it in accordance to its type.
+    # @param [Object] value The value to convert.
+    #   * JayAPI::Configuration objects are transformed to hashes recursively.
+    #   * Hashes are kept as Hashes but its values are transformed recursively.
+    #   * Arrays are transformed recursively.
+    #   * Any other value is left as is.
+    # @return [Object] The converted value (or the same value if the method
+    #   doesn't know how to convert it).
+    def value_for_h(value)
+      case value
+      when self.class
+        value.deep_to_h
+      when Hash
+        value.to_h { |hash_key, hash_value| [hash_key, value_for_h(hash_value)] }
+      when Array
+        value.map { |element| value_for_h(element) }
+      else
+        value
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/async.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'concurrent/promise'
+require 'forwardable'
+
+require_relative 'errors/query_execution_error'
+require_relative 'errors/query_execution_failure'
+require_relative 'tasks'
+
+module JayAPI
+  module Elasticsearch
+    # Provides functionality to perform asynchronous operations on an
+    # elasticsearch index. For more information:
+    # https://ruby-concurrency.github.io/concurrent-ruby/1.3.4/Concurrent
+    class Async
+      extend Forwardable
+
+      attr_reader :index
+
+      def_delegators :index, :index_name
+
+      # @param [JayAPI::Elasticsearch::Index] index The elasticsearch index on
+      #   which to execute asynchronous operations
+      def initialize(index)
+        @index = index
+      end
+
+      # Deletes asynchronously the documents matching the given query from the
+      # Index.
+      # @see JayAPI::Elasticsearch::Index#delete_by_query for more info
+      # @param [Hash] query The delete query
+      # @param [Integer, String] slices Number of slices to cut the operation
+      #   into for faster processing (i.e., run the operation in parallel). Use
+      #   "auto" to make elasticsearch decide how many slices to divide into
+      # @return [Concurrent::Promise] The eventual value returned from the
+      #   single completion of the delete operation
+      # @raise [Errors::QueryExecutionError] If executing the query results in
+      #   errors
+      # @raise [Errors::QueryExecutionFailure] If executing the query results in
+      #   failures
+      def delete_by_query(query, slices: 5)
+        Concurrent::Promise.execute do
+          async_response = index.delete_by_query(query, slices: slices, wait_for_completion: false)
+          result = tasks.by_id(async_response[:task])
+          validate_result(result)
+          result
+        end
+      end
+
+      private
+
+      # @param [Hash] result The operation result to be validated
+      # @raise [Errors::QueryExecutionError] If executing the query results in
+      #   errors
+      # @raise [Errors::QueryExecutionFailure] If executing the query results in
+      #   failures
+      def validate_result(result)
+        raise Errors::QueryExecutionError, "Errors on index '#{index_name}':\n #{result[:error]}" if result[:error]
+
+        failures = result&.dig(:response, :failures)
+        return if failures.nil? || failures.empty?
+
+        raise Errors::QueryExecutionFailure, "Failures on index '#{index_name}':\n #{failures}"
+      end
+
+      # @return [JayAPI::Elasticsearch::Tasks]
+      def tasks
+        @tasks ||= JayAPI::Elasticsearch::Tasks.new(client: index.client)
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/batch_counter.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module JayAPI
+  module Elasticsearch
+    # Manages and tracks the current batch within the QueryResults context. This class is responsible for
+    # keeping track of the current batch start position and calculating the start position for the next batch
+    # based on the batch size.
+    class BatchCounter
+      # The start of the batch to default to, if no other information is provided.
+      DEFAULT_START = 0
+
+      # @!attribute [r] batch_size
+      #   @return [Integer] The size of each batch as determined by the query or the default size
+      # @!attribute [r] start_current
+      #   @return [Integer] The starting index of the current batch
+      # @!attribute [r] start_next
+      #   @return [Integer] The calculated starting index of the next batch
+      attr_reader :batch_size, :start_current, :start_next
+
+      # Creates a new +BatchCounter+ object by either updating a copy of the *batch* instance with new values or
+      # creates a new instance if none exists.
+      # @param [BatchCounter, nil] batch An existing BatchCounter to update or nil to create a new one
+      # @param [Hash] query The Elasticsearch query containing the batch information
+      # @param [Integer] size The size of the current batch; also serves as a default batch size
+      # @return [BatchCounter] A new +BatchCounter+ created out of the given parameters.
+      def self.create_or_update(batch, query, size)
+        if batch
+          new(query, size, batch.start_next, batch.start_next + size, batch.batch_size)
+        else
+          new(query, size)
+        end
+      end
+
+      private
+
+      attr_reader :query, :size
+
+      # @param [Hash] query The Elasticsearch query which may contain :size and :from parameters
+      # @param [Integer] size The size of the batch; used as a default when no batch_size is provided
+      # @param [Integer, nil] start_current The starting index for the current batch; defaults to the query's :from
+      #   or DEFAULT_START
+      # @param [Integer, nil] start_next The starting index for the next batch; calculated from start_current and size
+      # @param [Integer, nil] batch_size The size of the batch; taken from the query's :size or the provided size
+      #   parameter
+      def initialize(query, size, start_current = nil, start_next = nil, batch_size = nil)
+        @query         = query.symbolize_keys
+        @size          = size
+
+        @start_current = start_current || fallback_start_current
+        @start_next    = start_next    || fallback_start_next
+        @batch_size    = batch_size    || fallback_batch_size
+      end
+
+      # Provides a default starting index for the next batch based on the current start index and batch size.
+      # @return [Integer] The calculated start index for the next batch
+      def fallback_start_next
+        start_current + size
+      end
+
+      # Provides a default starting index for the current batch from the query or a constant.
+      # @return [Integer] The starting index for the current batch
+      def fallback_start_current
+        query[:from] || DEFAULT_START
+      end
+
+      # Determines the batch size from the query or uses the provided size as a fallback.
+      # @return [Integer] The size of the batch
+      def fallback_batch_size
+        query[:size] || size
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/client.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require 'elasticsearch/api/namespace/tasks'
+require 'elasticsearch/transport/transport/errors'
+require 'faraday/error'
+
+require_relative '../abstract/connection'
+
+module JayAPI
+  module Elasticsearch
+    # The JayAPI wrapper class over the Elastisearch::Client object. It mirrors
+    # the object's API, but if one of the ERRORS is raised, this Wrapper class will
+    # rescue the error up to a few times and re-try the connection. This way the
+    # connection to Elasticsearch will be more robust.
+    class Client
+      # The errors that, if raised, must cause a retry of the connection.
+      ERRORS = [
+        ::Elasticsearch::Transport::Transport::ServerError,
+        Faraday::TimeoutError
+      ].freeze
+
+      # Subclasses of the +Elasticsearch::Transport::Transport::ServerError+
+      # for which a retry doesn't make sense.
+      NON_RETRIABLE_ERRORS = [
+        ::Elasticsearch::Transport::Transport::Errors::BadRequest,
+        ::Elasticsearch::Transport::Transport::Errors::Unauthorized,
+        ::Elasticsearch::Transport::Transport::Errors::Forbidden,
+        ::Elasticsearch::Transport::Transport::Errors::NotFound,
+        ::Elasticsearch::Transport::Transport::Errors::MethodNotAllowed,
+        ::Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge,
+        ::Elasticsearch::Transport::Transport::Errors::NotImplemented
+      ].freeze
+
+      attr_reader :transport_client, :logger, :max_attempts, :wait_strategy
+
+      # @param [Elasticsearch::Transport::Client] transport_client The Client
+      #   object that will be wrapped.
+      # @param [Logging::Logger] logger
+      # @param [Integer] max_attempts The maximum number of attempts that the connection shall be retried.
+      # @param [JayAPI::Elasticsearch::WaitStrategy] wait_strategy The waiting strategy for reconnections.
+      def initialize(transport_client, logger = nil, max_attempts:, wait_strategy:)
+        @transport_client = transport_client
+        @logger = logger || Logging.logger($stdout)
+        @max_attempts = max_attempts
+        @wait_strategy = wait_strategy
+      end
+
+      # Calls the Elasticsearch::Client's #index method and retries the connection a few times if
+      # a ServerError occurs.
+      # @see Elasticsearch::Client#index for information about the arguments and the returned value.
+      def index(**args)
+        retry_request { transport_client.index(**args) }
+      end
+
+      # Calls the Elasticsearch::Client's #search method and retries the connection a few times if
+      # a ServerError occurs.
+      # @see Elasticsearch::Client#index for information about the arguments and the returned value.
+      def search(**args)
+        retry_request { transport_client.search(**args) }
+      end
+
+      # Calls the Elasticsearch::Client's #bulk method and retries the connection a few times if
+      # a ServerError occurs.
+      # @see Elasticsearch::Client#index for information about the arguments and the returned value.
+      def bulk(**args)
+        retry_request { transport_client.bulk(**args) }
+      end
+
+      # Calls the +Elasticsearch::Client+'s #delete_by_query method forwarding
+      # the given parameters. If the request fails additional retries will be
+      # performed.
+      # @see Elasticsearch::Client#delete_by_query for information about the
+      #   arguments and the return value.
+      def delete_by_query(**args)
+        retry_request { transport_client.delete_by_query(**args) }
+      end
+
+      # Calls +Elasticsearch::Client+'s #tasks.get method forwarding the given
+      # parameters. If the request fails, additional retries will be performed.
+      # @see Elasticsearch::Client#tasks for more info about the arguments and
+      #   the return value.
+      def task_by_id(**args)
+        retry_request { transport_client.tasks.get(**args) }
+      end
+
+      private
+
+      # @param [Proc] block The block to execute.
+      # @yieldreturn [Object] Whatever the block returns
+      def retry_request(&block)
+        Abstract::Connection.new(max_attempts: max_attempts, wait_strategy: wait_strategy.dup, logger: logger)
+                            .retry(errors: ERRORS, except: NON_RETRIABLE_ERRORS, &block)
+      end
+    end
+  end
+end