jay_api 27.1.0

data/lib/jay_api/elasticsearch/client_factory.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'elasticsearch'
+require 'logging'
+
+require 'elasticsearch/transport/transport/errors'
+
+require_relative 'client'
+require_relative '../abstract/geometric_wait'
+require_relative '../abstract/constant_wait'
+
+module JayAPI
+  module Elasticsearch
+    # A factory class that creates an Elasticsearch Client object. More specifically, a JayAPI wrapper
+    # object over the Elasticsearch Client object.
+    class ClientFactory
+      # The default port for the Elasticsearch cluster
+      DEFAULT_ELASTICSEARCH_PORT = 9200
+
+      # Classes that define the available waiting strategies, used for deciding sleep time between
+      # the reconnections.
+      WAIT_STRATEGIES = {
+        geometric: JayAPI::Abstract::GeometricWait,
+        constant: JayAPI::Abstract::ConstantWait
+      }.freeze
+
+      # The maximum number of connection attempts to be made.
+      MAX_ATTEMPTS  = 4
+      # The default wait time to be passed to the wait strategy class.
+      WAIT_INTERVAL = 2
+
+      attr_reader :cluster_url, :port
+
+      # Creates a new instance of the class.
+      # @param [String] cluster_url The URL where the Elasticsearch service
+      #   is exposed.
+      # @param [Integer] port The port to use to connect to the
+      #   Elasticsearch instance (Needed only when different from the default
+      #   Elasticsearch port)
+      # @param [Logging::Logger] logger The logger object to use, if
+      #   none is given a new one will be created.
+      # @param [Hash] credentials
+      # @option credentials [String] :username The user name to use when
+      #   authenticating against the Elasticsearch instance.
+      # @option credentials [String] :password The password to use when
+      #   authenticating against the Elasticsearch instance.
+      # disabling :reek:ControlParameter
+      def initialize(cluster_url:, port: nil, logger: nil, **credentials)
+        @cluster_url = cluster_url
+        @port = port || DEFAULT_ELASTICSEARCH_PORT
+        @logger = logger || Logging.logger($stdout)
+        @username = credentials[:username]
+        @password = credentials[:password]
+      end
+
+      # Returns the current instance of the Elasticsearch client, or creates
+      # a new one.
+      # @param [Integer] max_attempts The maximum number of attempts that the connection
+      #   shall be retried.
+      # @param [Symbol] wait_strategy The waiting strategy for reconnections (:geometric or :constant).
+      # @param [Integer] wait_interval The wait interval for the wait strategy. The sleep time between
+      #   each connection will be:
+      #   * wait_interval with :constant wait strategy
+      #   * wait_interval**i with :geometric wait strategy (where i is the i'th re-try)
+      # @return [JayAPI::Elasticsearch::Client] The Elasticsearch client.
+      def create(max_attempts: MAX_ATTEMPTS, wait_strategy: :geometric, wait_interval: WAIT_INTERVAL)
+        JayAPI::Elasticsearch::Client.new(
+          ::Elasticsearch::Client.new(
+            hosts: [host],
+            log: false
+          ),
+          logger,
+          max_attempts: max_attempts,
+          wait_strategy: WAIT_STRATEGIES[wait_strategy].new(wait_interval: wait_interval, logger: logger)
+        )
+      end
+
+      private
+
+      attr_reader :logger, :username, :password
+
+      # @return [URI] A URI constructed by parsing the given +cluster_url+.
+      def cluster_uri
+        @cluster_uri = URI.parse(cluster_url)
+      end
+
+      # @return [Hash] A +Hash+ with the connection parameters for the
+      #   Elasticsearch instance.
+      def host
+        {}.tap do |host|
+          host[:host] = cluster_uri.host
+          host[:port] = port || DEFAULT_ELASTICSEARCH_PORT # Do not use cluster_uri.port, that will ALWAYS be defined.
+          host[:user] = username if username
+          host[:password] = password if password
+          host[:scheme] = cluster_uri.scheme
+        end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/elasticsearch_error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative '../../errors/error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when the ElasticSearch instance responds with
+      # an error.
+      class ElasticsearchError < JayAPI::Errors::Error; end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/end_of_query_results_error.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative 'elasticsearch_error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when an attempt is made to fetch more documents
+      # on a QueryResults class that has reached the end of the matched
+      # documents.
+      class EndOfQueryResultsError < ElasticsearchError
+        # :reek:ControlParameter (want to avoid the long string in as default value)
+        # Creates a new instance of the class with the specified message.
+        # @param [String] message The message to use, if none is given
+        #   the default message will be used.
+        def initialize(message = nil)
+          super(message || 'End of the query results reached')
+        end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/query_execution_error.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative '../../errors/error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when executing a query in Elasticsearch results
+      # in errors, i.e, when a query cannot be executed; they typically indicate
+      # fundamental problems with the request itself
+      class QueryExecutionError < JayAPI::Errors::Error
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/query_execution_failure.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../../errors/error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when executing a query in Elasticsearch results
+      # in failures, i.e., when a query can technically be processed, but
+      # encounters issues during execution; the query is completed but returns
+      # partial or problematic results. It does not necessarily stop the entire
+      # request from completing
+      class QueryExecutionFailure < JayAPI::Errors::Error
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/query_execution_timeout.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative '../../errors/error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when executing a query in Elasticsearch times out.
+      class QueryExecutionTimeout < JayAPI::Errors::Error
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/errors/search_after_error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'elasticsearch_error'
+
+module JayAPI
+  module Elasticsearch
+    module Errors
+      # An error to be raised when an issue arises while using Elasticsearch's
+      # 'search_after' parameter.
+      class SearchAfterError < ElasticsearchError; end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/index.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/json' # Needed because ActiveSupport 6 doesn't include it's own JSON module. 🤦
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/hash/keys'
+require 'elasticsearch'
+require 'logging'
+
+require_relative 'errors/elasticsearch_error'
+require_relative 'async'
+require_relative 'query_results'
+require_relative 'response'
+require_relative 'batch_counter'
+require_relative 'search_after_results'
+
+module JayAPI
+  module Elasticsearch
+    # Represents an Elasticsearch index. Allows data to be pushed to it one
+    # record at a time or in batches of the specified size.
+    class Index
+      attr_reader :client, :index_name, :batch_size
+
+      # Default type for documents indexed with the #index method.
+      DEFAULT_DOC_TYPE = 'nested'
+
+      # Supported document types (for the #index method)
+      SUPPORTED_TYPES = [DEFAULT_DOC_TYPE, nil].freeze
+
+      # :reek:ControlParameter (want to avoid the creating of the logger on method definition)
+      # Creates a new instance of the class.
+      # @param [JayAPI::Elasticsearch::Client] client The Elasticsearch Client object.
+      # @param [String] index_name The name of the Elasticsearch index.
+      # @param [Integer] batch_size The size of the batch. When this number of
+      #   items are pushed into the index they are flushed to the
+      #   Elasticsearch instance.
+      # @param [Logging::Logger, nil] logger The logger object to use, if
+      #   none is given a new one will be created.
+      def initialize(client:, index_name:, batch_size: 100, logger: nil)
+        @logger = logger || Logging.logger[self]
+
+        @client = client
+        @index_name = index_name
+        @batch_size = batch_size
+
+        @batch = []
+      end
+
+      # Pushes a record into the index. (This does not send the record to the
+      # Elasticsearch instance, only puts it into the send queue).
+      # @param [Hash] data The data to be pushed to the index.
+      def push(data)
+        @batch << { index: { _index: index_name, _type: 'nested', data: data } }
+        flush! if @batch.size >= batch_size
+      end
+
+      # Sends a record to the Elasticsearch instance right away.
+      # @param [Hash] data The data to be sent.
+      # @param [String, nil] type The type of the document. When set to +nil+
+      #   the decision is left to Elasticsearch's API. Which will normally
+      #   default to +_doc+.
+      # @return [Hash] A hash with information about the created document. An
+      #   example of such Hash is:
+      #
+      #  {
+      #    "_index" => "xyz01_unit_test",
+      #    "_type" => "nested",
+      #    "_id" => "SVY1mJEBQ5CNFZM8Lodt",
+      #    "_version" => 1,
+      #    "result" => "created",
+      #    "_shards" => { "total" => 2, "successful" => 1, "failed" => 0 },
+      #    "_seq_no" => 0,
+      #    "_primary_term" => 1
+      #  }
+      #
+      # For information on the contents of this Hash please see:
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#docs-index-api-response-body
+      def index(data, type: DEFAULT_DOC_TYPE)
+        raise ArgumentError, "Unsupported type: '#{type}'" unless SUPPORTED_TYPES.include?(type)
+
+        client.index index: index_name, type: type, body: data
+      end
+
+      # Performs a query on the index.
+      # For more information on how to build the query please refer to the
+      # Elasticsearch DSL query:
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+      # @param [Hash] query The query to perform.
+      # @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
+      # @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
+      # @return [JayAPI::Elasticsearch::QueryResults] The query results.
+      # @raise [Elasticsearch::Transport::Transport::ServerError] If the
+      #   query fails.
+      def search(query, batch_counter: nil, type: nil)
+        begin
+          response = Response.new(client.search(index: index_name, body: query))
+        rescue ::Elasticsearch::Transport::Transport::Errors::BadRequest
+          logger.error "The 'search' query is invalid: #{JSON.pretty_generate(query)}"
+          raise
+        end
+        query_results(query, response, batch_counter, type)
+      end
+
+      # Sends whatever is currently in the send queue to the Elasticsearch
+      # instance and clears the queue.
+      def flush
+        return unless @batch.any?
+
+        flush!
+      end
+
+      # Returns the number of elements currently on the send queue.
+      # @return [Integer] The number of items in the send queue.
+      def queue_size
+        @batch.size
+      end
+
+      # Delete the documents matching the given query from the Index.
+      # For more information on how to build the query please refer to the
+      # Elasticsearch DSL documentation:
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      # @param [Hash] query The delete query
+      # @param [Integer] slices Number of slices to cut the operation into for
+      #   faster processing (i.e., run the operation in parallel)
+      # @param [Boolean] wait_for_completion False if Elasticsearch should not
+      #   wait for completion and perform the request asynchronously, true if it
+      #   should wait for completion (i.e., run the operation asynchronously)
+      # @return [Hash] A Hash that details the results of the operation
+      # @example Returned Hash (with `wait_for_completion: true`):
+      #     {
+      #       took: 103,
+      #       timed_out: false,
+      #       total: 76,
+      #       deleted: 76,
+      #       batches: 1,
+      #       version_conflicts: 0,
+      #       noops: 0,
+      #       retries: { bulk: 0, search: 0 },
+      #       throttled_millis: 0,
+      #       requests_per_second: 1.0,
+      #       throttled_until_millis: 0,
+      #       failures: []
+      #     }
+      # @example Returned Hash (with `wait_for_completion: false`):
+      #     {
+      #       task: "B5oDyEsHQu2Q-wpbaMSMTg:577388264"
+      #     }
+      # @raise [Elasticsearch::Transport::Transport::ServerError] If the
+      #   query fails.
+      def delete_by_query(query, slices: nil, wait_for_completion: true)
+        request_params = { index: index_name, body: query }.tap do |params|
+          params.merge!(slices: slices) if slices
+          params.merge!(wait_for_completion: false) unless wait_for_completion
+        end
+
+        client.delete_by_query(**request_params).deep_symbolize_keys
+      end
+
+      # Deletes asynchronously the documents matching the given query from the
+      # Index.
+      # For more information on how to build the query please refer to the
+      # Elasticsearch DSL documentation:
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
+      # @param [Hash] query The delete query
+      # @param [Integer, String] slices Number of slices to cut the operation
+      #   into for faster processing (i.e., run the operation in parallel). Use
+      #   "auto" to make elasticsearch decide how many slices to divide into
+      # @return [Concurrent::Promise] The eventual value returned from the single
+      #   completion of the delete operation
+      def delete_by_query_async(query, slices: nil)
+        async.delete_by_query(query, slices: slices)
+      end
+
+      private
+
+      attr_reader :logger, :batch
+
+      # Flushes the current send queue to the Elasticsearch instance and
+      # clears the queue.
+      def flush!
+        logger.info "Pushing data to Elasticsearch (#{batch.size} records)..."
+        response = client.bulk body: batch
+
+        handle_errors(response) if response['errors']
+
+        logger.info 'Done'
+        @batch = []
+      end
+
+      # @param [Hash] query The elastic search query.
+      # @param [JayAPI::Elasticsearch::Response] response The response to the query.
+      # @param [JayAPI::Elasticsearch::BatchCounter, nil] batch_counter Object keeping track of batches.
+      # @param [Symbol, nil] type Type of query, at the moment either nil or :search_after.
+      # @return [QueryResults]
+      def query_results(query, response, batch_counter, type)
+        (type == :search_after ? SearchAfterResults : QueryResults).new(
+          index: self,
+          query: query,
+          response: response,
+          batch_counter: BatchCounter.create_or_update(batch_counter, query, response.size)
+        )
+      end
+
+      # Scans the Elasticsearch response in search for the first item that has
+      # en erroneous state and raises an error including the error details.
+      # @param [Hash] response The response returned by the Elasticsearch client.
+      # @raise [Errors::ElasticsearchError] Is always raised.
+      def handle_errors(response)
+        error_item = response['items'].find { |item| item['index']['error'] }
+
+        raise Errors::ElasticsearchError,
+              "An error occurred when pushing the data to Elasticsearch:\n#{error_item['index']['error'].inspect}"
+      end
+
+      # @return [JayAPI::Elasticsearch::Async]
+      def async
+        @async ||= JayAPI::Elasticsearch::Async.new(self)
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/query_builder/aggregations/aggregation.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'errors/aggregations_error'
+
+module JayAPI
+  module Elasticsearch
+    class QueryBuilder
+      class Aggregations
+        # Base class for all Elasticsearch aggregation types. For more
+        # information on what types of aggregations are available see:
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html
+        class Aggregation
+          attr_reader :name
+
+          # @param [String] name The name used by Elasticsearch to identify each
+          #   of the aggregations.
+          def initialize(name)
+            @name = name
+          end
+
+          # Creates nested aggregations inside the receiver.
+          # @yieldparam [JayAPI::Elasticsearch::QueryBuilder::Aggregations] The
+          #   receiver's nested aggregations object.
+          # @return [JayAPI::Elasticsearch::QueryBuilder::Aggregations, self] If
+          #   no block is given then the nested aggregations are returned
+          #   instead of yielded. If a block is given then the receiver is
+          #   returned.
+          def aggs(&block)
+            aggregations = self.aggregations ||= ::JayAPI::Elasticsearch::QueryBuilder::Aggregations.new
+            return aggregations unless block
+
+            block.call(aggregations)
+            self
+          end
+
+          # @raise [NotImplementedError] Is always raised. The child classes
+          #   **must** override the method.
+          def clone
+            raise NotImplementedError, "Please implement #{__method__} in #{self.class}"
+          end
+
+          protected
+
+          attr_accessor :aggregations
+
+          private
+
+          # @return [Hash] The Hash representation of the +Aggregation+.
+          #   Properly formatted for Elasticsearch.
+          def to_h(&block)
+            { name => block.call.merge(aggregations.to_h) } # nil.to_h -> {} and Aggregations#to_h = {} when empty.
+          end
+
+          # @param [String] aggregation The name of the aggregation that cannot
+          #   have nested aggregations
+          # @raise [JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError]
+          #   Is always raised..
+          def no_nested_aggregations(aggregation)
+            raise ::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError,
+                  "The #{aggregation} aggregation cannot have nested aggregations."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/query_builder/aggregations/avg.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'aggregation'
+
+module JayAPI
+  module Elasticsearch
+    class QueryBuilder
+      class Aggregations
+        # Represents an +avg+ aggregation in Elasticsearch.
+        # Information on this type of aggregation can be found here:
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-avg-aggregation.html
+        class Avg < ::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Aggregation
+          attr_reader :field, :missing
+
+          # @param [String] name The name used by Elasticsearch to identify each
+          #   of the aggregations.
+          # @param [String] field The field over which the average should be
+          #   calculated.
+          # @param [Float] missing The value to use for the documents where
+          #   +field+ is missing. If no value is provided for +missing+ these
+          #   documents are ignored,
+          def initialize(name, field:, missing: nil)
+            super(name)
+
+            @field = field
+            @missing = missing
+          end
+
+          # @raise [JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError]
+          #   Is always raised. The Avg aggregation cannot have nested aggregations.
+          def aggs
+            no_nested_aggregations('Avg')
+          end
+
+          # @return [self] A copy of the receiver.
+          def clone
+            self.class.new(name, field: field, missing: missing)
+          end
+
+          # @return [Hash] The Hash representation of the +Aggregation+.
+          #   Properly formatted for Elasticsearch.
+          def to_h
+            super do
+              {
+                avg: {
+                  field: field,
+                  missing: missing
+                }.compact
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/query_builder/aggregations/errors/aggregations_error.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../../../../errors/error'
+
+module JayAPI
+  module Elasticsearch
+    class QueryBuilder
+      class Aggregations
+        module Errors
+          # An error to be raised when aggregations are used or nested in an
+          # unsupported way.
+          class AggregationsError < ::JayAPI::Errors::Error; end
+        end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/query_builder/aggregations/errors.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative 'errors/aggregations_error'
+
+module JayAPI
+  module Elasticsearch
+    class QueryBuilder
+      class Aggregations
+        # Namespace for all the error classes related to Aggregations.
+        module Errors; end
+      end
+    end
+  end
+end

data/lib/jay_api/elasticsearch/query_builder/aggregations/filter.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/string/inflections'
+
+require_relative '../query_clauses'
+require_relative 'aggregation'
+require_relative 'errors/aggregations_error'
+
+module JayAPI
+  module Elasticsearch
+    class QueryBuilder
+      class Aggregations
+        # Represents a +filter+ aggregation in Elasticsearch.
+        # Information on this type of aggregation can be found here:
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-filter-aggregation.html
+        class Filter < ::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Aggregation
+          # @param [String] name The name used by Elasticsearch to identify each
+          #   of the aggregations.
+          # @yieldparam [JayAPI::Elasticsearch::QueryBuilder::QueryClauses] The
+          #   subquery for the +filter+ aggregation.
+          def initialize(name, &block)
+            super(name)
+
+            unless block
+              raise(::JayAPI::Elasticsearch::QueryBuilder::Aggregations::Errors::AggregationsError,
+                    "The #{self.class.name.demodulize} aggregation must be initialized with a block")
+            end
+
+            block.call(query)
+          end
+
+          # @return [self] A copy of the receiver.
+          def clone
+            # rubocop:disable Lint/EmptyBlock (The query will be assigned later)
+            copy = self.class.new(name) {}
+            # rubocop:enable Lint/EmptyBlock
+
+            copy.query = query.clone
+            copy.aggregations = aggregations.clone
+            copy
+          end
+
+          # @return [Hash] The Hash representation of the +Aggregation+.
+          #   Properly formatted for Elasticsearch.
+          def to_h
+            super do
+              {
+                filter: query.to_h
+              }
+            end
+          end
+
+          protected
+
+          attr_writer :query
+
+          # @return [JayAPI::Elasticsearch::QueryBuilder::QueryClauses] The
+          #   +filter+ aggregation's subquery.
+          def query
+            @query ||= ::JayAPI::Elasticsearch::QueryBuilder::QueryClauses.new
+          end
+        end
+      end
+    end
+  end
+end