elasticsearch_record 1.0.0

data/lib/elasticsearch_record/query.rb

@@ -0,0 +1,129 @@
+module ElasticsearchRecord
+  class Query
+    # STATUS CONSTANTS
+    STATUS_VALID  = :valid
+    STATUS_FAILED = :failed
+
+    # TYPE CONSTANTS
+    TYPE_UNDEFINED       = :undefined
+    TYPE_COUNT           = :count
+    TYPE_SEARCH          = :search
+    TYPE_MSEARCH         = :msearch
+    TYPE_SQL             = :sql
+    TYPE_CREATE          = :create
+    TYPE_UPDATE          = :update
+    TYPE_UPDATE_BY_QUERY = :update_by_query
+    TYPE_DELETE          = :delete
+    TYPE_DELETE_BY_QUERY = :delete_by_query
+
+    # includes valid types only
+    TYPES = [TYPE_COUNT, TYPE_SEARCH, TYPE_MSEARCH, TYPE_SQL, TYPE_CREATE, TYPE_UPDATE, TYPE_UPDATE_BY_QUERY, TYPE_DELETE, TYPE_DELETE_BY_QUERY].freeze
+
+    # includes reading types only
+    READ_TYPES = [TYPE_COUNT, TYPE_SEARCH, TYPE_MSEARCH, TYPE_SQL].freeze
+
+    # defines a query to be executed if the query fails - +(none)+ queries
+    # acts like the SQL-query "where('1=0')"
+    FAILED_SEARCH_BODY = { size: 0, query: { bool: { filter: [{ term: { _id: '_' } }] } } }.freeze
+
+    # defines special api gates to be used per type.
+    # if not defined it simply uses +[:core,self.type]+
+    GATES = {
+      TYPE_SQL => [:sql, :query]
+    }
+
+    # defines the index the query should be executed on
+    # @!attribute String
+    attr_reader :index
+
+    # defines the query type.
+    # @see TYPES
+    # @!attribute Symbol
+    attr_reader :type
+
+    # defines the query status.
+    # @see STATUSES
+    # @!attribute Symbol
+    attr_reader :status
+
+    # defines if the affected shards gets refreshed to make this operation visible to search
+    # @!attribute Boolean
+    attr_reader :refresh
+
+    # defines the query body - in most cases this is a hash
+    # @!attribute Hash
+    attr_reader :body
+
+    # defines the query arguments to be passed to the API
+    # @!attribute Hash
+    attr_reader :arguments
+
+    # defines the columns to assign from the query
+    # @!attribute Array
+    attr_reader :columns
+
+    def initialize(index: nil, type: TYPE_UNDEFINED, status: STATUS_VALID, body: nil, refresh: nil, arguments: {}, columns: [])
+      @index     = index
+      @type      = type
+      @status    = status
+      @refresh   = refresh
+      @body      = body
+      @arguments = arguments
+      @columns   = columns
+    end
+
+    # sets the failed status for this query.
+    # returns self
+    # @return [ElasticsearchRecord::Query]
+    def failed!
+      @status = STATUS_FAILED
+
+      self
+    end
+
+    # returns true, if the query is valid (e.g. index & type defined)
+    # @return [Boolean]
+    def valid?
+      # type mus be valid + index must be present (not required for SQL)
+      TYPES.include?(self.type) #&& (index.present? || self.type == TYPE_SQL)
+    end
+
+    # returns the API gate to be called to execute the query.
+    # each query type needs a different endpoint.
+    # @see Elasticsearch::API
+    # @return [Array<Symbol, Symbol>] - API gate [<namespace>,<action>]
+    def gate
+      GATES[self.type].presence || [:core, self.type]
+    end
+
+    # returns true if this is a write query
+    # @return [Boolean]
+    def write?
+      !READ_TYPES.include?(self.type)
+    end
+
+    # builds the final query arguments.
+    # Depends on the query status, index, body & refresh attributes.
+    # Also used possible PRE-defined arguments to be merged with those mentioned attributes.
+    # @return [Hash]
+    def query_arguments
+      # check for failed status
+      return { index: self.index, body: FAILED_SEARCH_BODY } if self.status == STATUS_FAILED
+
+      args           = @arguments.deep_dup
+
+      # set index, if present
+      args[:index]   = self.index if self.index.present?
+
+      # set body, if present
+      args[:body]    = self.body if self.body.present?
+
+      # set refresh, if defined (also includes false value)
+      args[:refresh] = self.refresh unless self.refresh.nil?
+
+      args
+    end
+
+    alias :to_query :query_arguments
+  end
+end

data/lib/elasticsearch_record/querying.rb

@@ -0,0 +1,90 @@
+module ElasticsearchRecord
+  module Querying
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      # define additional METHODS to be delegated to the Relation
+      # @see ::ActiveRecord::Querying::QUERYING_METHODS
+      ES_QUERYING_METHODS = [
+        :query,
+        :filter,
+        :must,
+        :must_not,
+        :should,
+        :aggregate,
+        :msearch
+      ].freeze # :nodoc:
+      delegate(*ES_QUERYING_METHODS, to: :all)
+
+      # finds records by sql, query-arguments or query-object.
+      #
+      # PLEASE NOTE: This method is used by different other methods:
+      # - ActiveRecord::Relation#exec_queries
+      # - ActiveRecord::StatementCache#execute
+      # - <directly on demand>
+      #
+      # We cannot rewrite all call-sources since this will mess up the whole logic end will end in other problems.
+      # So we check here what kind of query is provided and decide what to do.
+      #
+      # PLEASE NOTE: since ths is also used by +ActiveRecord::StatementCache#execute+ we cannot remove
+      # the unused params +preparable+.
+      # see @ ActiveRecord::Querying#find_by_sql
+      #
+      # @param [String, Hash, ElasticsearchRecord::Query] sql
+      # @param [Array] binds
+      # @param [nil] preparable
+      # @param [Proc] block
+      def find_by_sql(sql, binds = [], preparable: nil, &block)
+        query = case sql
+                when String # really find by SQL
+                  ElasticsearchRecord::Query.new(
+                    type: ElasticsearchRecord::Query::TYPE_SQL,
+                    body: { query: query_or_sql },
+                    # IMPORTANT: Always provide all columns
+                    columns: source_column_names)
+                when Hash
+                  ElasticsearchRecord::Query.new(
+                    type:      ElasticsearchRecord::Query::TYPE_SEARCH,
+                    arguments: sql,
+                    # IMPORTANT: Always provide all columns
+                    columns: source_column_names)
+                else
+                  sql
+                end
+
+        _load_from_sql(_query_by_sql(query, binds), &block)
+      end
+
+      # finds records by query arguments
+      def find_by_query(arguments, &block)
+        # build new query
+        query = ElasticsearchRecord::Query.new(
+          index:     table_name,
+          type:      ElasticsearchRecord::Query::TYPE_SEARCH,
+          arguments: arguments,
+          # IMPORTANT: Always provide all columns to prevent unknown attributes that should be nil ...
+          columns: source_column_names)
+
+        _load_from_sql(_query_by_sql(query), &block)
+      end
+
+      # executes a msearch by provided +RAW+ queries
+      def msearch(queries, async: false)
+        # build new msearch query
+        query = ElasticsearchRecord::Query.new(
+          index: table_name,
+          type:  ElasticsearchRecord::Query::TYPE_MSEARCH,
+          body:  queries.map { |q| { search: q } },
+          # IMPORTANT: Always provide all columns
+          columns: source_column_names)
+
+        connection.exec_query(query, "#{name} Msearch", async: async)
+      end
+
+      # execute query by msearch
+      def _query_by_msearch(queries, async: false)
+        connection.select_multiple(queries, "#{name} Msearch", async: async)
+      end
+    end
+  end
+end

data/lib/elasticsearch_record/relation/calculation_methods.rb

@@ -0,0 +1,155 @@
+module ElasticsearchRecord
+  module Relation
+    module CalculationMethods
+      # Count the records.
+      #
+      #   Person.count
+      #   => the total count of all people
+      #
+      #   Person.count(:age)
+      #   => returns the total count of all people whose age is present in database
+      def count(column_name = nil)
+        # fallback to default
+        return super() if block_given?
+
+        # reset column_name, if +:all+ was provided ...
+        column_name = nil if column_name == :all
+
+        # check for combined cases
+        if self.distinct_value && column_name
+          self.cardinality(column_name)
+        elsif column_name
+          where(:filter, { exists: { field: column_name } }).count
+        elsif self.group_values.any?
+          self.composite(*self.group_values)
+        elsif self.select_values.any?
+          self.composite(*self.select_values)
+        elsif limit_value == 0 # Shortcut when limit is zero.
+          return 0
+        elsif limit_value
+          # since total will be limited to 10000 results, we need to resolve the real values by a custom query.
+          # This query is called through +#select_count+.
+          #
+          # HINT: :__claim__ directly interacts with the query-object and sets a 'terminate_after' argument
+          # (see @ Arel::Collectors::ElasticsearchQuery#assign)
+          arel = spawn.unscope!(:offset, :limit, :order, :configure, :aggs).configure!(:__claim__, argument: { terminate_after: limit_value }).arel
+          klass.connection.select_count(arel, "#{klass.name} Count")
+        else
+          # since total will be limited to 10000 results, we need to resolve the real values by a custom query.
+          # This query is called through +#select_count+.
+          arel = spawn.unscope!(:offset, :limit, :order, :configure, :aggs)
+          klass.connection.select_count(arel, "#{klass.name} Count")
+        end
+      end
+
+      # A multi-value metrics aggregation that calculates one or more
+      # percentiles over numeric values extracted from the aggregated documents.
+      # Returns a hash with empty values (but keys still exists) if there is no row.
+      #
+      #   Person.percentiles(:year)
+      #   > {
+      #      "1.0" => 2016.0,
+      #      "5.0" => 2016.0,
+      #     "25.0" => 2016.0,
+      #     "50.0" => 2017.0,
+      #     "75.0" => 2017.0,
+      #     "95.0" => 2021.0,
+      #     "99.0" => 2022.0
+      #     }
+      # @param [Symbol, String] column_name
+      def percentiles(column_name)
+        calculate(:percentiles, column_name, node: :values)
+      end
+
+      # A multi-value metrics aggregation that calculates one or more
+      # percentile ranks over numeric values extracted from the aggregated documents.
+      #
+      # Percentile rank show the percentage of observed values which are below certain value.
+      # For example, if a value is greater than or equal to 95% of the observed values it is
+      # said to be at the 95th percentile rank.
+      #
+      #   Person.percentile_ranks(:year, [500,600])
+      #   > {
+      #      "1.0" => 2016.0,
+      #      "5.0" => 2016.0,
+      #     "25.0" => 2016.0,
+      #     "50.0" => 2017.0,
+      #     "75.0" => 2017.0,
+      #     "95.0" => 2021.0,
+      #     "99.0" => 2022.0
+      #     }
+      # @param [Symbol, String] column_name
+      # @param [Array] values
+      def percentile_ranks(column_name, values)
+        calculate(:percentiles, column_name, opts: { values: values }, node: :values)
+      end
+
+      # Calculates the cardinality on a given column. Returns +0+ if there's no row.
+      #
+      #   Person.cardinality(:age)
+      #   > 12
+      #
+      # @param [Symbol, String] column_name
+      def cardinality(column_name)
+        calculate(:cardinality, column_name)
+      end
+
+      # Calculates the average value on a given column. Returns +nil+ if there's no row. See #calculate for examples with options.
+      #
+      #   Person.average(:age) # => 35.8
+      #
+      # @param [Symbol, String] column_name
+      def average(column_name)
+        calculate(:avg, column_name)
+      end
+
+      # Calculates the minimum value on a given column. The value is returned
+      # with the same data type of the column, or +nil+ if there's no row.
+      #
+      #   Person.minimum(:age)
+      #   > 7
+      #
+      # @param [Symbol, String] column_name
+      def minimum(column_name)
+        calculate(:min, column_name)
+      end
+
+      # Calculates the maximum value on a given column. The value is returned
+      # with the same data type of the column, or +nil+ if there's no row. See
+      # #calculate for examples with options.
+      #
+      #   Person.maximum(:age) # => 93
+      #
+      # @param [Symbol, String] column_name
+      def maximum(column_name)
+        calculate(:max, column_name)
+      end
+
+      # Calculates the sum of values on a given column. The value is returned
+      # with the same data type of the column, +0+ if there's no row. See
+      # #calculate for examples with options.
+      #
+      #   Person.sum(:age) # => 4562
+      #
+      # @param [Symbol, String] column_name (optional)
+      def sum(column_name)
+        calculate(:sum, column_name)
+      end
+
+      # creates a aggregation with the provided metric (e.g. :sum) and column.
+      # returns the metric node (default: :value) from the aggregations result.
+      # @param [Symbol, String] metric
+      # @param [Symbol, String] column
+      # @param [Hash] opts - additional arguments that get merged with the metric definition
+      # @param [Symbol] node (default :value)
+      def calculate(metric, column, opts: {}, node: :value)
+        metric_key = "#{column}_#{metric}"
+
+        # spawn a new aggregation and return the aggs
+        response = aggregate(metric_key, { metric => { field: column }.merge(opts) }).aggregations
+
+        response[metric_key][node]
+      end
+    end
+  end
+end

data/lib/elasticsearch_record/relation/core_methods.rb

@@ -0,0 +1,64 @@
+module ElasticsearchRecord
+  module Relation
+    module CoreMethods
+      def instantiate_records(rows, &block)
+        # slurp the total value from the rows (rows = ElasticsearchRecord::Result)
+        @total = rows.is_a?(::ElasticsearchRecord::Result) ? rows.total : rows.length
+        super
+      end
+
+      # transforms the current relation into arel, compiles it to query and executes the query.
+      # returns the result object.
+      #
+      # PLEASE NOTE: This makes the query +immutable+ and raises a +ActiveRecord::ImmutableRelation+
+      # if you try to change it's values.
+      #
+      # PLEASE NOTE: resolving records _(instantiate)_ is never possible after calling this method!
+      #
+      # @return [ElasticsearchRecord::Result]
+      # @param [String] name - custom instrumentation name (default: 'Load')
+      def resolve(name = 'Load')
+        # this acts the same like +#_query_by_sql+ but we can customize the instrumentation name and
+        # do not store the records.
+        klass.connection.select_all(arel, "#{klass.name} #{name}")
+      end
+
+      # returns the query hash for the current relation
+      # @return [Hash]
+      def to_query
+        to_sql.query_arguments
+      end
+
+      # executes the elasticsearch msearch on the related klass
+      #
+      # @example
+      #   msearch([2020, 2019, 2018]).each{ |q, year| q.where!(year: year) }
+      #
+      # @param [Array] values - values to be yielded
+      # @param [nil, Symbol] response_type - optional type of search response (:took, :total , :hits , :aggregations , :length , :results, :each)
+      def msearch(values = nil, response_type = nil)
+        if values.nil?
+          arels = [arel]
+        else
+          arels = values.map { |value|
+            # spawn a new relation and return the arel-object
+            yield(spawn, value).arel
+          }
+        end
+
+        # returns a response object with multiple single responses
+        responses = klass._query_by_msearch(arels)
+
+        if response_type
+          responses.map(&response_type.to_sym)
+        else
+          responses
+        end
+      end
+    end
+  end
+end
+
+
+
+

data/lib/elasticsearch_record/relation/query_clause.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module ElasticsearchRecord # :nodoc:
+  module Relation
+    class QueryClause
+      delegate :any?, :empty?, to: :predicates
+
+      attr_reader :key
+
+      def initialize(key, predicates, opts = {})
+        @key       = key
+        @predicates = predicates
+        @opts       = opts
+      end
+
+      def hash
+        [self.class, key, predicates, opts].hash
+      end
+
+      def ast
+        [key, (predicates.one? ? predicates[0] : predicates), opts]
+      end
+
+      def +(other)
+        ::ElasticsearchRecord::Relation::QueryClause.new(key, predicates + other.predicates, opts.merge(other.opts))
+      end
+
+      def -(other)
+        ::ElasticsearchRecord::Relation::QueryClause.new(key, predicates - other.predicates, Hash[opts.to_a - other.opts.to_a])
+      end
+
+      def |(other)
+        ::ElasticsearchRecord::Relation::QueryClause.new(key, predicates | other.predicates, Hash[opts.to_a | other.opts.to_a])
+      end
+
+      protected
+
+      attr_reader :predicates
+      attr_reader :opts
+
+    end
+  end
+end

data/lib/elasticsearch_record/relation/query_clause_tree.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module ElasticsearchRecord # :nodoc:
+  module Relation
+    class QueryClauseTree
+      delegate :any?, :empty?, :key?, :each, to: :predicates
+
+      def self.empty
+        @empty ||= new({}).freeze
+      end
+
+      def initialize(predicates)
+        @predicates = predicates
+      end
+
+      def key
+        :tree
+      end
+
+      def hash
+        [self.class, predicates].hash
+      end
+
+      def ast
+        predicates.values.map(&:ast)
+      end
+
+      def merge(other)
+        dups = dupredicates
+
+        other.each do |key, values|
+          if dups.key?(key)
+            dups[key] = (dups[key] + values).uniq
+          else
+            dups[key] = values
+          end
+        end
+
+        QueryClauseTree.new(dups)
+      end
+
+      def +(other)
+        dups = dupredicates
+
+        if key?(other.key)
+          dups[other.key] += other
+        else
+          dups[other.key] = other
+        end
+
+        QueryClauseTree.new(dups)
+      end
+
+      def -(other)
+        dups = dupredicates
+
+        if key?(other.key)
+          dups[other.key] -= other
+          dups.delete(other.key) if dups[other.key].blank?
+        end
+
+        QueryClauseTree.new(dups)
+      end
+
+      def |(other)
+        dups = dupredicates
+
+        if key?(other.key)
+          dups[other.key] |= other
+        else
+          dups[other.key] = other
+        end
+
+        QueryClauseTree.new(dups)
+      end
+
+      def ==(other)
+        other.is_a?(::ElasticsearchRecord::Relation::QueryClauseTree) &&
+          predicates == other.predicates
+      end
+
+      protected
+
+      attr_reader :predicates
+
+      private
+
+      def dupredicates
+        # we only dup the hash - no need to dup the lower elements
+        predicates.dup
+      end
+    end
+  end
+end