search_flip 1.0.0

data/lib/search_flip/post_filterable.rb

@@ -0,0 +1,252 @@
+
+module SearchFlip
+  # The SearchFlip::PostFilterable mixin provides chainable methods like
+  # #post_where, #post_exists, #post_range, etc to add and apply search
+  # filters after aggregations have already been calculated.
+  #
+  # @example
+  #   query = ProductIndex.search("harry potter")
+  #
+  #   query = query.aggregate(price_ranges: {
+  #     range: {
+  #       field: "price",
+  #       ranges: [
+  #         { key: "range1", from: 0,  to: 20 },
+  #         { key: "range2", from: 20, to: 50 },
+  #         { key: "range3", from: 50, to: 100 }
+  #       ]
+  #     }
+  #   })
+  #
+  #   query = query.post_where(price: 20 ... 50)
+
+  module PostFilterable
+    def self.included(base)
+      base.class_eval do
+        attr_accessor :post_search_values, :post_must_values, :post_must_not_values, :post_should_values, :post_filter_values
+      end
+    end
+
+    # Adds a post query string query to the criteria while using AND as the
+    # default operator unless otherwise specified. Check out the
+    # ElasticSearch docs for further details.
+    #
+    # @example
+    #   CommentIndex.aggregate(:user_id).post_search("message:hello OR message:worl*")
+    #
+    # @param q [String] The query string query
+    #
+    # @param options [Hash] Additional options for the query string query, like
+    #   eg default_operator, default_field, etc.
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_search(q, options = {})
+      raise(SearchFlip::NotSupportedError) if SearchFlip.version.to_i < 2
+
+      fresh.tap do |criteria|
+        criteria.post_search_values = (post_search_values || []) + [query_string: { query: q, :default_operator => :AND }.merge(options)] if q.to_s.strip.length > 0
+      end
+    end
+
+    # Adds post filters to your criteria for the supplied hash composed of
+    # field-to-filter mappings which specify terms, term or range filters,
+    # depending on the type of the respective hash value, namely array, range
+    # or scalar type like Fixnum, String, etc.
+    #
+    # @example Array values
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where(id: [1, 2, 3], state: ["approved", "declined"])
+    #
+    # @example Range values
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where(created_at: Time.parse("2016-01-01") .. Time.parse("2017-01-01"))
+    #
+    # @example Scalar types
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where(id: 1, message: "hello world")
+    #
+    # @param hash [Hash] A field-to-filter mapping specifying filter values for
+    #   the respective fields
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_where(hash)
+      hash.inject(fresh) do |memo, (key, value)|
+        if value.is_a?(Array)
+          memo.post_filter terms: { key => value }
+        elsif value.is_a?(Range)
+          memo.post_filter range: { key => { gte: value.min, lte: value.max } }
+        else
+          memo.post_filter term: { key => value }
+        end
+      end
+    end
+
+    # Adds post filters to exclude documents in accordance to the supplied hash
+    # composed of field-to-filter mappings. Check out #post_where for further
+    # details.
+    #
+    # @example Array values
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where_not(id: [1, 2, 3])
+    #
+    # @example Range values
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where_not(created_at: Time.parse("2016-01-01") .. Time.parse("2017-01-01"))
+    #
+    # @example
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_where_not(state: "approved")
+    #
+    # @param hash [Hash] A field-to-filter mapping specifying filter values for the
+    #   respective fields
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_where_not(hash)
+      hash.inject(fresh) do |memo, (key,value)|
+        if value.is_a?(Array)
+          memo.post_must_not terms: { key => value }
+        elsif value.is_a?(Range)
+          memo.post_must_not range: { key => { gte: value.min, lte: value.max } }
+        else
+          memo.post_must_not term: { key => value }
+        end
+      end
+    end
+
+    # Adds raw post filter queries to the criteria.
+    #
+    # @example Raw post term filter query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_filter(term: { state: "new" })
+    #
+    # @example Raw post range filter query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_filter(range: { created_at: { gte: Time.parse("2016-01-01") }})
+    #
+    # @param args [Array, Hash] The raw filter query arguments
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_filter(*args)
+      fresh.tap do |criteria|
+        criteria.post_filter_values = (post_filter_values || []) + args
+      end
+    end
+
+    # Adds raw post must queries to the criteria.
+    #
+    # @example Raw post term must query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_must(term: { state: "new" })
+    #
+    # @example Raw post range must query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_must(range: { created_at: { gte: Time.parse("2016-01-01") }})
+    #
+    # @param args [Array, Hash] The raw must query arguments
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_must(*args)
+      fresh.tap do |criteria|
+        criteria.post_must_values = (post_must_values || []) + args
+      end
+    end
+
+    # Adds raw post must_not queries to the criteria.
+    #
+    # @example Raw post term must_not query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_must_not(term: { state: "new" })
+    #
+    # @example Raw post range must_not query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_must_not(range: { created_at: { gte: Time.parse("2016-01-01") }})
+    #
+    # @param args [Array, Hash] The raw must_not query arguments
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_must_not(*args)
+      fresh.tap do |criteria|
+        criteria.post_must_not_values = (post_must_not_values || []) + args
+      end
+    end
+
+    # Adds raw post should queries to the criteria.
+    #
+    # @example Raw post term should query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_should(term: { state: "new" })
+    #
+    # @example Raw post range should query
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_should(range: { created_at: { gte: Time.parse("2016-01-01") }})
+    #
+    # @param args [Array, Hash] The raw should query arguments
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_should(*args)
+      fresh.tap do |criteria|
+        criteria.post_should_values = (post_should_values || []) + args
+      end
+    end
+
+    # Adds a post range filter to the criteria without being forced to specify
+    # the left and right end of the range, such that you can eg simply specify
+    # lt, lte, gt and gte. For fully specified ranges, you can easily use
+    # #post_where, etc. Check out the ElasticSearch docs for further details
+    # regarding the range filter.
+    #
+    # @example
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_range(:created_at, gte: Time.parse("2016-01-01"))
+    #
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_range(:likes_count, gt: 10, lt: 100)
+    #
+    # @param field [Symbol, String] The field name to specify the range for
+    # @param options [Hash] The range filter specification, like lt, lte, etc
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_range(field, options = {})
+      post_filter range: { field => options }
+    end
+
+    # Adds a post exists filter to the criteria, which selects all documents
+    # for which the specified field has a non-null value.
+    #
+    # @example
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_exists(:notified_at)
+    #
+    # @param field [Symbol, String] The field that should have a non-null value
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_exists(field)
+      post_filter exists: { field: field }
+    end
+
+    # Adds a post exists not filter to the criteria, which selects all documents
+    # for which the specified field's value is null.
+    #
+    # @example
+    #   query = CommentIndex.aggregate("...")
+    #   query = query.post_exists_not(:notified_at)
+    #
+    # @param field [Symbol, String] The field that should have a null value
+    #
+    # @return [SearchFlip::Criteria] A newly created extended criteria
+
+    def post_exists_not(field)
+      post_must_not exists: { field: field }
+    end
+  end
+end
+

data/lib/search_flip/response.rb

@@ -0,0 +1,319 @@
+
+module SearchFlip
+  # The SearchFlip::Response class wraps a raw SearchFlip response and
+  # decorates it with methods for aggregations, hits, records, pagination, etc.
+
+  class Response
+    extend Forwardable
+
+    attr_accessor :criteria, :response
+
+    # @api private
+    #
+    # Initializes a new response object for the provided criteria and raw
+    # ElasticSearch response.
+
+    def initialize(criteria, response)
+      self.criteria = criteria
+      self.response = response
+    end
+
+    # Returns the raw response, ie a hash derived from the ElasticSearch JSON
+    # response.
+    #
+    # @example
+    #   CommentIndex.search("hello world").raw_response
+    #   # => {"took"=>3, "timed_out"=>false, "_shards"=>"..."}
+    #
+    # @return [Hash] The raw response hash
+
+    def raw_response
+      response
+    end
+
+    # Returns the total number of results.
+    #
+    # @example
+    #   CommentIndex.search("hello world").total_entries
+    #   # => 13
+    #
+    # @return [Fixnum] The total number of results
+
+    def total_entries
+      hits["total"]
+    end
+
+    alias_method :total_count, :total_entries
+
+    # Returns whether or not the current page is the first page.
+    #
+    # @example
+    #   CommentIndex.paginate(page: 1).first_page?
+    #   # => true
+    #
+    #   CommentIndex.paginate(page: 2).first_page?
+    #   # => false
+    #
+    # @return [Boolean] Returns true if the current page is the
+    #   first page or false otherwise
+
+    def first_page?
+      current_page == 1
+    end
+
+    # Returns whether or not the current page is the last page.
+    #
+    # @example
+    #   CommentIndex.paginate(page: 100).last_page?
+    #   # => true
+    #
+    #   CommentIndex.paginate(page: 1).last_page?
+    #   # => false
+    #
+    # @return [Boolean] Returns true if the current page is the
+    #   last page or false otherwise
+
+    def last_page?
+      current_page == total_pages
+    end
+
+    # Returns whether or not the current page is out of range,
+    # ie. smaller than 1 or larger than #total_pages
+    #
+    # @example
+    #   CommentIndex.paginate(page: 1_000_000).out_of_range?
+    #   # => true
+    #
+    #   CommentIndex.paginate(page: 1).out_of_range?
+    #   # => false
+    #
+    # @return [Boolean] Returns true if the current page is out
+    #   of range
+
+    def out_of_range?
+      current_page < 1 || current_page > total_pages
+    end
+
+    # Returns the current page number, useful for pagination.
+    #
+    # @example
+    #   CommentIndex.search("hello world").paginate(page: 10).current_page
+    #   # => 10
+    #
+    # @return [Fixnum] The current page number
+
+    def current_page
+      1 + (criteria.offset_value_with_default / criteria.limit_value_with_default.to_f).ceil
+    end
+
+    # Returns the number of total pages for the current pagination settings, ie
+    # per page/limit settings.
+    #
+    # @example
+    #   CommentIndex.search("hello world").paginate(per_page: 60).total_pages
+    #   # => 5
+    #
+    # @return [Fixnum] The total number of pages
+
+    def total_pages
+      [(total_entries / criteria.limit_value_with_default.to_f).ceil, 1].max
+    end
+
+    # Returns the previous page number or nil if no previous page exists, ie if
+    # the current page is the first page.
+    #
+    # @example
+    #   CommentIndex.search("hello world").paginate(page: 2).previous_page
+    #   # => 1
+    #
+    #   CommentIndex.search("hello world").paginate(page: 1).previous_page
+    #   # => nil
+    #
+    # @return [Fixnum, nil] The previous page number
+
+    def previous_page
+      return nil if current_page <= 1
+      return total_pages if current_page > total_pages
+
+      current_page - 1
+    end
+
+    alias_method :prev_page, :previous_page
+
+    # Returns the next page number or nil if there is no next page, ie the
+    # current page is the last page.
+    #
+    # @example
+    #   CommentIndex.search("hello world").paginate(page: 2).next_page
+    #   # => 3
+    #
+    # @return [Fixnum, nil] The next page number
+
+    def next_page
+      return nil if current_page >= total_pages
+      return 1 if current_page < 1
+
+      return current_page + 1
+    end
+
+    # Returns the results, ie hits, wrapped in a SearchFlip::Result object
+    # which basically is a Hashie::Mash. Check out the Hashie docs for further
+    # details.
+    #
+    # @example
+    #   CommentIndex.search("hello world").results
+    #   # => [#<SearchFlip::Result ...>, ...]
+    #
+    # @return [Array] An array of results
+
+    def results
+      @results ||= hits["hits"].map { |hit| Result.new hit["_source"].merge(hit["highlight"] ? { highlight: hit["highlight"] } : {}) }
+    end
+
+    # Returns the named sugggetion, if a name is specified or alle suggestions.
+    #
+    # @example
+    #   query = CommentIndex.suggest(:suggestion, text: "helo", term: { field: "message" })
+    #   query.suggestions # => {"suggestion"=>[{"text"=>...}, ...]}
+    #
+    # @example Named suggestions
+    #   query = CommentIndex.suggest(:suggestion, text: "helo", term: { field: "message" })
+    #   query.suggestions(:sugestion).first["text"] # => "hello"
+    #
+    # @return [Hash, Array] The named suggestion or all suggestions
+
+    def suggestions(name = nil)
+      if name
+        response["suggest"][name.to_s].first["options"]
+      else
+        response["suggest"]
+      end
+    end
+
+    # Returns the hits returned by ElasticSearch.
+    #
+    # @example
+    #   CommentIndex.search("hello world").hits
+    #   # => {"total"=>3, "max_score"=>2.34, "hits"=>[{...}, ...]}
+    #
+    # @return [Hash] The hits returned by ElasticSearch
+
+    def hits
+      response["hits"]
+    end
+
+    # Returns the scroll id returned by ElasticSearch, that can be used in the
+    # following request to fetch the next batch of records.
+    #
+    # @example
+    #   CommentIndex.scroll(timeout: "1m").scroll_id #=> "cXVlcnlUaGVuRmV0Y2..."
+    #
+    # @return [String] The scroll id returned by ElasticSearch
+
+    def scroll_id
+      response["_scroll_id"]
+    end
+
+    # Returns the database records, usually ActiveRecord objects, depending on
+    # the ORM you're using. The records are sorted using the order returned by
+    # ElasticSearch.
+    #
+    # @example
+    #   CommentIndex.search("hello world").records # => [#<Comment ...>, ...]
+    #
+    # @return [Array] An array of database records
+
+    def records(options = {})
+      @records ||= begin
+        sort_map = ids.each_with_index.each_with_object({}) { |(id, index), hash| hash[id.to_s] = index }
+
+        scope.to_a.sort_by { |record| sort_map[criteria.target.record_id(record).to_s] }
+      end
+    end
+
+    # Builds and returns a scope for the array of ids in the current result set
+    # returned by ElasticSearch, including the eager load, preload and includes
+    # associations, if specified. A scope is eg an ActiveRecord::Relation,
+    # depending on the ORM you're using.
+    #
+    # @example
+    #   CommentIndex.preload(:user).scope # => #<Comment::ActiveRecord_Criteria:0x0...>
+    #
+    # @return The scope for the array of ids in the current result set
+
+    def scope
+      res = criteria.target.fetch_records(ids)
+
+      res = res.includes(*criteria.includes_values) if criteria.includes_values
+      res = res.eager_load(*criteria.eager_load_values) if criteria.eager_load_values
+      res = res.preload(*criteria.preload_values) if criteria.preload_values
+
+      res
+    end
+
+    # Returns the array of ids returned by ElasticSearch for the current result
+    # set, ie the ids listed in the hits section of the response.
+    #
+    # @example
+    #   CommentIndex.match_all.ids # => [20341, 12942, ...]
+    #
+    # @return The array of ids in the current result set
+
+    def ids
+      @ids ||= hits["hits"].map { |hit| hit["_id"] }
+    end
+
+    def_delegators :ids, :size, :count, :length
+
+    # Returns the response time in milliseconds of ElasticSearch specified in
+    # the took info of the response.
+    #
+    # @example
+    #   CommentIndex.match_all.took # => 6
+    #
+    # @return [Fixnum] The ElasticSearch response time in milliseconds
+
+    def took
+      response["took"]
+    end
+
+    # Returns a single or all aggregations returned by ElasticSearch, depending
+    # on whether or not a name is specified. If no name is specified, the raw
+    # aggregation hash is simply returned. Contrary, if a name is specified,
+    # only this aggregation is returned. Moreover, if a name is specified and
+    # the aggregation includes a buckets section, a post-processed aggregation
+    # hash is returned.
+    #
+    # @example All aggregations
+    #   CommentIndex.aggregate(:user_id).aggregations
+    #   # => {"user_id"=>{..., "buckets"=>[{"key"=>4922, "doc_count"=>1129}, ...]}
+    #
+    # @example Specific and post-processed aggregations
+    #   CommentIndex.aggregate(:user_id).aggregations(:user_id)
+    #   # => {4922=>1129, ...}
+    #
+    # @return [Hash] Specific or all aggregations returned by ElasticSearch
+
+    def aggregations(name = nil)
+      return response["aggregations"] || {} unless name
+
+      @aggregations ||= {}
+
+      key = name.to_s
+
+      return @aggregations[key] if @aggregations.key?(key)
+
+      @aggregations[key] =
+        if response["aggregations"].nil? || response["aggregations"][key].nil?
+          Result.new
+        elsif response["aggregations"][key]["buckets"].is_a?(Array)
+          response["aggregations"][key]["buckets"].each_with_object({}) { |bucket, hash| hash[bucket["key"]] = Result.new(bucket) }
+        elsif response["aggregations"][key]["buckets"].is_a?(Hash)
+          Result.new response["aggregations"][key]["buckets"]
+        else
+          Result.new response["aggregations"][key]
+        end
+    end
+  end
+end
+

data/lib/search_flip/result.rb

@@ -0,0 +1,12 @@
+
+module SearchFlip
+  # The SearchFlip::Result class basically is a hash wrapper that uses
+  # Hashie::Mash to provide convenient method access to the hash attributes.
+
+  class Result < Hashie::Mash
+    def self.disable_warnings?
+      true
+    end
+  end
+end
+

data/lib/search_flip/to_json.rb

@@ -0,0 +1,31 @@
+
+require "time"
+require "date"
+require "json"
+
+class Time
+  def to_json
+    iso8601(6).to_json
+  end
+end
+
+class Date
+  def to_json
+    iso8601.to_json
+  end
+end
+
+class DateTime
+  def to_json
+    iso8601(6).to_json
+  end
+end
+
+if defined?(ActiveSupport)
+  class ActiveSupport::TimeWithZone
+    def to_json
+      iso8601(6).to_json
+    end
+  end
+end
+

data/lib/search_flip/version.rb

@@ -0,0 +1,5 @@
+
+module SearchFlip
+  VERSION = "1.0.0"
+end
+

data/lib/search_flip.rb

@@ -0,0 +1,82 @@
+
+require "forwardable"
+require "http"
+require "hashie"
+require "oj"
+require "set"
+
+require "search_flip/version"
+require "search_flip/json"
+require "search_flip/http_client"
+require "search_flip/config"
+require "search_flip/bulk"
+require "search_flip/filterable"
+require "search_flip/post_filterable"
+require "search_flip/aggregatable"
+require "search_flip/aggregation"
+require "search_flip/criteria"
+require "search_flip/response"
+require "search_flip/result"
+require "search_flip/index"
+require "search_flip/model"
+
+module SearchFlip
+  class NotSupportedError < StandardError; end
+  class ConnectionError < StandardError; end
+
+  class ResponseError < StandardError
+    attr_reader :code, :body
+
+    def initialize(code:, body:)
+      @code = code
+      @body = body
+    end
+
+    def to_s
+      "#{self.class.name} (#{code}): #{body}"
+    end
+  end
+
+  # Uses the ElasticSearch Multi Search API to execute multiple search requests
+  # within a single request. Raises SearchFlip::ResponseError in case any
+  # errors occur.
+  #
+  # @example
+  #   SearchFlip.msearch [ProductIndex.match_all, CommentIndex.match_all]
+  #
+  # @param criterias [Array<SearchFlip::Criteria>] An array of search
+  #   queries to execute in parallel
+  #
+  # @return [Array<SearchFlip::Response>] An array of responses
+
+  def self.msearch(criterias)
+    payload = criterias.flat_map do |criteria|
+      [SearchFlip::JSON.generate(index: criteria.target.index_name_with_prefix, type: criteria.target.type_name), SearchFlip::JSON.generate(criteria.request)]
+    end
+
+    payload = payload.join("\n")
+    payload << "\n"
+
+    SearchFlip::HTTPClient.headers(accept: "application/json", content_type: "application/x-ndjson").post("#{SearchFlip::Config[:base_url]}/_msearch", body: payload).parse["responses"].map.with_index do |response, index|
+      SearchFlip::Response.new(criterias[index], response)
+    end
+  end
+
+  # Used to manipulate, ie add and remove index aliases. Raises an
+  # SearchFlip::ResponseError in case any errors occur.
+  #
+  # @example
+  #   ElasticSearch.post_aliases(actions: [
+  #     { remove: { index: "test1", alias: "alias1" }},
+  #     { add: { index: "test2", alias: "alias1" }}
+  #   ])
+  #
+  # @param payload [Hash] The raw request payload
+  #
+  # @return [SearchFlip::Response] The raw response
+
+  def self.aliases(payload)
+    SearchFlip::HTTPClient.headers(accept: "application/json").post("#{SearchFlip::Config[:base_url]}/_aliases", body: SearchFlip::JSON.generate(payload))
+  end
+end
+