agnostic_backend 0.9.4 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f379b563c7fa6fc0d4fbbef69222931eaaa0efc0
-  data.tar.gz: cedf4be31fc43e9576fdcb49586427fcffd67dc2
+  metadata.gz: a58464b8656e2798472a29b29e094aad2a3ad371
+  data.tar.gz: b719bf7b58574861ea9237a32db5b788e1813a1c
 SHA512:
-  metadata.gz: 2dc5e38d7662a707d082e275a89450157dad30aa518f9c0bc19bf6dc097d2b44b93baf397d5598e12f94ccc7d6fd4b378404adc3d85ea814046fa99fbbe23b31
-  data.tar.gz: d2ad6dbdae2102814a100a4588365d67ad0b8c04806c21145db590370b96b1cdc6a366d10ddc4ba0045961f786d71d954878bab3a7d4d27895f7d9510b80f4b5
+  metadata.gz: b4f4431dd6d69edd313476822b7a76cbf1f271969bb7c4a72ecc78f93143f7558c78243919598a9fbd821b1b608fb8334f5d1c2c472846bc32e5740d407c69a2
+  data.tar.gz: 19ef97db6ecb648f1b4ca63d4902bd5c51f1e084601e7f7c9560385656a1e454fa803c7e16e88d791ed44ab2d4d44e56e2c3098f6ee5b17c0a29e27ff6e545a3

data/.travis.yml

@@ -8,3 +8,4 @@ script: bundle exec rspec spec
 branches:
   only:
     - master
+    - lp/elasticsearch

data/Gemfile

@@ -2,3 +2,8 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in agnostic_backend.gemspec
 gemspec
+
+
+group :test do
+  gem 'webmock', '~> 1.20.4'
+end

data/README.md

@@ -9,18 +9,17 @@ Version](https://badge.fury.io/rb/agnostic_backend.svg)](https://badge.fury.io/r
 `agnostic_backend` is a gem that adds indexing and searching
 capabilities to Ruby objects for various backends.
 
-It includes two modules: `Indexable` and `Queryable`.
-`Indexable` provides indexing functionality by
-specifying a way to define which object attributes should be
-transformed in order to be eventually indexed to a remote backend
-store. `Queryable` provides search and retrieval functionality by
-specifying a generic query language that seamlessly maps to specific
-backend languages.
+It includes two modules: `Indexable` and `Queryable`.  `Indexable`
+provides indexing functionality by specifying a way to define which
+object attributes should be transformed in order to be eventually
+indexed to a remote backend store. `Queryable` provides search and
+retrieval functionality by specifying a generic query language that
+seamlessly maps to specific backend languages.
 
 In addition to these two modules, `agnostic_backend` supplies
-additional classes (`Indexer` and `Index`) to support
-configuration and transformation functionality for remote backends (such as
-elasticsearch, AWS Cloudsearch etc).
+additional classes (`Indexer` and `Index`) to support configuration
+and transformation functionality for remote backends (such as
+Elasticsearch, AWS Cloudsearch etc).
 
 Although the motivation and use case for the gem relates to
 `ActiveRecord` models, no assumption is made as to the classes to
@@ -28,9 +27,17 @@ which `Indexable` and `Queryable` can be included. The objective is to
 maximize the flexibility of clients with respect to the use cases they
 need to address.
 
-## Supported backends
+## Supported Backends
+
+`agnostic_backend` currently includes implementations for the
+following backends:
 
 * [AWS Cloudsearch](https://aws.amazon.com/cloudsearch/)
+* [elasticsearch](https://www.elastic.co/products/elasticsearch) [experimental]
+
+The gem also supports the indexing of a document to multiple backends
+(multicast indexing) in a seamless manner, namely by means of extra
+configuration (rather than extra code) from the client's part.
 
 ## Installation
 
@@ -261,10 +268,13 @@ For more information about `Queryable` check out
 
 ### Backends
 
-Currently, the gem includes one concrete backend implementation that
-talks to [AWS Cloudsearch](https://aws.amazon.com/cloudsearch/). New
-backends can be implemented by subclassing `AgnosticBackend::Index`
-and `AgnosticBackend::Indexer` (more on that later).
+Currently, the gem includes two concrete backend implementations: one
+that talks to [AWS Cloudsearch](https://aws.amazon.com/cloudsearch/)
+and one that talks to [elasticsearch](https://www.elastic.co/products/elasticsearch).
+
+New backends can be implemented by subclassing
+`AgnosticBackend::Index` and `AgnosticBackend::Indexer` (more on that
+later).
 
 #### The Index
 

data/agnostic_backend.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '>= 2.1.0' # for mandatory method keyword arguments
 
-  spec.add_runtime_dependency "activesupport", "~> 3"
+  spec.add_runtime_dependency "activesupport"
   spec.add_runtime_dependency "aws-sdk", "~> 2"
+  spec.add_runtime_dependency "faraday"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake", "~> 10"

data/lib/agnostic_backend.rb

@@ -1,4 +1,5 @@
 require 'agnostic_backend/version'
+require 'active_support'
 require 'active_support/core_ext'
 
 require 'agnostic_backend/utilities'
@@ -45,6 +46,9 @@ require 'agnostic_backend/cloudsearch/index_field'
 require 'agnostic_backend/cloudsearch/indexer'
 require 'agnostic_backend/cloudsearch/remote_index_field'
 
+Dir[File.dirname(__FILE__) + '/agnostic_backend/elasticsearch/*.rb'].each {|file| require file }
+Dir[File.dirname(__FILE__) + '/agnostic_backend/queryable/elasticsearch/*.rb'].each {|file| require file }
+
 module AgnosticBackend
 
 end

data/lib/agnostic_backend/cloudsearch/index.rb

@@ -9,16 +9,6 @@ module AgnosticBackend
                   :access_key_id,
                   :secret_access_key
 
-      def initialize(indexable_klass, **options)
-        super(indexable_klass)
-        @region = parse_option(options, :region)
-        @domain_name = parse_option(options, :domain_name)
-        @document_endpoint = parse_option(options, :document_endpoint)
-        @search_endpoint = parse_option(options, :search_endpoint)
-        @access_key_id = parse_option(options, :access_key_id)
-        @secret_access_key = parse_option(options, :secret_access_key)
-      end
-
       def indexer
         AgnosticBackend::Cloudsearch::Indexer.new(self)
       end
@@ -87,13 +77,15 @@ module AgnosticBackend
         end
       end
 
-      def parse_option(options, option_name)
-        if options.has_key?(option_name)
-          options[option_name]
-        else
-          raise "#{option_name} must be specified"
-        end
+      def parse_options
+        @region = parse_option(:region)
+        @domain_name = parse_option(:domain_name)
+        @document_endpoint = parse_option(:document_endpoint)
+        @search_endpoint = parse_option(:search_endpoint)
+        @access_key_id = parse_option(:access_key_id)
+        @secret_access_key = parse_option(:secret_access_key)
       end
+
     end
   end
-end
+end

data/lib/agnostic_backend/cloudsearch/indexer.rb

@@ -2,44 +2,50 @@ require 'aws-sdk'
 
 module AgnosticBackend
   module Cloudsearch
+
+    class PayloadLimitExceededError < StandardError ; end
+
     class Indexer < AgnosticBackend::Indexer
       include AgnosticBackend::Utilities
 
-      def initialize(index)
-        @index = index
-      end
+      MAX_PAYLOAD_SIZE_IN_BYTES = 4_500_000
 
-      def publish(document)
-        with_exponential_backoff Aws::CloudSearch::Errors::Throttling do
-          client.upload_documents(
-            documents: document,
-            content_type:'application/json'
-          )
-        end
+      def delete(document_id)
+        delete_all([document_id])
       end
 
-      def delete(*document_ids)
+      def delete_all(document_ids)
         documents = document_ids.map do |document_id|
           {"type" => 'delete',
            "id" => document_id}
         end
+        publish_all(documents)
+      end
 
+      private
+
+      def publish(document)
+        publish_all([document])
+      end
+
+      def publish_all(documents)
+        return if documents.empty?
+        payload = ActiveSupport::JSON.encode(documents)
+        raise PayloadLimitExceededError.new if payload_too_heavy? payload
         with_exponential_backoff Aws::CloudSearch::Errors::Throttling do
           client.upload_documents(
-            documents: convert_to_json(documents),
+            documents: payload,
             content_type:'application/json'
-
           )
         end
       end
 
-      private
-
       def client
         index.cloudsearch_domain_client
       end
 
       def prepare(document)
+        raise IndexingError.new "Document does not have an ID field" unless document["id"].present?
         document
       end
 
@@ -51,8 +57,7 @@ module AgnosticBackend
         document = convert_bool_values_to_string_in document
         document = date_format document
         document = add_metadata_to document
-        document = convert_document_into_array(document)
-        convert_to_json document
+        document
 
       end
 
@@ -72,13 +77,10 @@ module AgnosticBackend
         }
       end
 
-      def convert_to_json(transformed_document)
-        ActiveSupport::JSON.encode(transformed_document)
+      def payload_too_heavy?(payload)
+        payload.bytesize > MAX_PAYLOAD_SIZE_IN_BYTES
       end
 
-      def convert_document_into_array(document)
-        [document]
-      end
     end
   end
 end

data/lib/agnostic_backend/elasticsearch/client.rb

@@ -0,0 +1,50 @@
+require 'faraday'
+
+module AgnosticBackend
+  module Elasticsearch
+
+    class Client
+      attr_reader :endpoint
+
+      def initialize(endpoint:)
+        @endpoint = endpoint
+        @connection = Faraday::Connection.new(url: endpoint)
+      end
+
+      # returns an array of RemoteIndexFields (or nil)
+      def describe_index_fields(index_name, type)
+        response = send_request(:get, path: "#{index_name}/_mapping/#{type}")
+        if response.success?
+          body = ActiveSupport::JSON.decode(response.body) if response.body.present?
+          return if body.blank?
+          fields = body[index_name.to_s]["mappings"][type.to_s]["properties"]
+
+          fields.map do |field_name, properties|
+            properties = Hash[ properties.map{|k,v| [k.to_sym, v]} ]
+            type = properties.delete(:type)
+            AgnosticBackend::Elasticsearch::RemoteIndexField.new field_name, type, **properties
+          end
+        end
+      end
+
+      # sends an HTTP request to the ES server
+      # @body is taken to be either
+      # (a) a Hash (in which case it will be encoded to JSON), or
+      # (b) a string (in which case it will be assumed to contain JSON data)
+      # returns a Faraday::Response instance
+      def send_request(http_method, path: "", body: nil)
+        body = ActiveSupport::JSON.encode(body) if body.is_a? Hash
+        @connection.run_request(http_method.downcase.to_sym,
+                                path.to_s,
+                                body,
+                                default_headers)
+      end
+
+      private
+
+      def default_headers
+        {'Content-Type' => 'application/json'}
+      end
+    end
+  end
+end

data/lib/agnostic_backend/elasticsearch/index.rb

@@ -0,0 +1,67 @@
+module AgnosticBackend
+  module Elasticsearch
+    class Index < AgnosticBackend::Index
+
+      attr_reader :index_name,
+                  :type,
+                  :endpoint,
+                  :enable_all
+
+      def indexer
+        AgnosticBackend::Elasticsearch::Indexer.new(self)
+      end
+
+      def query_builder
+        AgnosticBackend::Queryable::Elasticsearch::QueryBuilder.new(self)
+      end
+
+      def schema
+        @schema ||= @indexable_klass.schema { |ftype| ftype }
+      end
+
+      def client
+        @client ||= AgnosticBackend::Elasticsearch::Client.new(endpoint: endpoint)
+      end
+
+      def configure
+        body = mappings(indexer.flatten(schema))
+        client.send_request(:put, path: "#{index_name}/_mapping/#{type}", body: body)
+      end
+
+      def create
+        client.send_request(:put, path: index_name)
+      end
+
+      def destroy!
+        client.send_request(:delete, path: index_name)
+      end
+
+      def exists?
+        response = client.send_request(:head, path: index_name)
+        response.success?
+      end
+
+      private
+
+      def mappings(flat_schema)
+        {
+          "_all" => { "enabled" => enable_all },
+          "properties" => index_fields(flat_schema).map{|field| field.definition}.reduce({}, &:merge)
+        }
+      end
+
+      def index_fields(flat_schema)
+        flat_schema.map do |field_name, field_type|
+          AgnosticBackend::Elasticsearch::IndexField.new(field_name, field_type)
+        end
+      end
+
+      def parse_options
+        @index_name = parse_option(:index_name)
+        @type = parse_option(:type)
+        @endpoint = parse_option(:endpoint)
+        @enable_all = parse_option(:enable_all, optional: true, default: false)
+      end
+    end
+  end
+end

data/lib/agnostic_backend/elasticsearch/index_field.rb

@@ -0,0 +1,45 @@
+module AgnosticBackend
+  module Elasticsearch
+    class IndexField
+
+      TYPE_MAPPINGS = {
+        AgnosticBackend::Indexable::FieldType::STRING => "string",
+        AgnosticBackend::Indexable::FieldType::STRING_ARRAY => "string",
+        AgnosticBackend::Indexable::FieldType::DATE => "date",
+        AgnosticBackend::Indexable::FieldType::INTEGER => "integer",
+        AgnosticBackend::Indexable::FieldType::DOUBLE => "double",
+        AgnosticBackend::Indexable::FieldType::BOOLEAN => "boolean",
+        AgnosticBackend::Indexable::FieldType::TEXT => "string",
+        AgnosticBackend::Indexable::FieldType::TEXT_ARRAY => "string",
+      }.freeze
+
+      attr_reader :name, :type
+
+      def initialize(name, type)
+        @name = name
+        @type = type
+      end
+
+      def analyzed?
+        (type.type == AgnosticBackend::Indexable::FieldType::TEXT) ||
+        (type.type == AgnosticBackend::Indexable::FieldType::TEXT_ARRAY)
+      end
+
+      def elasticsearch_type
+        @elasticsearch_type ||= TYPE_MAPPINGS[type.type]
+      end
+
+      def definition
+        {
+          name.to_s => {
+            "type" => elasticsearch_type
+          }.merge(analyzed_property)
+        }
+      end
+
+      def analyzed_property
+        analyzed? ? {} : { "index" => "not_analyzed" }
+      end
+    end
+  end
+end

data/lib/agnostic_backend/elasticsearch/indexer.rb

@@ -0,0 +1,62 @@
+module AgnosticBackend
+  module Elasticsearch
+    class Indexer < AgnosticBackend::Indexer
+      include AgnosticBackend::Utilities
+
+      private
+
+      def client
+        index.client
+      end
+
+      def publish(document)
+        client.send_request(:put,
+                            path: "/#{index.index_name}/#{index.type}/#{document["id"]}",
+                            body: document)
+      end
+
+      def publish_all(documents)
+        return if documents.empty?
+        response = client.send_request(:post,
+                                       path: "/#{index.index_name}/#{index.type}/_bulk",
+                                       body: convert_to_bulk_upload_string(documents))
+        body = ActiveSupport::JSON.decode(response.body) rescue {}
+        # if at least one indexing attempt fails, raise the red flag
+        raise IndexingError.new, "Error in bulk upload" if body["errors"]
+        response
+      end
+
+
+      def prepare(document)
+        raise IndexingError.new, "Document does not have an ID field" unless document["id"].present?
+        document
+      end
+
+      def transform(document)
+        return {} if document.empty?
+
+        document = flatten document
+        document = reject_blank_values_from document
+        document = format_dates_in document
+        document
+      end
+
+      def format_dates_in(document)
+        document.each do |k, v|
+          if v.is_a?(Time)
+            document[k] = v.utc.strftime("%Y-%m-%dT%H:%M:%SZ")
+          end
+        end
+      end
+
+      def convert_to_bulk_upload_string(documents)
+        documents.map do |document|
+          next if document.empty?
+          header = { "index" => {"_id" => document["id"]}}.to_json
+          document = ActiveSupport::JSON.encode transform(prepare(document))
+          "#{header}\n#{document}\n"
+        end.compact.join("\n")
+      end
+    end
+  end
+end

data/lib/agnostic_backend/elasticsearch/remote_index_field.rb

@@ -0,0 +1,31 @@
+module AgnosticBackend
+  module Elasticsearch
+
+    class RemoteIndexField
+
+      attr_reader :name, :type
+
+      def initialize(name, type, **args)
+        @name = name
+        @type = to_local type
+        @options = args
+      end
+
+      def method_missing(method_name)
+        if @options.has_key? method_name
+          @options[method_name]
+        else
+          super
+        end
+      end
+
+      private
+
+      def to_local(remote_type)
+        AgnosticBackend::Elasticsearch::IndexField::TYPE_MAPPINGS.find{|ltype, rtype| remote_type == rtype}.try(:first)
+      end
+
+    end
+
+  end
+end

data/lib/agnostic_backend/index.rb

@@ -1,8 +1,17 @@
 module AgnosticBackend
   class Index
 
-    def initialize(indexable_klass)
+    attr_reader :options
+
+    def initialize(indexable_klass, primary: true, **options)
       @indexable_klass = indexable_klass
+      @primary = primary
+      @options = options
+      parse_options
+    end
+
+    def primary?
+      @primary
     end
 
     def name
@@ -20,5 +29,19 @@ module AgnosticBackend
     def configure(new_schema = nil)
       raise NotImplementedError
     end
+
+    def parse_options
+      raise NotImplementedError
+    end
+
+    def parse_option(option_name, optional: false, default: nil)
+      if options.has_key?(option_name)
+        options[option_name]
+      elsif optional
+        default
+      else
+        raise "#{option_name} must be specified"
+      end
+    end
   end
-end
+end

data/lib/agnostic_backend/indexable/config.rb

@@ -3,20 +3,57 @@ module AgnosticBackend
 
     class Config
 
-      class ConfigEntry < Struct.new(:index_class, :options);
+      class Entry
+
+        attr_reader :index_class,
+                    :options
+
+        def initialize(index_class:, indexable_class:, primary: true, **options)
+          @index_class = index_class
+          @indexable_class = indexable_class
+          @primary = primary
+          @options = options
+        end
+
+        def primary?
+          @primary
+        end
+
+        def create_index
+          @index_class.new(@indexable_class, primary: @primary, **@options)
+        end
       end
 
+
       def self.indices
         @indices ||= {}
       end
 
       def self.configure_index(indexable_class, index_class, **options)
-        indices[indexable_class.name] = ConfigEntry.new index_class, options
+        indices[indexable_class.name] = [Entry.new(index_class: index_class,
+                                                   indexable_class: indexable_class,
+                                                   primary: true,
+                                                   **options)]
+      end
+
+      def self.configure_secondary_index(indexable_class, index_class, **options)
+        unless indices.has_key? indexable_class.name
+          raise "No primary index exists for class #{indexable_class.name}"
+        end
+        indices[indexable_class.name] << Entry.new(index_class: index_class,
+                                                   indexable_class: indexable_class,
+                                                   primary: false,
+                                                   **options)
       end
 
       def self.create_index_for(indexable_class)
-        entry = indices[indexable_class.name]
-        entry.index_class.try(:new, indexable_class, entry.options)
+        entry = indices[indexable_class.name].find(&:primary?)
+        entry.try(:create_index)
+      end
+
+      def self.create_indices_for(indexable_class, include_primary: true)
+        all = indices[indexable_class.name].map {|entry| entry.try(:create_index)}.compact
+        include_primary ? all : all.reject(&:primary?)
       end
 
     end

data/lib/agnostic_backend/indexable/indexable.rb

@@ -22,6 +22,11 @@ module AgnosticBackend
         AgnosticBackend::Indexable::Config.create_index_for(self)
       end
 
+      def create_indices(include_primary: true)
+        AgnosticBackend::Indexable::Config.create_indices_for(self,
+                                                              include_primary: include_primary)
+      end
+
       # establishes the convention for determining the index name from the class name
       def index_name(source=nil)
         (source.nil? ? name : source.to_s).split('::').last.underscore.pluralize
@@ -102,9 +107,10 @@ module AgnosticBackend
                             self.class :
                             AgnosticBackend::Indexable.indexable_class(index_name)
 
-        index = indexable_class.create_index
-        indexer = index.indexer
-        indexer.put(self)
+        indexable_class.create_indices.map do |index|
+          indexer = index.indexer
+          indexer.put(self)
+        end
       end
 
       def index_object(index_name)

data/lib/agnostic_backend/indexer.rb

@@ -1,4 +1,7 @@
 module AgnosticBackend
+
+  class IndexingError < StandardError; end
+
   class Indexer
 
     attr_reader :index
@@ -8,32 +11,43 @@ module AgnosticBackend
     end
 
     # Sends the specified document to the remote backend.
-    # This is a template method.
     # @param [Indexable] an Indexable object
-    # @returns [boolean] true if success, false if failure
-    # returns nil if no indexing attempt is made (e.g. generated document is empty)
     def put(indexable)
-      document = indexable.generate_document
-      return if document.blank?
-      begin
-        publish(transform(prepare(document)))
-        true
-      rescue => e
-        false
+      put_all([indexable])
+    end
+
+    # Sends the specified documents to the remote backend
+    # using bulk upload (if supported by the backend)
+    # @param [Indexable] an Indexable object
+    def put_all(indexables)
+      documents = indexables.map do |indexable|
+        transform(prepare(indexable.generate_document))
       end
+      documents.reject!(&:empty?)
+      publish_all(documents) unless documents.empty?
     end
 
-    # Deletes the specified document from the index, This is an abstract
-    # method which concrete index classes must implement in order to provide
-    # its functionality.
+    # Deletes the specified document from the index
     # @param [document_id] the document id of the indexed document
     def delete(document_id)
+      delete_all([document_id])
+    end
+
+    # Deletes the specified documents from the index.
+    # This is an abstract method which concrete index classes
+    # must implement in order to provide its functionality.
+    # @param [document_ids] an array of document ids
+    def delete_all(document_ids)
       raise NotImplementedError
     end
 
     private
 
     def publish(document)
+      publish_all([document])
+    end
+
+    def publish_all(documents)
       raise NotImplementedError
     end
 
@@ -45,4 +59,4 @@ module AgnosticBackend
       raise NotImplementedError
     end
   end
-end
+end

data/lib/agnostic_backend/queryable/elasticsearch/executor.rb

@@ -0,0 +1,40 @@
+module AgnosticBackend
+  module Queryable
+    module Elasticsearch
+      class Executor < AgnosticBackend::Queryable::Executor
+        include AgnosticBackend::Utilities
+
+        def execute
+          if scroll_cursor.present?
+            response = client.send_request(:post, path: "_search/scroll", body: params)
+          else 
+            response = client.send_request(:post, path: "#{index.index_name}/#{index.type}/_search", body: params)
+          end          
+          ResultSet.new(ActiveSupport::JSON.decode(response.body), query)
+        end
+
+        def to_s
+          params
+        end
+
+        def params
+          scroll_cursor.present? ? scroll_cursor : query.accept(visitor)
+        end
+
+        private
+
+        def client
+          index.client
+        end
+
+        def index
+          query.context.index
+        end
+
+        def scroll_cursor
+          scroll_cursor_expression.accept(visitor) if scroll_cursor_expression
+        end
+      end
+    end
+  end
+end

data/lib/agnostic_backend/queryable/elasticsearch/query.rb

@@ -0,0 +1,29 @@
+module AgnosticBackend
+  module Queryable
+    module Elasticsearch
+      class Query < AgnosticBackend::Queryable::Query
+
+        def initialize(base)
+          super
+          @executor = Executor.new(self, Visitor.new)
+        end
+
+        def execute
+          @executor.execute if valid?
+        end
+
+        def execute!
+          if valid?
+            @executor.execute
+          else
+            raise StandardError, errors
+          end
+        end
+
+        def to_s
+          @executor.to_s
+        end
+      end
+    end
+  end
+end

data/lib/agnostic_backend/queryable/elasticsearch/query_builder.rb

@@ -0,0 +1,13 @@
+module AgnosticBackend
+  module Queryable
+    module Elasticsearch
+      class QueryBuilder < AgnosticBackend::Queryable::QueryBuilder
+        private
+
+        def create_query(context)
+          AgnosticBackend::Queryable::Elasticsearch::Query.new(context)
+        end
+      end
+    end
+  end
+end

data/lib/agnostic_backend/queryable/elasticsearch/result_set.rb

@@ -0,0 +1,27 @@
+module AgnosticBackend
+  module Queryable
+    module Elasticsearch
+      class ResultSet < AgnosticBackend::Queryable::ResultSet
+        include AgnosticBackend::Utilities
+
+        def total_count
+          raw_results["hits"]["total"]
+        end
+
+        def scroll_cursor
+          raw_results["_scroll_id"]
+        end
+
+        private
+
+        def filtered_results
+          raw_results["hits"]["hits"].map{|h| h["fields"]}
+        end
+
+        def transform(result)
+          transform_nested_values(unflatten(result), Proc.new{|value| value.size > 1 ? value.split.join('|') : value.first})
+        end
+      end
+    end
+  end
+end

data/lib/agnostic_backend/queryable/elasticsearch/visitor.rb

@@ -0,0 +1,195 @@
+module AgnosticBackend
+  module Queryable
+    module Elasticsearch
+      class Visitor < AgnosticBackend::Queryable::Visitor
+
+        private
+
+        def visit_criteria_equal(subject)
+          { "term" => { visit(subject.attribute) => visit(subject.value) } }
+        end
+
+        def visit_criteria_not_equal(subject)
+          { "must_not" => visit_criteria_equal(subject) }
+        end
+
+        def visit_criteria_greater(subject)
+          {
+            "range" => {
+              visit(subject.attribute) => {
+                "gt" => visit(subject.value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_less(subject)
+          {
+            "range" => {
+              visit(subject.attribute) => {
+                "lt" => visit(subject.value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_greater_equal(subject)
+          {
+            "range" => {
+              visit(subject.attribute) => {
+                "gte" => visit(subject.value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_less_equal(subject)
+          {
+            "range" => {
+              visit(subject.attribute) => {
+                "lte" => visit(subject.value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_greater_and_less(subject)
+          {
+            "range" => {
+              visit(subject.attribute) => {
+                "gt" => visit(subject.left_value),
+                "lt" => visit(subject.right_value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_greater_equal_and_less(subject)
+           {
+            "range" => {
+              visit(subject.attribute) => {
+                "gte" => visit(subject.left_value),
+                "lt" => visit(subject.right_value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_greater_and_less_equal(subject)
+           {
+            "range" => {
+              visit(subject.attribute) => {
+                "gt" => visit(subject.left_value),
+                "lte" => visit(subject.right_value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_greater_equal_and_less_equal(subject)
+           {
+            "range" => {
+              visit(subject.attribute) => {
+                "gte" => visit(subject.left_value),
+                "lte" => visit(subject.right_value)
+              }
+            }
+          }
+        end
+
+        def visit_criteria_contains(subject)
+          { "wildcard" => { visit(subject.attribute) => '*' + visit(subject.value) + '*'} }
+        end
+
+        def visit_criteria_starts(subject)
+          { "wildcard" => { visit(subject.attribute) => '*' +   visit(subject.value) } }
+        end
+
+        def visit_operations_not(subject)
+          {"bool" =>  { "must_not" => subject.operands.map{|o| visit(o)} } }
+        end
+
+        def visit_operations_and(subject)
+          {"bool" =>  { "must" => subject.operands.map{|o| visit(o)} } }
+        end
+
+        def visit_operations_or(subject)
+          {"bool" => { "should" => subject.operands.map{|o| visit(o)} } }
+        end
+
+        def visit_operations_ascending(subject)
+          { visit(subject.attribute) => {"order" => "asc" } }
+        end
+
+        def visit_operations_descending(subject)
+          { visit(subject.attribute) => {"order" => "desc" } }
+        end
+
+        def visit_query(subject)
+          result = {}
+          subject.children.each do |c|
+            result.merge!(visit(c)) do |key, v1, v2|
+              v1 + v2
+            end
+          end
+          result
+        end
+
+        def visit_expressions_where(subject)
+          { "filter" => visit(subject.criterion)  }
+        end
+
+        def visit_expressions_select(subject)
+          { "fields" => subject.projections.map{|c| visit(c)} } #return=
+        end
+
+        def visit_expressions_order(subject)
+          { "sort" => subject.qualifiers.map{|o| visit(o)} }
+        end
+
+        def visit_expressions_limit(subject)
+          { "size" =>  visit(subject.limit) }
+        end
+
+        def visit_expressions_offset(subject)
+          { "from" =>  visit(subject.offset) }
+        end
+
+        def visit_expressions_scroll_cursor(subject)
+          result =  { 
+            "scroll" => "1m"
+          }
+
+          result.merge!(
+            {
+              "scroll_id" => visit(subject.scroll_cursor)
+            }
+          ) if visit(subject.scroll_cursor) != 'initial'
+
+          result
+        end
+
+        def visit_attribute(subject)
+          subject.name.split('.').join('__')
+        end
+
+        def visit_value(subject)
+          case subject.type
+          when :integer
+            subject.value
+          when :date
+            "#{subject.value.utc.strftime("%Y-%m-%dT%H:%M:%SZ")}"
+          when :double
+            subject.value
+          when :boolean
+            subject.value
+          when :string,:string_array,:text,:text_array
+            subject.value
+          else
+            subject.value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/agnostic_backend/rspec/matchers.rb

@@ -63,7 +63,6 @@ module AgnosticBackend
       def define_index_field(*args)
         DefineIndexField.new(*args)
       end
-
     end
   end
 end

data/lib/agnostic_backend/version.rb

@@ -1,3 +1,3 @@
 module AgnosticBackend
-  VERSION = "0.9.4"
+  VERSION = "0.9.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agnostic_backend
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.9.8
 platform: ruby
 authors:
 - Iasonas Gavriilidis
@@ -10,22 +10,22 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-14 00:00:00.000000000 Z
+date: 2016-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -40,6 +40,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -139,6 +153,11 @@ files:
 - lib/agnostic_backend/cloudsearch/index_field.rb
 - lib/agnostic_backend/cloudsearch/indexer.rb
 - lib/agnostic_backend/cloudsearch/remote_index_field.rb
+- lib/agnostic_backend/elasticsearch/client.rb
+- lib/agnostic_backend/elasticsearch/index.rb
+- lib/agnostic_backend/elasticsearch/index_field.rb
+- lib/agnostic_backend/elasticsearch/indexer.rb
+- lib/agnostic_backend/elasticsearch/remote_index_field.rb
 - lib/agnostic_backend/index.rb
 - lib/agnostic_backend/indexable/config.rb
 - lib/agnostic_backend/indexable/content_manager.rb
@@ -157,6 +176,11 @@ files:
 - lib/agnostic_backend/queryable/criteria/criterion.rb
 - lib/agnostic_backend/queryable/criteria/ternary.rb
 - lib/agnostic_backend/queryable/criteria_builder.rb
+- lib/agnostic_backend/queryable/elasticsearch/executor.rb
+- lib/agnostic_backend/queryable/elasticsearch/query.rb
+- lib/agnostic_backend/queryable/elasticsearch/query_builder.rb
+- lib/agnostic_backend/queryable/elasticsearch/result_set.rb
+- lib/agnostic_backend/queryable/elasticsearch/visitor.rb
 - lib/agnostic_backend/queryable/executor.rb
 - lib/agnostic_backend/queryable/expressions/expression.rb
 - lib/agnostic_backend/queryable/operations/n_ary.rb