elasticsearch-model 0.0.1 → 0.1.0.rc1

data/lib/elasticsearch/model/adapters/active_record.rb

@@ -0,0 +1,97 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # An adapter for ActiveRecord-based models
+      #
+      module ActiveRecord
+
+        Adapter.register self,
+                         lambda { |klass| !!defined?(::ActiveRecord::Base) && klass.ancestors.include?(::ActiveRecord::Base) }
+
+        module Records
+          # Returns an `ActiveRecord::Relation` instance
+          #
+          def records
+            sql_records = klass.where(id: ids)
+
+            # Re-order records based on the order from Elasticsearch hits
+            # by redefining `to_a`, unless the user has called `order()`
+            #
+            sql_records.instance_exec(response.response['hits']['hits']) do |hits|
+              define_singleton_method :to_a do
+                if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4
+                  self.load
+                else
+                  self.__send__(:exec_queries)
+                end
+                @records.sort_by { |record| hits.index { |hit| hit['_id'].to_s == record.id.to_s } }
+              end
+            end
+
+            sql_records
+          end
+
+          # Prevent clash with `ActiveSupport::Dependencies::Loadable`
+          #
+          def load
+            records.load
+          end
+
+          # Intercept call to the `order` method, so we can ignore the order from Elasticsearch
+          #
+          def order(*args)
+            sql_records = records.__send__ :order, *args
+
+            # Redefine the `to_a` method to the original one
+            #
+            sql_records.instance_exec do
+              define_singleton_method(:to_a) do
+                if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4
+                  self.load
+                else
+                  self.__send__(:exec_queries)
+                end
+                @records
+              end
+            end
+
+            sql_records
+          end
+        end
+
+        module Callbacks
+
+          # Handle index updates (creating, updating or deleting documents)
+          # when the model changes, by hooking into the lifecycle
+          #
+          # @see http://guides.rubyonrails.org/active_record_callbacks.html
+          #
+          def self.included(base)
+            base.class_eval do
+              after_commit lambda { __elasticsearch__.index_document  },  on: :create
+              after_commit lambda { __elasticsearch__.update_document },  on: :update
+              after_commit lambda { __elasticsearch__.delete_document },  on: :destroy
+            end
+          end
+        end
+
+        module Importing
+
+          # Fetch batches of records from the database
+          #
+          # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches
+          #
+          def __find_in_batches(options={}, &block)
+            find_in_batches(options) do |batch|
+              batch_for_bulk = batch.map { |a| { index: { _id: a.id, data: a.__elasticsearch__.as_indexed_json } } }
+              yield batch_for_bulk
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/adapters/default.rb

@@ -0,0 +1,44 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # The default adapter for models which haven't one registered
+      #
+      module Default
+
+        # Module for implementing methods and logic related to fetching records from the database
+        #
+        module Records
+
+          # Return the collection of records fetched from the database
+          #
+          # By default uses `MyModel#find[1, 2, 3]`
+          #
+          def records
+            klass.find(@ids)
+          end
+        end
+
+        # Module for implementing methods and logic related to hooking into model lifecycle
+        # (e.g. to perform automatic index updates)
+        #
+        # @see http://api.rubyonrails.org/classes/ActiveModel/Callbacks.html
+        module Callbacks
+          # noop
+        end
+
+        # Module for efficiently fetching records from the database to import them into the index
+        #
+        module Importing
+
+          # @abstract Implement this method in your adapter
+          #
+          def __find_in_batches(options={}, &block)
+            raise NotImplemented, "Method not implemented for default adapter"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/adapters/mongoid.rb

@@ -0,0 +1,90 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # An adapter for Mongoid-based models
+      #
+      # @see http://mongoid.org
+      #
+      module Mongoid
+
+        Adapter.register self,
+                         lambda { |klass| !!defined?(::Mongoid::Document) && klass.ancestors.include?(::Mongoid::Document) }
+
+        module Records
+
+          # Return a `Mongoid::Criteria` instance
+          #
+          def records
+            criteria = klass.where(:id.in => ids)
+
+            criteria.instance_exec(response.response['hits']['hits']) do |hits|
+              define_singleton_method :to_a do
+                self.entries.sort_by { |e| hits.index { |hit| hit['_id'].to_s == e.id.to_s } }
+              end
+            end
+
+            criteria
+          end
+
+          # Intercept call to sorting methods, so we can ignore the order from Elasticsearch
+          #
+          %w| asc desc order_by |.each do |name|
+            define_method name do |*args|
+              criteria = records.__send__ name, *args
+              criteria.instance_exec do
+                define_singleton_method(:to_a) { self.entries }
+              end
+
+              criteria
+            end
+          end
+        end
+
+        module Callbacks
+
+          # Handle index updates (creating, updating or deleting documents)
+          # when the model changes, by hooking into the lifecycle
+          #
+          # @see http://mongoid.org/en/mongoid/docs/callbacks.html
+          #
+          def self.included(base)
+            base.after_create  { |document| document.__elasticsearch__.index_document  }
+            base.after_update  { |document| document.__elasticsearch__.update_document }
+            base.after_destroy { |document| document.__elasticsearch__.delete_document }
+          end
+        end
+
+        module Importing
+
+          # Fetch batches of records from the database
+          #
+          # @see https://github.com/mongoid/mongoid/issues/1334
+          # @see https://github.com/karmi/retire/pull/724
+          #
+          def __find_in_batches(options={}, &block)
+            options[:batch_size] ||= 1_000
+            items = []
+
+            all.each do |item|
+              items << item
+
+              if items.length % options[:batch_size] == 0
+                batch_for_bulk = items.map { |a| { index: { _id: a.id.to_s, data: a.as_indexed_json } } }
+                yield batch_for_bulk
+                items = []
+              end
+            end
+
+            unless items.empty?
+              batch_for_bulk = items.map { |a| { index: { _id: a.id.to_s, data: a.as_indexed_json } } }
+              yield batch_for_bulk
+            end
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/callbacks.rb

@@ -0,0 +1,35 @@
+module Elasticsearch
+  module Model
+
+    # Allows to automatically update index based on model changes,
+    # by hooking into the model lifecycle.
+    #
+    # @note A blocking HTTP request is done during the update process.
+    #       If you need a more performant/resilient way of updating the index,
+    #       consider adapting the callbacks behaviour, and use a background
+    #       processing solution such as [Sidekiq](http://sidekiq.org)
+    #       or [Resque](https://github.com/resque/resque).
+    #
+    module Callbacks
+
+      # When included in a model, automatically injects the callback subscribers (`after_save`, etc)
+      #
+      # @example Automatically update Elasticsearch index when the model changes
+      #
+      #     class Article
+      #       include Elasticsearch::Model
+      #       include Elasticsearch::Model::Callbacks
+      #     end
+      #
+      #     Article.first.update_attribute :title, 'Updated'
+      #     #  SQL (0.3ms)  UPDATE "articles" SET "title" = ...
+      #     #  2013-11-20 15:08:52 +0100: POST http://localhost:9200/articles/article/1/_update ...
+      #
+      def self.included(base)
+        adapter = Adapter.from_class(base)
+        base.__send__ :include, adapter.callbacks_mixin
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/client.rb

@@ -0,0 +1,61 @@
+module Elasticsearch
+  module Model
+
+    # Contains an `Elasticsearch::Client` instance
+    #
+    module Client
+
+      module ClassMethods
+
+        # Get the client for a specific model class
+        #
+        # @example Get the client for `Article` and perform API request
+        #
+        #     Article.client.cluster.health
+        #     # => { "cluster_name" => "elasticsearch" ... }
+        #
+        def client client=nil
+          @client ||= Elasticsearch::Model.client
+        end
+
+        # Set the client for a specific model class
+        #
+        # @example Configure the client for the `Article` model
+        #
+        #     Article.client = Elasticsearch::Client.new host: 'http://api.server:8080'
+        #     Article.search ...
+        #
+        def client=(client)
+          @client = client
+        end
+      end
+
+      module InstanceMethods
+
+        # Get or set the client for a specific model instance
+        #
+        # @example Get the client for a specific record and perform API request
+        #
+        #     @article = Article.first
+        #     @article.client.info
+        #     # => { "name" => "Node-1", ... }
+        #
+        def client
+          @client ||= self.class.client
+        end
+
+        # Set the client for a specific model instance
+        #
+        # @example Set the client for a specific record
+        #
+        #     @article = Article.first
+        #     @article.client = Elasticsearch::Client.new host: 'http://api.server:8080'
+        #
+        def client=(client)
+          @client = client
+        end
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/importing.rb

@@ -0,0 +1,94 @@
+module Elasticsearch
+  module Model
+
+    # Provides support for easily and efficiently importing large amounts of
+    # records from the including class into the index.
+    #
+    # @see ClassMethods#import
+    #
+    module Importing
+
+      # When included in a model, adds the importing methods.
+      #
+      # @example Import all records from the `Article` model
+      #
+      #     Article.import
+      #
+      # @see #import
+      #
+      def self.included(base)
+        base.__send__ :extend, ClassMethods
+
+        adapter = Adapter.from_class(base)
+        base.__send__ :include, adapter.importing_mixin
+        base.__send__ :extend,  adapter.importing_mixin
+      end
+
+      module ClassMethods
+
+        # Import all model records into the index
+        #
+        # The method will pick up correct strategy based on the `Importing` module
+        # defined in the corresponding adapter.
+        #
+        # @param options [Hash] Options passed to the underlying `__find_in_batches`method
+        # @param block  [Proc] Optional block to evaluate for each batch
+        #
+        # @yield [Hash] Gives the Hash with the Elasticsearch response to the block
+        #
+        # @return [Fixnum] Number of errors encountered during importing
+        #
+        # @example Import all records into the index
+        #
+        #     Article.import
+        #
+        # @example Set the batch size to 100
+        #
+        #     Article.import batch_size: 100
+        #
+        # @example Process the response from Elasticsearch
+        #
+        #     Article.import do |response|
+        #       puts "Got " + response['items'].select { |i| i['index']['error'] }.size.to_s + " errors"
+        #     end
+        #
+        # @example Delete and create the index with appropriate settings and mappings
+        #
+        #    Article.import force: true
+        #
+        # @example Refresh the index after importing all batches
+        #
+        #    Article.import refresh: true
+        #
+        #
+        def import(options={}, &block)
+          errors = 0
+
+          if options.delete(:force)
+            self.create_index! force: true
+          end
+
+          refresh = options.delete(:refresh) || false
+
+          __find_in_batches(options) do |batch|
+            response = client.bulk \
+                         index:   index_name,
+                         type:    document_type,
+                         body:    batch
+
+            yield response if block_given?
+
+            errors += response['items'].map { |k, v| k.values.first['error'] }.compact.length
+          end
+
+          self.refresh_index! if refresh
+
+          return errors
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/elasticsearch/model/indexing.rb

@@ -0,0 +1,332 @@
+module Elasticsearch
+  module Model
+
+    # Provides the necessary support to set up index options (mappings, settings)
+    # as well as instance methods to create, update or delete documents in the index.
+    #
+    # @see ClassMethods#settings
+    # @see ClassMethods#mapping
+    #
+    # @see InstanceMethods#index_document
+    # @see InstanceMethods#update_document
+    # @see InstanceMethods#delete_document
+    #
+    module Indexing
+
+      # Wraps the [index settings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html#configuration-index-settings)
+      #
+      class Settings
+        attr_accessor :settings
+
+        def initialize(settings={})
+          @settings = settings
+        end
+
+        def to_hash
+          @settings
+        end
+
+        def as_json(options={})
+          to_hash
+        end
+      end
+
+      # Wraps the [index mappings](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping.html)
+      #
+      class Mappings
+        attr_accessor :options
+
+        def initialize(type, options={})
+          @type    = type
+          @options = options
+          @mapping = {}
+        end
+
+        def indexes(name, options = {}, &block)
+          @mapping[name] = options
+
+          if block_given?
+            @mapping[name][:type] ||= 'object'
+            properties = @mapping[name][:type] == 'multi_field' ? :fields : :properties
+
+            @mapping[name][properties] ||= {}
+
+            previous = @mapping
+            begin
+              @mapping = @mapping[name][properties]
+              self.instance_eval(&block)
+            ensure
+              @mapping = previous
+            end
+          end
+
+          # Set the type to `string` by default
+          #
+          @mapping[name][:type] ||= 'string'
+
+          self
+        end
+
+        def to_hash
+          { @type.to_sym => @options.merge( properties: @mapping ) }
+        end
+
+        def as_json(options={})
+          to_hash
+        end
+      end
+
+      module ClassMethods
+
+        # Defines mappings for the index
+        #
+        # @example Define mapping for model
+        #
+        #     class Article
+        #       mapping dynamic: 'strict' do
+        #         indexes :foo do
+        #           indexes :bar
+        #         end
+        #         indexes :baz
+        #       end
+        #     end
+        #
+        #     Article.mapping.to_hash
+        #
+        #     # => { :article =>
+        #     #        { :dynamic => "strict",
+        #     #          :properties=>
+        #     #            { :foo => {
+        #     #                :type=>"object",
+        #     #                :properties => {
+        #     #                  :bar => { :type => "string" }
+        #     #                }
+        #     #              }
+        #     #            },
+        #     #           :baz => { :type=> "string" }
+        #     #        }
+        #     #    }
+        #
+        # @example Define index settings and mappings
+        #
+        #     class Article
+        #       settings number_of_shards: 1 do
+        #         mappings do
+        #           indexes :foo
+        #         end
+        #       end
+        #     end
+        #
+        # @example Call the mapping method directly
+        #
+        #     Article.mapping(dynamic: 'strict') { indexes :foo, type: 'long' }
+        #
+        #     Article.mapping.to_hash
+        #
+        #     # => {:article=>{:dynamic=>"strict", :properties=>{:foo=>{:type=>"long"}}}}
+        #
+        # The `mappings` and `settings` methods are accessible directly on the model class,
+        # when it doesn't already defines them. Use the `__elasticsearch__` proxy otherwise.
+        #
+        def mapping(options={}, &block)
+          @mapping ||= Mappings.new(document_type, options)
+
+          if block_given?
+            @mapping.options.update(options)
+
+            @mapping.instance_eval(&block)
+            return self
+          else
+            @mapping
+          end
+        end; alias_method :mappings, :mapping
+
+        # Define settings for the index
+        #
+        # @example Define index settings
+        #
+        #     Article.settings(index: { number_of_shards: 1 })
+        #
+        #     Article.settings.to_hash
+        #
+        #     # => {:index=>{:number_of_shards=>1}}
+        #
+        def settings(settings={}, &block)
+          @settings ||= Settings.new(settings)
+
+          @settings.settings.update(settings) unless settings.empty?
+
+          if block_given?
+            self.instance_eval(&block)
+            return self
+          else
+            @settings
+          end
+        end
+
+        # Creates an index with correct name, automatically passing
+        # `settings` and `mappings` defined in the model
+        #
+        # @example Create an index for the `Article` model
+        #
+        #     Article.__elasticsearch__.create_index!
+        #
+        # @example Forcefully create (delete first) an index for the `Article` model
+        #
+        #     Article.__elasticsearch__.create_index! force: true
+        #
+        def create_index!(options={})
+          delete_index!(options) if options[:force]
+
+          unless ( self.client.indices.exists(index: self.index_name) rescue false )
+            begin
+              self.client.indices.create index: self.index_name,
+                                                           body: {
+                                                             settings: self.settings.to_hash,
+                                                             mappings: self.mappings.to_hash }
+            rescue Exception => e
+              unless e.class.to_s =~ /NotFound/ && options[:force]
+                STDERR.puts "[!!!] Error when creating the index: #{e.class}", "#{e.message}"
+              end
+            end
+          else
+          end
+        end
+
+        # Deletes the index with corresponding name
+        #
+        # @example Delete the index for the `Article` model
+        #
+        #     Article.__elasticsearch__.delete_index!
+        #
+        def delete_index!(options={})
+          begin
+            self.client.indices.delete index: self.index_name
+          rescue Exception => e
+            unless e.class.to_s =~ /NotFound/ && options[:force]
+              STDERR.puts "[!!!] Error when deleting the index: #{e.class}", "#{e.message}"
+            end
+          end
+        end
+
+        # Performs the "refresh" operation for the index (useful e.g. in tests)
+        #
+        # @example Refresh the index for the `Article` model
+        #
+        #     Article.__elasticsearch__.refresh_index!
+        #
+        # @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-refresh.html
+        #
+        def refresh_index!(options={})
+          begin
+            self.client.indices.refresh index: self.index_name
+          rescue Exception => e
+            unless e.class.to_s =~ /NotFound/ && options[:force]
+              STDERR.puts "[!!!] Error when refreshing the index: #{e.class}", "#{e.message}"
+            end
+          end
+        end
+      end
+
+      module InstanceMethods
+
+        def self.included(base)
+          # Register callback for storing changed attributes for models
+          # which implement `before_save` and `changed_attributes` methods
+          #
+          # @note This is typically triggered only when the module would be
+          #       included in the model directly, not within the proxy.
+          #
+          # @see #update_document
+          #
+          base.before_save do |instance|
+            instance.instance_variable_set(:@__changed_attributes,
+                                  Hash[ instance.changes.map { |key, value| [key, value.last] } ])
+          end if base.respond_to?(:before_save) && base.instance_methods.include?(:changed_attributes)
+        end
+
+        # Serializes the model instance into JSON (by calling `as_indexed_json`),
+        # and saves the document into the Elasticsearch index.
+        #
+        # @param options [Hash] Optional arguments for passing to the client
+        #
+        # @example Index a record
+        #
+        #     @article.__elasticsearch__.index_document
+        #     2013-11-20 16:25:57 +0100: PUT http://localhost:9200/articles/article/1 ...
+        #
+        # @return [Hash] The response from Elasticsearch
+        #
+        # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:index
+        #
+        def index_document(options={})
+          document = self.as_indexed_json
+
+          client.index(
+            { index: index_name,
+              type:  document_type,
+              id:    self.id,
+              body:  document }.merge(options)
+          )
+        end
+
+        # Deletes the model instance from the index
+        #
+        # @param options [Hash] Optional arguments for passing to the client
+        #
+        # @example Delete a record
+        #
+        #     @article.__elasticsearch__.delete_document
+        #     2013-11-20 16:27:00 +0100: DELETE http://localhost:9200/articles/article/1
+        #
+        # @return [Hash] The response from Elasticsearch
+        #
+        # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:delete
+        #
+        def delete_document(options={})
+          client.delete(
+            { index: index_name,
+              type:  document_type,
+              id:    self.id }.merge(options)
+          )
+        end
+
+        # Tries to gather the changed attributes of a model instance
+        # (via [ActiveModel::Dirty](http://api.rubyonrails.org/classes/ActiveModel/Dirty.html)),
+        # performing a _partial_ update of the document.
+        #
+        # When the changed attributes are not available, performs full re-index of the record.
+        #
+        # @param options [Hash] Optional arguments for passing to the client
+        #
+        # @example Update a document corresponding to the record
+        #
+        #     @article = Article.first
+        #     @article.update_attribute :title, 'Updated'
+        #     # SQL (0.3ms)  UPDATE "articles" SET "title" = ?...
+        #
+        #     @article.__elasticsearch__.update_document
+        #     # 2013-11-20 17:00:05 +0100: POST http://localhost:9200/articles/article/1/_update ...
+        #     # 2013-11-20 17:00:05 +0100: > {"doc":{"title":"Updated"}}
+        #
+        # @return [Hash] The response from Elasticsearch
+        #
+        # @see http://rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions:update
+        #
+        def update_document(options={})
+          if changed_attributes = self.instance_variable_get(:@__changed_attributes)
+            client.update(
+              { index: index_name,
+                type:  document_type,
+                id:    self.id,
+                body:  { doc: changed_attributes } }.merge(options)
+            )
+          else
+            index_document(options)
+          end
+        end
+      end
+
+    end
+  end
+end