elasticsearch-model-queryable 0.1.5

data/gemfiles/4.0.gemfile

@@ -0,0 +1,11 @@
+# Usage:
+#
+# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle install
+# $ BUNDLE_GEMFILE=./gemfiles/4.0.gemfile bundle exec rake test:integration
+
+source 'https://rubygems.org'
+
+gemspec path: '../'
+
+gem 'activemodel',  '~> 4'
+gem 'activerecord', '~> 4'

data/lib/elasticsearch/model/adapter.rb

@@ -0,0 +1,145 @@
+module Elasticsearch
+  module Model
+
+    # Contains an adapter which provides OxM-specific implementations for common behaviour:
+    #
+    # * {Adapter::Adapter#records_mixin   Fetching records from the database}
+    # * {Adapter::Adapter#callbacks_mixin Model callbacks for automatic index updates}
+    # * {Adapter::Adapter#importing_mixin Efficient bulk loading from the database}
+    #
+    # @see Elasticsearch::Model::Adapter::Default
+    # @see Elasticsearch::Model::Adapter::ActiveRecord
+    # @see Elasticsearch::Model::Adapter::Mongoid
+    #
+    module Adapter
+
+      # Returns an adapter based on the Ruby class passed
+      #
+      # @example Create an adapter for an ActiveRecord-based model
+      #
+      #     class Article < ActiveRecord::Base; end
+      #
+      #     myadapter = Elasticsearch::Model::Adapter.from_class(Article)
+      #     myadapter.adapter
+      #     # => Elasticsearch::Model::Adapter::ActiveRecord
+      #
+      # @see Adapter.adapters The list of included adapters
+      # @see Adapter.register Register a custom adapter
+      #
+      def from_class(klass)
+        Adapter.new(klass)
+      end; module_function :from_class
+
+      # Returns registered adapters
+      #
+      # @see ::Elasticsearch::Model::Adapter::Adapter.adapters
+      #
+      def adapters
+        Adapter.adapters
+      end; module_function :adapters
+
+      # Registers an adapter
+      #
+      # @see ::Elasticsearch::Model::Adapter::Adapter.register
+      #
+      def register(name, condition)
+        Adapter.register(name, condition)
+      end; module_function :register
+
+      # Contains an adapter for specific OxM or architecture.
+      #
+      class Adapter
+        attr_reader :klass
+
+        def initialize(klass)
+          @klass = klass
+        end
+
+        # Registers an adapter for specific condition
+        #
+        # @param name      [Module] The module containing the implemented interface
+        # @param condition [Proc]   An object with a `call` method which is evaluated in {.adapter}
+        #
+        # @example Register an adapter for DataMapper
+        #
+        #     module DataMapperAdapter
+        #
+        #       # Implement the interface for fetching records
+        #       #
+        #       module Records
+        #         def records
+        #           klass.all(id: @ids)
+        #         end
+        #
+        #         # ...
+        #       end
+        #     end
+        #
+        #     # Register the adapter
+        #     #
+        #     Elasticsearch::Model::Adapter.register(
+        #       DataMapperAdapter,
+        #       lambda { |klass|
+        #         defined?(::DataMapper::Resource) and klass.ancestors.include?(::DataMapper::Resource)
+        #       }
+        #     )
+        #
+        def self.register(name, condition)
+          self.adapters[name] = condition
+        end
+
+        # Return the collection of registered adapters
+        #
+        # @example Return the currently registered adapters
+        #
+        #     Elasticsearch::Model::Adapter.adapters
+        #     # => {
+        #     #  Elasticsearch::Model::Adapter::ActiveRecord => #<Proc:0x007...(lambda)>,
+        #     #  Elasticsearch::Model::Adapter::Mongoid => #<Proc:0x007... (lambda)>,
+        #     # }
+        #
+        # @return [Hash] The collection of adapters
+        #
+        def self.adapters
+          @adapters ||= {}
+        end
+
+        # Return the module with {Default::Records} interface implementation
+        #
+        # @api private
+        #
+        def records_mixin
+          adapter.const_get(:Records)
+        end
+
+        # Return the module with {Default::Callbacks} interface implementation
+        #
+        # @api private
+        #
+        def callbacks_mixin
+          adapter.const_get(:Callbacks)
+        end
+
+        # Return the module with {Default::Importing} interface implementation
+        #
+        # @api private
+        #
+        def importing_mixin
+          adapter.const_get(:Importing)
+        end
+
+        # Returns the adapter module
+        #
+        # @api private
+        #
+        def adapter
+          @adapter ||= begin
+            self.class.adapters.find( lambda {[]} ) { |name, condition| condition.call(klass) }.first \
+            || Elasticsearch::Model::Adapter::Default
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/adapters/active_record.rb

@@ -0,0 +1,104 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # An adapter for ActiveRecord-based models
+      #
+      module ActiveRecord
+
+        Adapter.register self,
+                         lambda { |klass| !!defined?(::ActiveRecord::Base) && klass.ancestors.include?(::ActiveRecord::Base) }
+
+        module Records
+          # Returns an `ActiveRecord::Relation` instance
+          #
+          def records
+            sql_records = klass.where(klass.primary_key => ids)
+
+            # Re-order records based on the order from Elasticsearch hits
+            # by redefining `to_a`, unless the user has called `order()`
+            #
+            sql_records.instance_exec(response.response['hits']['hits']) do |hits|
+              define_singleton_method :to_a do
+                if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4
+                  self.load
+                else
+                  self.__send__(:exec_queries)
+                end
+                @records.sort_by { |record| hits.index { |hit| hit['_id'].to_s == record.id.to_s } }
+              end
+            end
+
+            sql_records
+          end
+
+          # Prevent clash with `ActiveSupport::Dependencies::Loadable`
+          #
+          def load
+            records.load
+          end
+
+          # Intercept call to the `order` method, so we can ignore the order from Elasticsearch
+          #
+          def order(*args)
+            sql_records = records.__send__ :order, *args
+
+            # Redefine the `to_a` method to the original one
+            #
+            sql_records.instance_exec do
+              define_singleton_method(:to_a) do
+                if defined?(::ActiveRecord) && ::ActiveRecord::VERSION::MAJOR >= 4
+                  self.load
+                else
+                  self.__send__(:exec_queries)
+                end
+                @records
+              end
+            end
+
+            sql_records
+          end
+        end
+
+        module Callbacks
+
+          # Handle index updates (creating, updating or deleting documents)
+          # when the model changes, by hooking into the lifecycle
+          #
+          # @see http://guides.rubyonrails.org/active_record_callbacks.html
+          #
+          def self.included(base)
+            base.class_eval do
+              after_commit lambda { __elasticsearch__.index_document  },  on: :create
+              after_commit lambda { __elasticsearch__.update_document },  on: :update
+              after_commit lambda { __elasticsearch__.delete_document },  on: :destroy
+            end
+          end
+        end
+
+        module Importing
+
+          # Fetch batches of records from the database (used by the import method)
+          #
+          #
+          # @see http://api.rubyonrails.org/classes/ActiveRecord/Batches.html ActiveRecord::Batches.find_in_batches
+          #
+          def __find_in_batches(options={}, &block)
+            named_scope = options.delete(:scope)
+            preprocess = options.delete(:preprocess)
+
+            scope = named_scope ? self.__send__(named_scope) : self
+
+            scope.find_in_batches(options) do |batch|
+              yield (preprocess ? self.__send__(preprocess, batch) : batch)
+            end
+          end
+
+          def __transform
+            lambda { |model|  { index: { _id: model.id, data: model.__elasticsearch__.as_indexed_json } } }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/adapters/default.rb

@@ -0,0 +1,50 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # The default adapter for models which haven't one registered
+      #
+      module Default
+
+        # Module for implementing methods and logic related to fetching records from the database
+        #
+        module Records
+
+          # Return the collection of records fetched from the database
+          #
+          # By default uses `MyModel#find[1, 2, 3]`
+          #
+          def records
+            klass.find(@ids)
+          end
+        end
+
+        # Module for implementing methods and logic related to hooking into model lifecycle
+        # (e.g. to perform automatic index updates)
+        #
+        # @see http://api.rubyonrails.org/classes/ActiveModel/Callbacks.html
+        module Callbacks
+          # noop
+        end
+
+        # Module for efficiently fetching records from the database to import them into the index
+        #
+        module Importing
+
+          # @abstract Implement this method in your adapter
+          #
+          def __find_in_batches(options={}, &block)
+            raise NotImplemented, "Method not implemented for default adapter"
+          end
+
+          # @abstract Implement this method in your adapter
+          #
+          def __transform
+            raise NotImplemented, "Method not implemented for default adapter"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/elasticsearch/model/adapters/mongoid.rb

@@ -0,0 +1,92 @@
+module Elasticsearch
+  module Model
+    module Adapter
+
+      # An adapter for Mongoid-based models
+      #
+      # @see http://mongoid.org
+      #
+      module Mongoid
+
+        Adapter.register self,
+                         lambda { |klass| !!defined?(::Mongoid::Document) && klass.ancestors.include?(::Mongoid::Document) }
+
+        module Records
+
+          # Return a `Mongoid::Criteria` instance
+          #
+          def records
+            criteria = klass.where(:id.in => ids)
+
+            criteria.instance_exec(response.response['hits']['hits']) do |hits|
+              define_singleton_method :to_a do
+                self.entries.sort_by { |e| hits.index { |hit| hit['_id'].to_s == e.id.to_s } }
+              end
+            end
+
+            criteria
+          end
+
+          # Intercept call to sorting methods, so we can ignore the order from Elasticsearch
+          #
+          %w| asc desc order_by |.each do |name|
+            define_method name do |*args|
+              criteria = records.__send__ name, *args
+              criteria.instance_exec do
+                define_singleton_method(:to_a) { self.entries }
+              end
+
+              criteria
+            end
+          end
+        end
+
+        module Callbacks
+
+          # Handle index updates (creating, updating or deleting documents)
+          # when the model changes, by hooking into the lifecycle
+          #
+          # @see http://mongoid.org/en/mongoid/docs/callbacks.html
+          #
+          def self.included(base)
+            base.after_create  { |document| document.__elasticsearch__.index_document  }
+            base.after_update  { |document| document.__elasticsearch__.update_document }
+            base.after_destroy { |document| document.__elasticsearch__.delete_document }
+          end
+        end
+
+        module Importing
+
+          # Fetch batches of records from the database
+          #
+          # @see https://github.com/mongoid/mongoid/issues/1334
+          # @see https://github.com/karmi/retire/pull/724
+          #
+          def __find_in_batches(options={}, &block)
+            options[:batch_size] ||= 1_000
+            items = []
+
+            all.each do |item|
+              items << item
+
+              if items.length % options[:batch_size] == 0
+                yield items
+                items = []
+              end
+            end
+
+            unless items.empty?
+              yield items
+            end
+          end
+
+          def __transform
+            lambda {|a|  { index: { _id: a.id.to_s, data: a.as_indexed_json } }}
+          end
+        end
+
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/callbacks.rb

@@ -0,0 +1,35 @@
+module Elasticsearch
+  module Model
+
+    # Allows to automatically update index based on model changes,
+    # by hooking into the model lifecycle.
+    #
+    # @note A blocking HTTP request is done during the update process.
+    #       If you need a more performant/resilient way of updating the index,
+    #       consider adapting the callbacks behaviour, and use a background
+    #       processing solution such as [Sidekiq](http://sidekiq.org)
+    #       or [Resque](https://github.com/resque/resque).
+    #
+    module Callbacks
+
+      # When included in a model, automatically injects the callback subscribers (`after_save`, etc)
+      #
+      # @example Automatically update Elasticsearch index when the model changes
+      #
+      #     class Article
+      #       include Elasticsearch::Model
+      #       include Elasticsearch::Model::Callbacks
+      #     end
+      #
+      #     Article.first.update_attribute :title, 'Updated'
+      #     #  SQL (0.3ms)  UPDATE "articles" SET "title" = ...
+      #     #  2013-11-20 15:08:52 +0100: POST http://localhost:9200/articles/article/1/_update ...
+      #
+      def self.included(base)
+        adapter = Adapter.from_class(base)
+        base.__send__ :include, adapter.callbacks_mixin
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/client.rb

@@ -0,0 +1,61 @@
+module Elasticsearch
+  module Model
+
+    # Contains an `Elasticsearch::Client` instance
+    #
+    module Client
+
+      module ClassMethods
+
+        # Get the client for a specific model class
+        #
+        # @example Get the client for `Article` and perform API request
+        #
+        #     Article.client.cluster.health
+        #     # => { "cluster_name" => "elasticsearch" ... }
+        #
+        def client client=nil
+          @client ||= Elasticsearch::Model.client
+        end
+
+        # Set the client for a specific model class
+        #
+        # @example Configure the client for the `Article` model
+        #
+        #     Article.client = Elasticsearch::Client.new host: 'http://api.server:8080'
+        #     Article.search ...
+        #
+        def client=(client)
+          @client = client
+        end
+      end
+
+      module InstanceMethods
+
+        # Get or set the client for a specific model instance
+        #
+        # @example Get the client for a specific record and perform API request
+        #
+        #     @article = Article.first
+        #     @article.client.info
+        #     # => { "name" => "Node-1", ... }
+        #
+        def client
+          @client ||= self.class.client
+        end
+
+        # Set the client for a specific model instance
+        #
+        # @example Set the client for a specific record
+        #
+        #     @article = Article.first
+        #     @article.client = Elasticsearch::Client.new host: 'http://api.server:8080'
+        #
+        def client=(client)
+          @client = client
+        end
+      end
+
+    end
+  end
+end

data/lib/elasticsearch/model/ext/active_record.rb

@@ -0,0 +1,14 @@
+# Prevent `MyModel.inspect` failing with `ActiveRecord::ConnectionNotEstablished`
+# (triggered by elasticsearch-model/lib/elasticsearch/model.rb:79:in `included')
+#
+ActiveRecord::Base.instance_eval do
+  class << self
+    def inspect_with_rescue
+      inspect_without_rescue
+    rescue ActiveRecord::ConnectionNotEstablished
+      "#{self}(no database connection)"
+    end
+
+    alias_method_chain :inspect, :rescue
+  end
+end if defined?(ActiveRecord) && ActiveRecord::VERSION::STRING < '4'

data/lib/elasticsearch/model/hash_wrapper.rb

@@ -0,0 +1,15 @@
+module Elasticsearch
+  module Model
+
+    # Subclass of `Hashie::Mash` to wrap Hash-like structures
+    # (responses from Elasticsearch, search definitions, etc)
+    #
+    # The primary goal of the subclass is to disable the
+    # warning being printed by Hashie for re-defined
+    # methods, such as `sort`.
+    #
+    class HashWrapper < ::Hashie::Mash
+      disable_warnings if respond_to?(:disable_warnings)
+    end
+  end
+end

data/lib/elasticsearch/model/importing.rb

@@ -0,0 +1,144 @@
+module Elasticsearch
+  module Model
+
+    # Provides support for easily and efficiently importing large amounts of
+    # records from the including class into the index.
+    #
+    # @see ClassMethods#import
+    #
+    module Importing
+
+      # When included in a model, adds the importing methods.
+      #
+      # @example Import all records from the `Article` model
+      #
+      #     Article.import
+      #
+      # @see #import
+      #
+      def self.included(base)
+        base.__send__ :extend, ClassMethods
+
+        adapter = Adapter.from_class(base)
+        base.__send__ :include, adapter.importing_mixin
+        base.__send__ :extend,  adapter.importing_mixin
+      end
+
+      module ClassMethods
+
+        # Import all model records into the index
+        #
+        # The method will pick up correct strategy based on the `Importing` module
+        # defined in the corresponding adapter.
+        #
+        # @param options [Hash] Options passed to the underlying `__find_in_batches`method
+        # @param block  [Proc] Optional block to evaluate for each batch
+        #
+        # @yield [Hash] Gives the Hash with the Elasticsearch response to the block
+        #
+        # @return [Fixnum] Number of errors encountered during importing
+        #
+        # @example Import all records into the index
+        #
+        #     Article.import
+        #
+        # @example Set the batch size to 100
+        #
+        #     Article.import batch_size: 100
+        #
+        # @example Process the response from Elasticsearch
+        #
+        #     Article.import do |response|
+        #       puts "Got " + response['items'].select { |i| i['index']['error'] }.size.to_s + " errors"
+        #     end
+        #
+        # @example Delete and create the index with appropriate settings and mappings
+        #
+        #    Article.import force: true
+        #
+        # @example Refresh the index after importing all batches
+        #
+        #    Article.import refresh: true
+        #
+        # @example Import the records into a different index/type than the default one
+        #
+        #    Article.import index: 'my-new-index', type: 'my-other-type'
+        #
+        # @example Pass an ActiveRecord scope to limit the imported records
+        #
+        #    Article.import scope: 'published'
+        #
+        # @example Transform records during the import with a lambda
+        #
+        #    transform = lambda do |a|
+        #      {index: {_id: a.id, _parent: a.author_id, data: a.__elasticsearch__.as_indexed_json}}
+        #    end
+        #
+        #    Article.import transform: transform
+        #
+        # @example Update the batch before yielding it
+        #
+        #     class Article
+        #       # ...
+        #       def enrich(batch)
+        #         batch.each do |item|
+        #           item.metadata = MyAPI.get_metadata(item.id)
+        #         end
+        #         batch
+        #       end
+        #     end
+        #
+        #    Article.import preprocess: enrich
+        #
+        # @example Return an array of error elements instead of the number of errors, eg.
+        #          to try importing these records again
+        #
+        #    Article.import return: 'errors'
+        #
+        def import(options={}, &block)
+          errors       = []
+          refresh      = options.delete(:refresh)   || false
+          target_index = options.delete(:index)     || index_name
+          target_type  = options.delete(:type)      || document_type
+          transform    = options.delete(:transform) || __transform
+          return_value = options.delete(:return)    || 'count'
+
+          unless transform.respond_to?(:call)
+            raise ArgumentError,
+                  "Pass an object responding to `call` as the :transform option, #{transform.class} given"
+          end
+
+          if options.delete(:force)
+            self.create_index! force: true, index: target_index
+          end
+
+          __find_in_batches(options) do |batch|
+            response = client.bulk \
+                         index:   target_index,
+                         type:    target_type,
+                         body:    __batch_to_bulk(batch, transform)
+
+            yield response if block_given?
+
+            errors +=  response['items'].select { |k, v| k.values.first['error'] }
+          end
+
+          self.refresh_index! if refresh
+
+          case return_value
+            when 'errors'
+              errors
+            else
+              errors.size
+          end
+        end
+
+        def __batch_to_bulk(batch, transform)
+          batch.map { |model| transform.call(model) }
+        end
+      end
+
+    end
+
+  end
+end