rmla 1.0

data/Rakefile

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "mebla"
+  gem.homepage = "http://github.com/cousine/mebla"
+  gem.license = "MIT"
+  gem.summary = %Q{An elasticsearch wrapper for mongoid odm based on slingshot.}
+  gem.description = %Q{
+    An elasticsearch wrapper for mongoid odm based on slingshot. Makes integration between ElasticSearch full-text 
+    search engine and Mongoid documents seemless and simple.
+  }
+  gem.email = "omar.mekky@mashsolvents.com"
+  gem.authors = ["Omar Mekky"]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/TODO.md

@@ -0,0 +1,23 @@
+TODO for version 1.1.0
+==============
+
+* <strike>add ability to index embedded documents fields (as part of the parent document)</strike>
+* <strike>add instructions for indexing methods to README.md</strike>
+* <strike>add ability to index methods results</strike>
+
+TODO for version 1.0.1
+==============
+
+* <strike>properly handle sub classes</strike>
+
+TODO for version 1.0.0
+==============
+
+* <strike>add documentation for mebla in README.md</strike>
+* <strike>add logging capabilities</strike>
+
+Future plan
+=======
+
+* optimize : refractor result_set
+* <strike>optimize : should find a solution for not refreshing the index while indexing embedded documents in lib/mebla/context</strike> not necessary since indexing/reindexing

data/VERSION

@@ -0,0 +1 @@
+1.1.14

data/lib/generators/mebla/install/USAGE

@@ -0,0 +1,7 @@
+Description:
+    Generates mebla's configuration file.
+
+Example:
+    rails generate mebla:install
+
+    This will generate mebla's configuration file in config folder

data/lib/generators/mebla/install/install_generator.rb

@@ -0,0 +1,35 @@
+# A wrapper for slingshot  elastic-search adapter for Mongoid
+module Mebla  
+  # Generates the required files for Mebla to function
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path('../templates', __FILE__)
+    
+    # Generates mebla's configuration file
+    def generate_configuration
+      template "mebla.yml", "config/mebla.yml"
+    end
+    
+    private
+    # Returns the rails application name
+    # @return [String]
+    def app_name
+      @app_name ||= defined_app_const_base? ? defined_app_name : File.basename(destination_root)
+    end
+
+    # @private
+    # Returns the rails application name underscored
+    # @return [String]
+    def defined_app_name
+      defined_app_const_base.underscore
+    end
+
+    # @private
+    # Returns the application CONSTANT    
+    def defined_app_const_base
+      Rails.respond_to?(:application) && defined?(Rails::Application) &&
+        Rails.application.is_a?(Rails::Application) && Rails.application.class.name.sub(/::Application$/, "")
+    end
+
+    alias :defined_app_const_base? :defined_app_const_base    
+  end
+end

data/lib/generators/mebla/install/templates/mebla.yml

@@ -0,0 +1,15 @@
+defaults: &defaults
+  host: localhost
+  port: 9200
+  
+development:
+  <<: *defaults
+  index: <%= app_name %>_development
+  
+test:
+  <<: *defaults
+  index: <%= app_name %>_test
+  
+production:
+  <<: *defaults
+  index: <%= app_name %>_production

data/lib/mebla.rb

@@ -0,0 +1,117 @@
+require 'active_support'
+require 'mebla/railtie' if defined?(Rails)
+
+# A wrapper for slingshot  elastic-search adapter for Mongoid
+module Mebla
+  extend ActiveSupport::Autoload
+  
+  # Dependencies
+  autoload :Mongoid,  'mongoid'
+  autoload :Slingshot,  'slingshot'
+  # Main modules
+  autoload :Configuration
+  autoload :Context
+  autoload :LogSubscriber
+  autoload :ResultSet
+  autoload :Search
+  # Errors
+  autoload :Errors
+  # Mongoid extensions
+  autoload :Mebla,  'mebla/mongoid/mebla'  
+    
+  # Register the logger
+  Mebla::LogSubscriber.attach_to :mebla
+  
+  @@mebla_mutex = Mutex.new
+  @@context      = nil
+  
+  # Returns Mebla's context for minipulating the index
+  # @return [nil]
+  def self.context
+    if @@context.nil?
+      @@mebla_mutex.synchronize do
+        if @@context.nil?
+          @@context = Mebla::Context.new          
+        end
+      end
+    end
+    
+    @@context
+  end
+  
+  # Resets the context (reloads Mebla)
+  # @return [nil]
+  def self.reset_context!
+    @@mebla_mutex.synchronize do
+      @@context = nil
+    end
+  end
+  
+  # Check if mongoid is loaded
+  # @return [Boolean]
+  def self.mongoid?
+    !defined?(Mongoid).nil?
+  end
+  
+  # Check if slingshot is loaded
+  # @return [Boolean]
+  def self.slingshot?
+    !defined?(Slingshot).nil?
+  end
+  
+  # Check if elasticsearch is running
+  # @return [Boolean]
+  def self.elasticsearch?
+    result = Slingshot::Configuration.client.get "#{Slingshot::Configuration.url}/_status"
+    return (result =~ /error/) ? false: true
+  rescue RestClient::Exception
+    false
+  end
+  
+  # Configure Mebla  
+  #
+  # Example::
+  # 
+  #   Mebla.configure do |config|
+  #     index = "mebla_index"
+  #     host = "localhost"
+  #     port = 9200
+  #   end
+  def self.configure(&block)
+    yield Mebla::Configuration.instance
+  end
+  
+  
+  # Writes out a message to the log file according to the level given
+  # @note If no level is given a message of type Logger::UNKNOWN will be written to the log file
+  # @param [String] message
+  # @param [Symbol] level can be :debug, :warn or :info
+  # @return [nil]
+  def self.log(message, level = :none)    
+    case level
+    when :debug
+      hook = "mebla_debug.mebla"
+    when :warn
+      hook = "mebla_warn.mebla"
+    when :info
+      hook = "mebla_info.mebla"
+    else
+      hook = "mebla_unknown.mebla"
+    end
+    
+    ::ActiveSupport::Notifications.
+      instrument(hook, :message => message)
+  end  
+  
+  # Search the index
+  # @param [String] query a string representing the search query
+  # @param [String, Symbol, Array] type_names a string, symbol or array representing the models to be searcheds
+  # @return [Mebla::Search]
+  #
+  # Search for all documents with a field 'title' with a value 'Testing Search'::
+  #
+  #  Mebla.search "title: Testing Search"  
+  def self.search(query = "", type_names = nil)
+    Mebla::Search.new(query, type_names)
+  end
+end

data/lib/mebla/configuration.rb

@@ -0,0 +1,71 @@
+require 'erb'
+require 'singleton'
+
+# A wrapper for slingshot  elastic-search adapter for Mongoid
+module Mebla
+  # Parses the configuration file and holds important configuration attributes  
+  class Configuration
+    include Singleton
+    
+    attr_reader :log_dir
+    attr_accessor :index, :host, :port, :logger
+    
+    # @private
+    # Initializes a new configuration object
+    def initialize
+      @log_dir = "#{Dir.pwd}/tmp/log"
+      parse_config      
+      
+      # Setup defaults
+      @index ||= "mebla"
+      @host ||= "localhost"
+      @port ||= 9200
+      
+      make_tmp_dir
+      @logger = ActiveSupport::BufferedLogger.new(
+        open("#{@log_dir}/mebla.log", "a")
+      )
+      @logger.level = ActiveSupport::BufferedLogger::Severity::DEBUG
+      
+      setup_logger        
+      
+      # Setup slingshot
+      Slingshot::Configuration.url(self.url)
+    end
+    
+    # Sets up the default settings of the logger
+    # @return [nil]
+    def setup_logger
+      @logger.auto_flushing = true      
+    end
+    
+    # Returns the proper url for elasticsearch
+    # @return [String] url representation of the configuration options host and port
+    def url
+      "http://#{@host}:#{@port}"
+    end
+    
+    private    
+    # Creates tmp directory if it doesn't exist
+    # @return [nil]
+    def make_tmp_dir
+      FileUtils.mkdir_p @log_dir
+      Dir["#{@log_dir}/*"].each do |file|
+        FileUtils.rm_rf file
+      end
+    end
+    
+    # Loads the configuration file
+    # @return [nil]
+    def parse_config      
+      path = "#{Rails.root}/config/mebla.yml"
+      return unless File.exists?(path)
+      
+      conf = YAML::load(ERB.new(IO.read(path)).result)[Rails.env]
+      
+      conf.each do |key,value|
+        self.send("#{key}=", value) if self.respond_to?("#{key}=")
+      end unless conf.nil?
+    end
+  end
+end

data/lib/mebla/context.rb

@@ -0,0 +1,298 @@
+# A wrapper for slingshot  elastic-search adapter for Mongoid
+module Mebla
+  # Handles indexing and reindexing
+  class Context    
+    attr_reader  :indexed_models, :slingshot_index, :slingshot_index_name
+    attr_reader  :mappings
+    
+    # @private
+    # Creates a new context object
+    def initialize            
+      @indexed_models = []
+      @mappings = {}
+      @slingshot_index = Slingshot::Index.new(Mebla::Configuration.instance.index)
+      @slingshot_index_name = Mebla::Configuration.instance.index
+    end
+    
+    # @private
+    # Adds a model to the list of indexed models
+    def add_indexed_model(model, mappings = {})
+      model = model.name if model.is_a?(Class)
+      
+      @indexed_models << model
+      @indexed_models.uniq!
+      @indexed_models.sort!
+      
+      @mappings.merge!(mappings)
+    end
+    
+    # Deletes and rebuilds the index
+    # @note Doesn't index the data, use Mebla::Context#reindex_data to rebuild the index and index the data
+    # @return [nil]
+    def rebuild_index
+      # Only rebuild if the index exists
+      raise Mebla::Errors::MeblaIndexException.new("#{@slingshot_index_name} does not exist !! use #create_index to create the index first.") unless index_exists?
+
+      Mebla.log("Rebuilding index")      
+      
+      # Delete the index
+      if drop_index
+        # Create the index
+        return build_index
+      end
+    end
+    
+    # Creates and indexes the document
+    # @note Doesn't index the data, use Mebla::Context#index_data to create the index and index the data
+    # @return [Boolean] true if operation is successful
+    def create_index
+      # Only create the index if it doesn't exist
+      raise Mebla::Errors::MeblaIndexException.new("#{@slingshot_index_name} already exists !! use #rebuild_index to rebuild the index.") if index_exists?
+      
+      Mebla.log("Creating index")
+      
+      # Create the index
+      build_index
+    end
+    
+    # Deletes the index of the document
+    # @return [Boolean] true if operation is successful
+    def drop_index
+      # Only drop the index if it exists
+      return true unless index_exists?
+      
+      Mebla.log("Dropping index: #{self.slingshot_index_name}", :debug)
+      
+      # Drop the index
+      result = @slingshot_index.delete
+      
+      Mebla.log("Dropped #{self.slingshot_index_name}: #{result.to_s}", :debug)
+      
+      # Check that the index doesn't exist
+      !index_exists?
+    end
+    
+    # Checks if the index exists and is available
+    # @return [Boolean] true if the index exists and is available, false otherwise
+    def index_exists?
+      begin
+        result = Slingshot::Configuration.client.get "#{Mebla::Configuration.instance.url}/#{@slingshot_index_name}/_status"
+        return (result =~ /error/) ? false : true
+      rescue RestClient::ResourceNotFound
+        return false
+      end
+    end
+    
+    # Creates the index and indexes the data for all models or a list of models given
+    # @param *models a list of symbols each representing a model name to be indexed
+    # @return [nil]
+    def index_data(*models)
+      if models.nil? || models.empty?
+        only_index = @indexed_models
+      else
+        only_index = models.collect{|m| m.to_s}
+      end      
+      
+      Mebla.log("Indexing #{only_index.join(", ")}", :debug)
+      
+      # Build up a bulk query to save processing and time
+      bulk_query = ""
+      # Keep track of indexed documents
+      indexed_count = {}
+      
+      # Create the index
+      if create_index
+        # Start collecting documents
+        only_index.each do |model|
+          Mebla.log("Indexing: #{model}")
+          # Get the class
+          to_index = model.camelize.constantize
+          
+          # Get the records    
+          entries = []
+          unless to_index.embedded?
+            if to_index.sub_class?
+              entries = to_index.any_in(:_type => [to_index.name])
+            else              
+              entries = to_index.any_in(:_type => [nil, to_index.name])
+            end
+          else
+            parent = to_index.embedded_parent
+            access_method = to_index.embedded_as
+            
+            parent.all.each do |parent_record|
+              if to_index.sub_class?
+                entries += parent_record.send(access_method.to_sym).any_in(:_type => [to_index.name])
+              else
+                entries += parent_record.send(access_method.to_sym).any_in(:_type => [nil, to_index.name])
+              end
+            end
+          end
+          
+          # Save the number of entries to be indexed
+          indexed_count[model] = entries.count          
+          
+          # Build the queries for this model          
+          entries.each do |document|
+            attrs = {} #document.attributes.dup # make sure we dont modify the document it self
+            attrs[:id] = document.attributes["_id"] # the id is already added in the meta data of the action part of the query
+            
+            # only index search fields  and methods
+            document.class.search_fields.each do |field|
+              if document.attributes.keys.include?(field.to_s)
+                attrs[field] = document.attributes[field.to_s] # attribute
+              else
+                attrs[field] = document.send(field) # method
+              end
+            end
+            
+            # index relational fields
+            document.class.search_relations.each do |relation, fields|              
+              items = document.send(relation.to_sym) # get the relation document
+              
+              next if items.nil?
+              
+              # N relation side
+              if items.is_a?(Array) || items.is_a?(Mongoid::Relations::Targets::Enumerable)
+                next if items.empty?
+                attrs[relation] = []
+                items.each do |item|
+                  if fields.is_a?(Array) # given multiple fields to index
+                    fields_values = {}
+                    fields.each do |field|
+                      if item.attributes.keys.include?(field.to_s)
+                        fields_values.merge!({ field => item.attributes[field.to_s] }) # attribute
+                      else
+                        fields_values.merge!({ field => item.send(field) }) # method
+                      end
+                    end
+                    attrs[relation] << fields_values
+                  else # only index one field in the relation
+                    if item.attributes.keys.include?(fields.to_s)
+                      attrs[relation] << { fields => item.attributes[fields.to_s] } # attribute
+                    else
+                      attrs[relation] << { fields => item.send(fields) } # method
+                    end
+                  end
+                end
+              # 1 relation side
+              else
+                attrs[relation] = {}
+                if fields.is_a?(Array) # given multiple fields to index
+                  fields_values = {}
+                  fields.each do |field|
+                    if items.attributes.keys.include?(field.to_s)
+                      fields_values.merge!({ field => items.attributes[field.to_s] }) # attribute
+                    else
+                      fields_values.merge!({ field => items.send(field) }) # method
+                    end
+                  end
+                  attrs[relation].merge!(fields_values)
+                else # only index one field in the relation
+                  if items.attributes.keys.include?(fields.to_s)
+                    attrs[relation].merge!({ fields => items.attributes[fields.to_s] }) # attribute
+                  else
+                    attrs[relation].merge!({ fields => items.send(fields) }) # method
+                  end
+                end
+              end
+            end  
+            
+            # If embedded get the parent id
+            if document.embedded?
+              parent_id = document.send(document.class.embedded_parent_foreign_key.to_sym).id.to_s        
+              attrs[(document.class.embedded_parent_foreign_key + "_id").to_sym] = parent_id
+              attrs[:_parent] = parent_id
+              
+              # Build add to the bulk query
+              bulk_query << build_bulk_query(@slingshot_index_name, to_index.slingshot_type_name, document.id.to_s, attrs, parent_id)
+            else
+              # Build add to the bulk query
+              bulk_query << build_bulk_query(@slingshot_index_name, to_index.slingshot_type_name, document.id.to_s, attrs)
+            end
+          end
+        end
+      else
+        raise Mebla::Errors::MeblaIndexException.new("Could not create #{@slingshot_index_name}!!!")
+      end      
+      
+      Mebla.log("Bulk indexing:\n#{bulk_query}", :debug)      
+      
+      # Send the query
+      response = Slingshot::Configuration.client.post "#{Mebla::Configuration.instance.url}/_bulk", bulk_query
+      
+      # Only refresh the index if no error ocurred
+      unless response =~ /error/                              
+        # Log results
+        Mebla.log("Indexed #{only_index.count} model(s) to #{self.slingshot_index_name}: #{response}")
+        Mebla.log("Indexing Report:")
+        indexed_count.each do |model_name, count|
+          Mebla.log("Indexed #{model_name}: #{count} document(s)")
+        end
+        
+        # Refresh the index
+        refresh_index
+      else
+        raise Mebla::Errors::MeblaIndexException.new("Indexing #{only_index.join(", ")} failed with the following response:\n #{response}")
+      end
+    rescue RestClient::Exception => error
+      raise Mebla::Errors::MeblaIndexException.new("Indexing #{only_index.join(", ")} failed with the following error: #{error.message}")
+    end
+    
+    # Rebuilds the index and indexes the data for all models or a list of models given
+    # @param *models a list of symbols each representing a model name to rebuild it's index
+    # @return [nil]
+    def reindex_data(*models)   
+      Mebla.log("Rendexing: #{self.slingshot_index_name}")
+      
+      unless drop_index
+        raise Mebla::Errors::MeblaIndexException.new("Could not drop #{@slingshot_index_name}!!!")
+      end        
+      
+      # Create the index and index the data
+      if models && !models.empty?
+        index_data(models)
+      else
+        index_data
+      end
+    end
+        
+    # Refreshes the index
+    # @return [nil]
+    def refresh_index
+      Mebla.log("Refreshing: #{self.slingshot_index_name}", :debug)
+      
+      result = @slingshot_index.refresh
+      
+      Mebla.log("Refreshed #{self.slingshot_index_name}: #{result}")
+    end
+    
+    private          
+    # Builds the index according to the mappings set
+    # @return [Boolean] true if the index was created successfully, false otherwise
+    def build_index 
+      Mebla.log("Building #{self.slingshot_index_name}", :debug)
+      # Create the index
+      result = @slingshot_index.create :mappings => @mappings 
+      
+      Mebla.log("Created #{self.slingshot_index_name}: #{result.to_s}")
+      
+      # Check if the index exists
+      index_exists?
+    end
+    
+    # --
+    # OPTIMIZE: should find a solution for not refreshing the index while indexing embedded documents
+    # ++
+
+    # Builds a bulk index query
+    # @return [String]
+    def build_bulk_query(index_name, type, id, attributes, parent = nil)
+      attrs_to_json = ActiveSupport::JSON.encode(attributes).gsub(/\n/, " ")
+      <<-eos
+        { "index" : { "_index" : "#{index_name}", "_type" : "#{type}", "_id" : "#{id}"#{", \"_parent\" : \"#{parent}\"" if parent}, "refresh" : "true"} }        
+        #{attrs_to_json}
+      eos
+    end
+  end
+end