bigindex 0.1.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 openplaces.org
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,58 @@
+= BigIndex
+
+A Rails plugin that drops into models and provides indexing functionality. Uses an adapter/repository pattern inspired by Datamapper to abstract the actual indexer used in the background, and exposes the model to a simple indexing API.
+
+This should be used in conjunction with BigRecord in order to provide a more complete ORM.
+
+== Supported search servers
+
+* Solr
+* Sphinx (planned)
+
+== Installation
+
+(1) Download and install Solr. Take a note of the url that solr is running on.
+
+(2) In your Rails application, add Bigindex as a gem to your config/environment.rb file:
+
+  config.gem "bigindex", :source => "http://gemcutter.org"
+  
+and run the following rake task to install all the gems listed for your Rails app:
+
+  [sudo] rake gems:install
+
+(3) Bootstrap Bigindex into your Rails application:
+
+  script/generate bigindex
+  
+(4) Modify the config file config/bigindex.yml[.sample] to correspond to your Solr server.
+
+== Getting Started
+
+Modify your Ruby class/model similarly to the following:
+
+  class Model < BigRecord::Base
+    include BigIndex::Resource                    # 1. Include the BigIndex::Resource module into your model.
+
+    column :name, :string
+    column :description, :text
+
+    index :name,  :string                         # 2. Define each attribute you want to index along with its type.
+    index :description                            # Defaults to type :text
+  end
+
+BigIndex will then override the default Model.find() method to pass through the indexer first. Model.find() will also accept the option {:bypass_index => true}, which bypasses the indexed #find method and dispatches it to the original Model.find() method, e.g. Model.find(:all, :bypass_index => true). Alternatively, you can use Model.find_without_index(:all) for the same functionality.
+
+== License
+
+Bigindex is released under the MIT license.
+
+== Contributions
+
+Bigindex was derived from the work of Data Mapper and parts of acts_as_solr.
+
+== Links
+
+* Contact Us
+  * Website - http://www.bigrecord.org
+  * IRC Channel - <tt>#bigrecord</tt> on irc.freenode.net

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rubygems'
+require 'rake'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'pathname'
+require 'spec/rake/spectask'
+
+DATA_STORES = ["bigrecord", "activerecord"]
+
+ROOT = Pathname(__FILE__).dirname.expand_path
+
+require ROOT + 'tasks/gem'
+require ROOT + 'tasks/rdoc'
+require ROOT + 'tasks/spec'

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/examples/bigindex.yml

@@ -0,0 +1,9 @@
+development:
+  adapter: solr
+  solr_url: http://localhost:8981/solr
+test:
+  adapter: solr
+  solr_url: http://localhost:8982/solr
+production:
+  adapter: solr
+  solr_url: http://localhost:8983/solr

data/generators/bigindex/bigindex_generator.rb

@@ -0,0 +1,17 @@
+# This generator bootstraps a Rails project for use with Bigindex
+class BigindexGenerator < Rails::Generator::Base
+ 
+  def initialize(runtime_args, runtime_options = {})
+    require File.join(File.dirname(__FILE__), "..", "..", "install.rb")
+    Dir.mkdir('lib/tasks') unless File.directory?('lib/tasks')
+    super
+  end
+ 
+  def manifest
+    record do |m|
+      m.directory 'lib/tasks'
+      m.file 'bigindex.rake', 'lib/tasks/bigindex.rake'
+    end
+  end
+ 
+end

data/generators/bigindex/templates/bigindex.rake

@@ -0,0 +1,3 @@
+namespace :bigrecord do
+
+end

data/init.rb

@@ -0,0 +1,27 @@
+require 'big_index'
+
+def config_file
+  "#{RAILS_ROOT}/config/bigindex.yml"
+end
+
+def full_config
+  begin
+    YAML::load(File.open(config_file))
+  rescue
+    puts "Missing environment '#{RAILS_ENV}' in config file #{config_file}"
+    return {}
+  end
+end
+
+def get_config_for_environment
+  if hash = full_config[RAILS_ENV]
+    BigIndex.symbolize_keys(hash)
+  elsif hash = full_config[RAILS_ENV.to_sym]
+    hash
+  else
+    {}
+  end
+end
+
+
+BigIndex.setup(:default, get_config_for_environment) unless get_config_for_environment.empty?

data/install.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+
+puts "[Bigindex] Copying example config file to your RAILS_ROOT...\n"
+
+config_dir = File.join(RAILS_ROOT, "config")
+source = File.join(File.dirname(__FILE__), "examples", "bigindex.yml")
+target = File.join(config_dir, "bigindex.yml")
+alternate_target = File.join(config_dir, "bigindex.yml.sample")
+
+if !File.exist?(target)
+  FileUtils.cp(source, target)
+else
+  puts "[Bigindex] RAILS_ROOT/config/bigindex.yml file already exists. Copying it as bigindex.yml.sample for reference."
+  FileUtils.cp(source, alternate_target)
+end

data/lib/big_index/adapters/abstract_adapter.rb

@@ -0,0 +1,70 @@
+module BigIndex
+  module Adapters
+
+    class AbstractAdapter
+
+      attr_reader :name, :options, :connection
+
+      def adapter_name
+        'abstract'
+      end
+
+      def default_type_field
+        raise NotImplementedError
+      end
+
+      def default_primary_key_field
+        raise NotImplementedError
+      end
+
+      def process_index_batch(items, loop, options={})
+        raise NotImplementedError
+      end
+
+      def drop_index(model)
+        raise NotImplementedError
+      end
+
+      def get_field_type(field_type)
+        field_type
+      end
+
+      def execute(request)
+        raise NotImplementedError
+      end
+
+      def index_save(model)
+        raise NotImplementedError
+      end
+
+      def index_destroy(model)
+        raise NotImplementedError
+      end
+
+      def find_by_index(model, query, options={})
+        raise NotImplementedError
+      end
+
+      def find_values_by_index(model, query, options={})
+        raise NotImplementedError
+      end
+
+      def find_ids_by_index(model, query, options={})
+        raise NotImplementedError
+      end
+
+      def optimize_index
+        raise NotImplementedError
+      end
+
+    private
+
+      def initialize(name, options)
+        @name = name
+        @options = options
+      end
+
+    end # class AbstractAdapter
+
+  end # module Adapters
+end # module BigIndex

data/lib/big_index/adapters/solr_adapter.rb

@@ -0,0 +1,180 @@
+module BigIndex
+  module Adapters
+
+    class SolrAdapter < AbstractAdapter
+
+      include ::Solr::AdapterMethods
+
+      # BigIndex Adapter API methods ====================================
+
+      def adapter_name
+        'solr'
+      end
+
+      def default_type_field
+        "type_s_mv"
+      end
+
+      def default_primary_key_field
+        "pk_s"
+      end
+
+      def process_index_batch(items, loop, options = {})
+        unless items.empty?
+          # This checks that if the item has a method indexable? defined, then it will determine
+          # whether or not to index the item based on that method's returned boolean value.
+          items_to_index = items.select { |item| item.respond_to?(:indexable?) ? item.indexable? : true }
+
+          unless items_to_index.empty?
+            docs = items_to_index.collect{|content| to_solr_doc(content)}
+            if options[:only_generate]
+              # Collect the documents. This is to be used within a mapred job.
+              docs.each do |doc|
+                key = doc['id']
+
+                # Cannot have \n and \t in the value since they are
+                # document and field separators respectively
+                value = doc.to_xml.to_s
+                value = value.gsub("\n", "__ENDLINE__")
+                value = value.gsub("\t", "__TAB__")
+
+                puts "#{key}\t#{value}"
+              end
+            else
+              solr_add(docs)
+              solr_commit if options[:commit]
+            end
+          else
+            break
+          end
+        end
+      end
+
+      def drop_index(model)
+        @connection.logger = model.logger if model.respond_to?(:logger)
+        result = @connection.solr_execute(Solr::Request::Delete.new(:query => "type_s_mv:\"#{model.name}\""))
+
+        result.status_message == "0"
+      end
+
+      def get_field_type(field_type)
+        if field_type.is_a?(Symbol)
+          case field_type
+            when :float then                      return "f"
+            when :integer then                    return "i"
+            when :boolean then                    return "b"
+            when :string then                     return "s"
+            when :date then                       return "d"
+            when :range_float then                return "rf"
+            when :range_integer then              return "ri"
+            when :ngrams then                     return "ngrams"
+            when :autocomplete then               return "auto"
+            when :lowercase then                  return "lc"
+            when :exact_match then                return "em"
+            when :geo then                        return "geo"
+            when :text then                       return "t"
+            when :text_not_stored then            return "t_ns"
+            when :text_not_indexed then           return "t_ni"
+            when :integer_array then              return "i_mv"
+            when :integer_array_not_stored then   return "i_mv_ns"
+            when :text_array then                 return "t_mv"
+            when :text_array_not_stored then      return "t_mv_ns"
+            when :float_array then                return "f_mv"
+            when :boolean_array then              return "b_mv"
+            when :date_array then                 return "d_mv"
+            when :string_array then               return "s_mv"
+            when :range_integer_array then        return "ri_mv"
+            when :range_float_array then          return "rf_mv"
+            when :ngrams_array then               return "ngrams_mv"
+            when :autocomplete_array then         return "auto_mv"
+            when :lowercase_array then            return "lc_mv"
+            when :exact_match_array then          return "em_mv"
+          else
+            raise "Unknown field_type symbol: #{field_type}"
+          end
+        elsif field_type.is_a?(String)
+          return field_type
+        else
+          raise "Unknown field_type class: #{field_type.class}: #{field_type}"
+        end
+      end
+
+      def execute(request)
+        @connection.solr_execute(request)
+      end
+
+      def index_save(model)
+        configuration = model.index_configuration
+
+        results = []
+        if configuration[:if] && evaluate_condition(configuration[:if], model)
+          results << solr_add(to_solr_doc(model))
+          results << solr_commit if configuration[:auto_commit]
+        end
+
+        !results.map{|result| result.status_code == "0"}.include?(false)
+      end
+
+      def index_destroy(model)
+        configuration = model.index_configuration
+
+        results = []
+
+        results << solr_delete(model.index_id)
+        results << solr_delete(":#{model.record_id}")
+        results << solr_commit if configuration[:auto_commit]
+
+        !results.map{|result| result.status_code == "0"}.include?(false)
+      end
+
+      def find_by_index(model, query, options={})
+        raw_result = options.delete(:raw_result)
+        data = parse_query(model, query, options)
+
+        if data
+          parsed = parse_results(model, data, options)
+
+          return raw_result ? parsed : parsed.results
+        end
+      end
+
+      def find_values_by_index(model, query, options={})
+        raw_result = options.delete(:raw_result)
+        data = parse_query(model, query, options)
+
+        if data
+          parsed = parse_results(model, data, {:format => :values})
+
+          return raw_result ? parsed : parsed.results
+        end
+      end
+
+      def find_ids_by_index(model, query, options={})
+        raw_result = options.delete(:raw_result)
+        data = parse_query(model, query, options)
+
+        if data
+          parsed = parse_results(model, data, {:format => :ids})
+
+          return raw_result ? parsed : parsed.results
+        end
+      end
+
+      def optimize_index
+        solr_optimize
+      end
+
+      # End of BigIndex Adapter API ====================================
+
+    private
+
+      def initialize(name, options)
+        @connection = Solr::Base.new(options)
+
+        super(name, options)
+      end
+
+    end # class SolrAdapter
+
+  end # module Adapters
+end # module BigIndex

data/lib/big_index/adapters.rb

@@ -0,0 +1,11 @@
+dir = Pathname(__FILE__).dirname.expand_path + 'adapters'
+
+require dir + 'abstract_adapter'
+
+%w[ solr ].each do |gem|
+  begin
+    require dir + "#{gem}_adapter"
+  rescue LoadError, Gem::Exception
+    # ignore it
+  end
+end

data/lib/big_index/index_field.rb

@@ -0,0 +1,41 @@
+require File.join(File.dirname(__FILE__), 'support')
+
+module BigIndex
+
+  class IndexField
+    include Assertions
+
+    attr_reader :field, :field_name, :field_type, :options, :block
+
+    def initialize(params, block = nil)
+      raise "IndexField requires at least a field name" unless params.size > 0
+
+      @params = params.dup
+      @block = block
+
+      @field_name = params.shift
+      assert_kind_of 'field_name', @field_name, Symbol, String
+
+      unless params.empty? || ![Symbol, String].include?(params.first.class)
+        @field_type = params.shift
+      end
+
+      @options = params.shift || {}
+      assert_kind_of 'options', @options, Hash
+
+      # Setting the default values
+      @options[:finder_name] ||= field_name
+      @field_type ||= :text
+    end
+
+    def [](name)
+      @options[name]
+    end
+
+    def method_missing(name)
+      @options[name.to_sym] || super
+    end
+
+  end # class IndexField
+
+end # module BigIndex

data/lib/big_index/repository.rb

@@ -0,0 +1,77 @@
+module BigIndex
+  class Repository
+    include Assertions
+
+    @adapters = {}
+    @default_name = :default
+
+    ##
+    #
+    # @return <Adapter> the adapters registered for this repository
+    def self.adapters
+      @adapters
+    end
+
+    def self.context
+      Thread.current[:bigindex_repository_contexts] ||= []
+    end
+
+    def self.default_name
+      @default_name ||= :default
+    end
+
+    def self.default_name=(name)
+      @default_name = name
+    end
+
+    # TODO: Make sure this isn't dangerous
+    def self.clear_adapters
+      @adapters = {}
+    end
+
+    attr_reader :name
+
+    def adapter
+      # Make adapter instantiation lazy so we can defer repository setup until it's actually
+      # needed. Do not remove this code.
+      @adapter ||= begin
+        raise ArgumentError, "Adapter not set: #{@name}. Did you forget to setup?" \
+          unless self.class.adapters.has_key?(@name)
+
+        self.class.adapters[@name]
+      end
+    end
+
+    # TODO: spec this
+    def scope
+      Repository.context << self
+
+      begin
+        return yield(self)
+      ensure
+        Repository.context.pop
+      end
+    end
+
+    def eql?(other)
+      return true if super
+      name == other.name
+    end
+
+    alias == eql?
+
+    def to_s
+      "#<BigIndex::Repository:#{@name}>"
+    end
+
+
+    private
+
+    def initialize(name)
+      assert_kind_of 'name', name, Symbol
+
+      @name = name
+    end
+
+  end # class Repository
+end # module BigIndex