ruben-sunspot 1.1.0

data/lib/sunspot/field_factory.rb

@@ -0,0 +1,147 @@
+module Sunspot
+  # 
+  # The FieldFactory module contains classes for generating fields. FieldFactory
+  # implementation classes should implement a #build method, although the arity
+  # of the method depends on the type of factory. They also must implement a
+  # #populate_document method, which extracts field data from a given model and
+  # adds it into the Solr document for indexing.
+  #
+  module FieldFactory #:nodoc:all
+    # 
+    # Base class for field factories.
+    #
+    class Abstract
+      attr_reader :name
+
+      def initialize(name, options = {}, &block)
+        @name = name.to_sym
+        @data_extractor =
+          if block
+            DataExtractor::BlockExtractor.new(&block)
+          else
+            DataExtractor::AttributeExtractor.new(options.delete(:using) || name)
+          end
+      end
+    end
+
+    # 
+    # A StaticFieldFactory generates normal static fields. Each factory instance
+    # contains an eager-initialized field instance, which is returned by the
+    # #build method.
+    #
+    class Static < Abstract
+      def initialize(name, type, options = {}, &block)
+        super(name, options, &block)
+        unless name.to_s =~ /^\w+$/
+          raise ArgumentError, "Invalid field name #{name}: only letters, numbers, and underscores are allowed."
+        end
+        @field =
+          if type.is_a?(Type::TextType)
+            FulltextField.new(name, options)
+          else
+            AttributeField.new(name, type, options)
+          end
+      end
+
+      # 
+      # Return the field instance built by this factory
+      #
+      def build
+        @field
+      end
+
+      # 
+      # Extract the encapsulated field's data from the given model and add it
+      # into the Solr document for indexing.
+      #
+      def populate_document(document, model) #:nodoc:
+        unless (value = @data_extractor.value_for(model)).nil?
+          Util.Array(@field.to_indexed(value)).each do |scalar_value|
+            options = {}
+            options[:boost] = @field.boost if @field.boost
+            document.add_field(
+              @field.indexed_name.to_sym,
+              scalar_value,
+              options
+            )
+          end
+        end
+      end
+
+      # 
+      # A unique signature identifying this field by name and type.
+      #
+      def signature
+        [@field.name, @field.type]
+      end
+    end
+
+    # 
+    # DynamicFieldFactories create dynamic field instances based on dynamic
+    # configuration.
+    #
+    class Dynamic < Abstract
+      attr_accessor :name, :type
+
+      def initialize(name, type, options = {}, &block)
+        super(name, options, &block)
+        @type, @options = type, options
+      end
+
+      #
+      # Build a field based on the dynamic name given.
+      #
+      def build(dynamic_name)
+        AttributeField.new("#{@name}:#{dynamic_name}", @type, @options.dup)
+      end
+      # 
+      # This alias allows a DynamicFieldFactory to be used in place of a Setup
+      # or CompositeSetup instance by query components.
+      #
+      alias_method :field, :build
+
+      # 
+      # Generate dynamic fields based on hash returned by data accessor and
+      # add the field data to the document.
+      #
+      def populate_document(document, model)
+        if values = @data_extractor.value_for(model)
+          values.each_pair do |dynamic_name, value|
+            field_instance = build(dynamic_name)
+            Util.Array(field_instance.to_indexed(value)).each do |scalar_value|
+              document.add_field(
+                field_instance.indexed_name.to_sym,
+                scalar_value
+              )
+            end
+          end
+        end
+      end
+
+      # 
+      # Unique signature identifying this dynamic field based on name and type
+      #
+      def signature
+        [@name, @type]
+      end
+    end
+
+    class Coordinates
+      def initialize(name = nil, &block)
+        if block
+          @data_extractor = DataExtractor::BlockExtractor.new(&block)
+        else
+          @data_extractor = DataExtractor::AttributeExtractor.new(name)
+        end
+      end
+
+      def populate_document(document, model)
+        if coordinates = @data_extractor.value_for(model)
+          coordinates = Util::Coordinates.new(coordinates)
+          document.add_field(:lat, coordinates.lat)
+          document.add_field(:lng, coordinates.lng)
+        end
+      end
+    end
+  end
+end

data/lib/sunspot/indexer.rb

@@ -0,0 +1,131 @@
+module Sunspot
+  # 
+  # This class presents a service for adding, updating, and removing data
+  # from the Solr index. An Indexer instance is associated with a particular
+  # setup, and thus is capable of indexing instances of a certain class (and its
+  # subclasses).
+  #
+  class Indexer #:nodoc:
+    include RSolr::Char
+
+    def initialize(connection)
+      @connection = connection
+    end
+
+    # 
+    # Construct a representation of the model for indexing and send it to the
+    # connection for indexing
+    #
+    # ==== Parameters
+    #
+    # model<Object>:: the model to index
+    #
+    def add(model)
+      documents = Util.Array(model).map { |m| prepare(m) }
+      if @batch.nil?
+        add_documents(documents)
+      else
+        @batch.concat(documents)
+      end
+    end
+
+    # 
+    # Remove the given model from the Solr index
+    #
+    def remove(*models)
+      @connection.delete_by_id(
+        models.map { |model| Adapters::InstanceAdapter.adapt(model).index_id }
+      )
+    end
+
+    # 
+    # Remove the model from the Solr index by specifying the class and ID
+    #
+    def remove_by_id(class_name, *ids)
+      @connection.delete_by_id(
+        ids.map { |id| Adapters::InstanceAdapter.index_id_for(class_name, id) }
+      )
+    end
+
+    # 
+    # Delete all documents of the class indexed by this indexer from Solr.
+    #
+    def remove_all(clazz = nil)
+      if clazz
+        @connection.delete_by_query("type:#{escape(clazz.name)}")
+      else
+        @connection.delete_by_query("*:*")
+      end
+    end
+
+    # 
+    # Remove all documents that match the scope given in the Query
+    #
+    def remove_by_scope(scope)
+      @connection.delete_by_query(scope.to_boolean_phrase)
+    end
+
+    # 
+    # Start batch processing
+    #
+    def start_batch
+      @batch = []
+    end
+
+    #
+    # Write batch out to Solr and clear it
+    #
+    def flush_batch
+      add_documents(@batch)
+      @batch = nil
+    end
+
+    private
+
+    # 
+    # Convert documents into hash of indexed properties
+    #
+    def prepare(model)
+      document = document_for(model)
+      setup = setup_for(model)
+      if boost = setup.document_boost_for(model)
+        document.attrs[:boost] = boost
+      end
+      setup.all_field_factories.each do |field_factory|
+        field_factory.populate_document(document, model)
+      end
+      document
+    end
+
+    def add_documents(documents)
+      @connection.add(documents)
+    end
+
+    # 
+    # All indexed documents index and store the +id+ and +type+ fields.
+    # This method constructs the document hash containing those key-value
+    # pairs.
+    #
+    def document_for(model)
+      RSolr::Message::Document.new(
+        :id => Adapters::InstanceAdapter.adapt(model).index_id,
+        :type => Util.superclasses_for(model.class).map { |clazz| clazz.name }
+      )
+    end
+
+    # 
+    # Get the Setup object for the given object's class.
+    #
+    # ==== Parameters
+    #
+    # object<Object>:: The object whose setup is to be retrieved
+    #
+    # ==== Returns
+    #
+    # Sunspot::Setup:: The setup for the object's class
+    #
+    def setup_for(object)
+      Setup.for(object.class) || raise(NoSetupError, "Sunspot is not configured for #{object.class.inspect}")
+    end
+  end
+end

data/lib/sunspot/installer.rb

@@ -0,0 +1,31 @@
+%w(task_helper library_installer schema_builder solrconfig_updater).each do |file|
+  require File.join(File.dirname(__FILE__), 'installer', file)
+end
+
+module Sunspot
+  class Installer
+    class <<self
+      def execute(solr_home, options = {})
+        new(solr_home, options).execute
+      end
+
+      private :new
+    end
+
+    def initialize(solr_home, options)
+      @solr_home, @options = solr_home, options
+    end
+
+    def execute
+      SchemaBuilder.execute(
+        File.join(@solr_home, 'conf', 'schema.xml'),
+        @options
+      )
+      SolrconfigUpdater.execute(
+        File.join(@solr_home, 'conf', 'solrconfig.xml'),
+        @options
+      )
+      LibraryInstaller.execute(File.join(@solr_home, 'lib'), @options)
+    end
+  end
+end

data/lib/sunspot/installer/library_installer.rb

@@ -0,0 +1,45 @@
+require 'fileutils'
+
+module Sunspot
+  class Installer
+    class LibraryInstaller
+      class <<self
+        def execute(library_path, options)
+          new(library_path, options).execute
+        end
+      end
+
+      def initialize(library_path, options)
+        @library_path = library_path
+        @verbose = !!options[:verbose]
+        @force = !!options[:force]
+      end
+
+      def execute
+        sunspot_library_path = File.join(File.dirname(__FILE__), '..', '..',
+                                         '..', 'solr', 'solr', 'lib')
+        return if File.expand_path(sunspot_library_path) == File.expand_path(@library_path)
+        FileUtils.mkdir_p(@library_path)
+        Dir.glob(File.join(sunspot_library_path, '*.jar')).each do |jar|
+          jar = File.expand_path(jar)
+          dest = File.join(@library_path, File.basename(jar))
+          if File.exist?(dest)
+            if @force
+              say("Removing existing library #{dest}")
+            else
+              next
+            end
+          end
+          say("Copying #{jar} => #{dest}")
+          FileUtils.cp(jar, dest)
+                           end
+      end
+
+      def say(message)
+        if @verbose
+          STDOUT.puts(message)
+        end
+      end
+    end
+  end
+end

data/lib/sunspot/installer/schema_builder.rb

@@ -0,0 +1,219 @@
+require 'yaml'
+require 'rexml/rexml'
+require 'rexml/document'
+require 'fileutils'
+
+module Sunspot
+  class Installer
+    # 
+    # This class modifies an existing Solr schema.xml file to work with Sunspot.
+    # It makes the minimum necessary changes to the schema, adding fields and
+    # types only when they don't already exist. It also comments all fields and
+    # types that Sunspot needs, whether or not they were preexisting, so that
+    # users can modify the resulting schema without unwittingly breaking it.
+    #
+    class SchemaBuilder
+      include TaskHelper
+
+      CONFIG_PATH = File.join(
+        File.dirname(__FILE__), '..', '..', '..', 'installer', 'config', 'schema.yml'
+      )
+
+      class <<self
+        def execute(schema_path, options = {})
+          new(schema_path, options).execute
+        end
+        
+        private :new
+      end
+
+      def initialize(schema_path, options = {})
+        @schema_path = schema_path
+        @config = File.open(CONFIG_PATH) { |f| YAML.load(f) }
+        @verbose = !!options[:verbose]
+        @force = !!options[:force]
+      end
+
+      def execute
+        if @force
+          FileUtils.mkdir_p(File.dirname(@schema_path))
+          source_path = File.join(File.dirname(__FILE__), '..', '..', '..', 'solr', 'solr', 'conf', 'schema.xml')
+          FileUtils.cp(source_path, @schema_path)
+          say("Copied default schema.xml to #{@schema_path}")
+        else
+          @document = File.open(@schema_path) do |f|
+            Nokogiri::XML(
+              f, nil, nil,
+              Nokogiri::XML::ParseOptions::DEFAULT_XML |
+              Nokogiri::XML::ParseOptions::NOBLANKS
+            )
+          end
+          @root = @document.root
+          @added_types = Set[]
+          add_fixed_fields
+          add_dynamic_fields
+          set_misc_settings
+          original_path = "#{@schema_path}.orig"
+          FileUtils.cp(@schema_path, original_path)
+          say("Saved backup of original to #{original_path}")
+          File.open(@schema_path, 'w') do |file|
+            @document.write_xml_to(file, :indent => 2)
+          end
+          say("Wrote schema to #{@schema_path}")
+        end
+      end
+
+      private
+
+      def add_fixed_fields
+        @config['fixed'].each do |name, options|
+          maybe_add_field(name, options['type'], :indexed, *Array(options['attributes']))
+        end
+      end
+
+      def add_dynamic_fields
+        @config['types'].each do |type, options|
+          if suffix = options['suffix']
+            variant_combinations(options).each do |variants|
+              variants_suffix = variants.map { |variant| variant[1] || '' }.join
+              maybe_add_field("*_#{suffix}#{variants_suffix}", type, *variants.map { |variant| variant[0] })
+            end
+          end
+        end
+      end
+
+      def maybe_add_field(name, type, *flags)
+        node_name = name =~ /\*/ ? 'dynamicField' : 'field'
+        if field_node = fields_node.xpath(%Q(#{node_name}[@name="#{name}"])).first
+          say("Using existing #{node_name} #{name.inspect}")
+          add_comment(field_node)
+          return
+        end
+        maybe_add_type(type)
+        say("Adding field #{name.inspect}")
+        attributes = {
+          'name' => name,
+          'type' => type,
+          'indexed' => 'true',
+          'stored' => 'false',
+          'multiValued' => 'false'
+        }
+        flags.each do |flag|
+          attributes[flag.to_s] = 'true'
+        end
+        field_node = add_element(fields_node, node_name, attributes)
+        add_comment(field_node)
+      end
+
+      def maybe_add_type(type)
+        unless @added_types.include?(type)
+          @added_types << type
+          if type_node = types_node.xpath(%Q(fieldType[@name="#{type}"])).first ||
+             types_node.xpath(%Q(fieldtype[@name="#{type}"])).first
+            say("Using existing type #{type.inspect}")
+            add_comment(type_node)
+            return
+          end
+          say("Adding type #{type.inspect}")
+          type_config = @config['types'][type]
+          type_node = add_element(
+            types_node,
+            'fieldType',
+            'name' => type,
+            'class' => to_solr_class(type_config['class']),
+            'omitNorms' => type_config['omit_norms'].nil? ? 'true' : type_config['omit_norms'].to_s
+          )
+          if type_config['tokenizer']
+            add_analyzer(type_node, type_config)
+          end
+          add_comment(type_node)
+        end
+      end
+
+      def add_analyzer(node, config)
+        analyzer_node = add_element(node, 'analyzer')
+        add_element(
+          analyzer_node,
+          'tokenizer',
+          'class' => to_solr_class(config['tokenizer'])
+        )
+        Array(config['filters']).each do |filter|
+          add_element(analyzer_node, 'filter', 'class' => to_solr_class(filter))
+        end
+      end
+
+      def set_misc_settings
+        if @root.xpath('uniqueKey[normalize-space()="id"]').any?
+          say('Unique key already set')
+        else
+          say("Creating unique key node")
+          add_element(@root, 'uniqueKey').content = 'id'
+        end
+        say('Setting default operator to AND')
+        solr_query_parser_node = @root.xpath('solrQueryParser').first ||
+                                 add_element(@root, 'solrQueryParser')
+        solr_query_parser_node['defaultOperator'] = 'AND'
+      end
+
+      def add_comment(node)
+        comment_message = " *** This #{node.name} is used by Sunspot! *** "
+        unless (comment = previous_non_text_sibling(node)) && comment.comment? && comment.content =~ /Sunspot/
+          node.add_previous_sibling(Nokogiri::XML::Comment.new(@document, comment_message))
+        end
+      end
+
+      def types_node
+        @types_node ||= @root.xpath('/schema/types').first
+      end
+
+      def fields_node
+        @fields_node ||= @root.xpath('/schema/fields').first
+      end
+
+      def to_solr_class(class_name)
+        if class_name =~ /\./
+          class_name
+        else
+          "solr.#{class_name}"
+        end
+      end
+
+      def previous_non_text_sibling(node)
+        if previous_sibling = node.previous_sibling
+          if previous_sibling.node_type == :text
+            previous_non_text_sibling(previous_sibling)
+          else
+            previous_sibling
+          end
+        end
+      end
+
+      #
+      # All of the possible combinations of variants
+      #
+      def variant_combinations(type_options)
+        invariants = type_options['invariants'] || {}
+        combinations = []
+        variants = @config['variants'].reject { |name, suffix| invariants.has_key?(name) }
+        0.upto(2 ** variants.length - 1) do |b|
+          combinations << combination = []
+          variants.each_with_index do |variant, i|
+            combination << variant if b & 1<<i > 0
+          end
+          invariants.each do |name, value|
+            if value
+              combination << [name]
+            end
+          end
+        end
+        combinations
+      end
+
+      def say(message)
+        if @verbose
+          STDOUT.puts(message)
+        end
+      end
+    end
+  end
+end