mongoid-sphinx 0.0.1

data/README.markdown

@@ -0,0 +1,135 @@
+# MongoidSphinx
+
+## Fork info
+
+This is a fork of http://github.com/burke/mongosphinx with many changes to simplify and support Mongoid.
+
+## General info
+
+The MongoidSphinx library implements an interface between MongoDB and Sphinx 
+supporting Mongoid to automatically index objects in Sphinx. It tries to
+act as transparent as possible: Just an additional method in Mongoid
+and some Sphinx configuration are needed to get going.
+
+## Prerequisites
+
+MongoidSphinx needs gems Mongoid and Riddle as well as a running Sphinx
+and a MongoDB installation. Just add this to your Gemfile:
+
+    gem riddle
+    gem mongoid
+    gem mongoidsphinx, :require => 'mongoid_sphinx'
+
+No additional configuraton is needed for interfacing with MongoDB: Setup is
+done when Mongoid is able to talk to the MongoDB server.
+
+A proper "sphinx.conf" file and a script for retrieving index data have to
+be provided for interfacing with Sphinx: Sorry, no ThinkingSphinx like
+magic... :-) Depending on the amount of data, more than one index may be used
+and indexes may be consolidated from time to time.
+
+This is a sample configuration for a single "main" index:
+
+    searchd {
+      address = 0.0.0.0
+      port = 3312
+
+      log = ./sphinx/searchd.log
+      query_log = ./sphinx/query.log
+      pid_file = ./sphinx/searchd.pid
+    }
+
+    source mongoblog {
+      type = xmlpipe2
+  
+      xmlpipe_command = rake sphinx:genxml --silent
+    }
+
+    index mongoblog {
+      source = mongoblog
+
+      charset_type = utf-8
+      path = ./sphinx/sphinx_index_main
+    }
+
+Notice the line "xmlpipe_command =". This is what the indexer runs to generate 
+its input. You can change this to whatever works best for you, but I set it up as 
+a rake task, with the following in `lib/tasks/sphinx.rake` .
+
+Here :fields is a list of fields to export. Performance tends to suffer if you export
+everything, so you'll probably want to just list the fields you're indexing.
+
+    namespace :sphinx do
+      task :genxml => :environment do
+        MongoidSphinx::Indexer::XMLDocset.stream(Food)
+      end
+    end
+
+This uses MongoDB cursor to better stream collection. Instead of offset. See: http://groups.google.com/group/mongodb-user/browse_thread/thread/35f01db45ea3b0bd/96ebc49b511a6b41?lnk=gst&q=skip#96ebc49b511a6b41
+
+## Models
+
+Use method _search_index_ to enable indexing of a model. You must provide a list of
+attribute keys.
+
+A side effect of calling this method is, that MongoidSphinx overrides the
+default of letting MongoDB create new IDs: Sphinx only allows numeric IDs and
+MongoidSphinx forces new objects with the name of the class, a hyphen and an
+integer as ID (e.g. _Post-38497238_). Again: Only these objects are
+indexed due to internal restrictions of Sphinx.
+
+Sample:
+
+    class Post
+      include Mongoid::Sphinx
+
+      field :title
+      field :body
+
+      search_index :title, :body
+    end
+
+You must also create a config/sphinx.yml file with the host and port of your sphinxd process like so:
+
+    development:
+      address: localhost
+      port: 3312
+      
+    staging:
+      address: localhost
+      port: 3312
+      
+    production:
+      address: localhost
+      port: 3312
+
+## Queries
+
+An additional instance method <tt>search</tt> is added for each
+search indexed model. This method takes a Sphinx query like
+`foo @title bar`, runs it within the context of the current class and returns
+an Array of matching MongoDB documents.
+
+Samples:
+
+    Post.search('first')
+    => [...]
+
+    post = Post.search('this is @title post').first
+    post.title
+    => "First Post"
+    post.class
+    => Post
+
+Additional options _:match_mode_, _:limit_ and
+_:max_matches_ can be provided to customize the behavior of Riddle.
+Option _:raw_ can be set to _true_ to do no lookup of the
+document IDs but return the raw IDs instead.
+
+Sample:
+
+    Post.search('my post', :limit => 100)
+
+## Copyright
+
+Copyright (c) 2010 RedBeard Tech. See LICENSE for details.

data/lib/mongoid_sphinx.rb

@@ -0,0 +1,8 @@
+require "mongoid"
+require "riddle"
+
+require 'mongoid_sphinx/configuration'
+require 'mongoid_sphinx/indexer'
+require 'mongoid_sphinx/multi_attribute'
+require 'mongoid_sphinx/mongoid/identity'
+require 'mongoid_sphinx/mongoid/sphinx'

data/lib/mongoid_sphinx/configuration.rb

@@ -0,0 +1,31 @@
+require 'erb'
+require 'singleton'
+
+module MongoidSphinx
+  class Configuration
+    include Singleton
+    
+    attr_accessor :address, :port, :configuration
+    
+    def client
+      @configuration ||= parse_config
+      Riddle::Client.new address, port
+    end
+    
+    private
+
+    # Parse the config/sphinx.yml file - if it exists
+    #
+    def parse_config
+      path = "#{Rails.root}/config/sphinx.yml"
+      return unless File.exists?(path)
+
+      conf = YAML::load(ERB.new(IO.read(path)).result)[Rails.env]
+
+      conf.each do |key,value|
+        self.send("#{key}=", value) if self.respond_to?("#{key}=")
+      end
+    end
+    
+  end
+end

data/lib/mongoid_sphinx/indexer.rb

@@ -0,0 +1,123 @@
+# MongoidSphinx, a full text indexing extension for MongoDB using
+# Sphinx.
+#
+# This file contains the MongoidSphinx::Indexer::XMLDocset and
+# MongoidSphinx::Indexer::XMLDoc classes.
+
+module MongoidSphinx #:nodoc:
+
+  # Module Indexer contains classes for creating XML input documents for the
+  # indexer. Each Sphinx index consists of a single "sphinx:docset" with any
+  # number of "sphinx:document" tags.
+  #
+  # The XML source can be generated from an array of CouchRest objects or from
+  # an array of Hashes containing at least fields "classname" and "_id"
+  # as returned by MongoDB view "MongoSphinxIndex/couchrests_by_timestamp".
+  #
+  # Sample:
+  #
+  #   rows = [{ 'name' => 'John', 'phone' => '199 43828',
+  #             'classname' => 'Address', '_id' => 'Address-234164'
+  #           },
+  #           { 'name' => 'Sue', 'mobile' => '828 19439',
+  #             'classname' => 'Address', '_id' => 'Address-422433'
+  #          }
+  #   ]
+  #   puts MongoSphinx::Indexer::XMLDocset.new(rows).to_s
+  #
+  #   <?xml version="1.0" encoding="utf-8"?>
+  #   <sphinx:docset>
+  #     <sphinx:schema>
+  #       <sphinx:attr name="csphinx-class" type="multi"/>
+  #       <sphinx:field name="classname"/>
+  #       <sphinx:field name="name"/>
+  #       <sphinx:field name="phone"/>
+  #       <sphinx:field name="mobile"/>
+  #       <sphinx:field name="created_at"/>
+  #     </sphinx:schema>
+  #     <sphinx:document id="234164">
+  #       <csphinx-class>336,623,883,1140</csphinx-class>
+  #       <classname>Address</classname>
+  #       <name><![CDATA[[John]]></name>
+  #       <phone><![CDATA[[199 422433]]></phone>
+  #       <mobile><![CDATA[[]]></mobile>
+  #       <created_at><![CDATA[[]]></created_at>
+  #     </sphinx:document>
+  #     <sphinx:document id="423423">
+  #       <csphinx-class>336,623,883,1140</csphinx-class>
+  #       <classname>Address</classname>
+  #       <name><![CDATA[[Sue]]></name>
+  #       <phone><![CDATA[[]]></phone>
+  #       <mobile><![CDATA[[828 19439]]></mobile>
+  #       <created_at><![CDATA[[]]></created_at>
+  #     </sphinx:document>
+  #   </sphinx:docset>"
+
+  module Indexer
+
+    # Class XMLDocset wraps the XML representation of a document to index. It
+    # contains a complete "sphinx:docset" including its schema definition.
+    
+    class XMLDocset
+
+      # Streams xml of all objects in a klass to the stdout. This makes sure you can process large collections.
+      #
+      # Options:
+      #  attributes (required) - The attributes that are put in the sphinx xml.
+      #
+      # Example:
+      #  MongoSphinx::Indexer::XMLDocset.stream(Document, :attributes => %w(title content))
+      # This will create an XML stream to stdout. 
+      #
+      # Configure in your sphinx.conf like
+      #  xmlpipe_command = ./script/runner "MongoSphinx::Indexer::XMLDocset.stream(Document, :attributes => %w(title content))"
+      #
+      def self.stream(klass)
+        STDOUT.sync = true # Make sure we really stream..
+
+        puts '<?xml version="1.0" encoding="utf-8"?>'
+        puts '<sphinx:docset>'
+
+        # Schema
+        puts '<sphinx:schema>'
+        klass.search_fields.each do |key, value|
+          puts "<sphinx:field name=\"#{key}\"/>"
+        end
+        # FIXME: What is this attribute?
+        puts '<sphinx:field name="classname"/>'
+        puts '<sphinx:attr name="csphinx-class" type="multi"/>'
+        puts '</sphinx:schema>'
+        
+        collection = Mongoid.database.collection(klass.collection.name)
+        collection.find.each do |document_hash|
+          XMLDoc.stream_for_hash(document_hash, klass)
+        end
+
+        puts '</sphinx:docset>'
+      end
+      
+    end
+
+    class XMLDoc
+
+      def self.stream_for_hash(hash, klass)
+        sphinx_compatible_id = hash['_id'].to_s.to_i - 100000000000000000000000
+        
+        puts "<sphinx:document id=\"#{sphinx_compatible_id}\">"
+        # FIXME: Should we include this?
+        puts '<csphinx-class>'
+        puts MongoidSphinx::MultiAttribute.encode(klass.to_s)
+        puts '</csphinx-class>'
+        puts "<classname>#{klass.to_s}</classname>"
+        
+        klass.search_fields.each do |key|
+          value = hash[key.to_s]
+          puts "<#{key}><![CDATA[[#{value}]]></#{key}>"
+        end
+
+        puts '</sphinx:document>'
+      end
+      
+    end
+  end
+end

data/lib/mongoid_sphinx/mongoid/identity.rb

@@ -0,0 +1,23 @@
+module Mongoid
+  class Identity
+    
+    protected
+      
+      # Return an id that is sphinx compatible
+      def generate_id
+        while true
+          id = 100000000000000000000000 + rand(4294967294) # 4,294,967,295 is the theoretical max number of documents a 32 bit sphinx install can index
+          candidate = id.to_s
+        
+          begin
+            @document.class.find(candidate) # Resource not found exception if available
+          rescue Mongoid::Errors::DocumentNotFound
+            id = BSON::ObjectId.from_string(candidate)
+            break
+          end
+        end
+        @document.using_object_ids? ? id : id.to_s
+      end
+    
+  end
+end

data/lib/mongoid_sphinx/mongoid/sphinx.rb

@@ -0,0 +1,49 @@
+# MongoidSphinx, a full text indexing extension for MongoDB/Mongoid using
+# Sphinx.
+
+module Mongoid
+  module Sphinx
+    extend ActiveSupport::Concern
+    included do
+      cattr_accessor :search_fields 
+    end
+    
+    module ClassMethods
+      def search_index(*fields)
+        self.search_fields = fields
+      end
+      
+      def search(query, options = {})
+        client = MongoidSphinx::Configuration.instance.client
+                 
+        query = query + " @classname #{@document.class.to_s}"
+        
+        client.match_mode = options[:match_mode] || :extended
+        client.limit = options[:limit] if options.key?(:limit)
+        client.max_matches = options[:max_matches] if options.key?(:max_matches)
+        
+        if options.key?(:sort_by)
+          client.sort_mode = :extended
+          client.sort_by = options[:sort_by]
+        end
+        
+        result = client.query(query)
+        
+        #TODO
+        if result and result[:status] == 0 and (matches = result[:matches])
+          classname = nil
+          ids = matches.collect do |row|
+            classname = MongoidSphinx::MultiAttribute.decode(row[:attributes]['csphinx-class'])
+            row[:doc].to_s rescue nil
+          end.compact
+          
+          return ids if options[:raw]
+          return Object.const_get(classname).find(ids)
+        else
+          return []
+        end
+      end
+    end
+    
+  end
+end

data/lib/mongoid_sphinx/multi_attribute.rb

@@ -0,0 +1,57 @@
+# MongoSphinx, a full text indexing extension for MongoDB using
+# Sphinx.
+#
+# This file contains the MongoSphinx::MultiAttribute class.
+
+# Namespace module for the MongoSphinx gem.
+
+module MongoidSphinx #:nodoc:
+
+  # Module MultiAttribute implements helpers to translate back and
+  # forth between Ruby Strings and an array of integers suitable for Sphinx
+  # attributes of type "multi".
+  #
+  # Background: Getting an ID as result for a query is OK, but for example to
+  # allow cast safety, we need an aditional attribute. Sphinx supports
+  # attributes which are returned together with the ID, but they behave a
+  # little different than expected: Instead we can use arrays of integers with
+  # ASCII character codes. These values are returned in ascending (!) order of
+  # value (yes, sounds funny but is reasonable from an internal view to
+  # Sphinx). So we mask each byte with 0x0100++ to keep the order...
+  #
+  # Sample:
+  #
+  #   MongoSphinx::MultiAttribute.encode('Hello')
+  #   => "328,613,876,1132,1391"
+  #   MongoSphinx::MultiAttribute.decode('328,613,876,1132,1391')
+  #   => "Hello"
+
+  module MultiAttribute
+
+    # Returns an numeric representation of a Ruby String suitable for "multi"
+    # attributes of Sphinx.
+    #
+    # Parameters:
+    #
+    # [str] String to translate
+
+    def self.encode(str)
+      offset = 0
+      return str.bytes.collect { |c| (offset+= 0x0100) + c }.join(',')
+    end
+
+    # Returns the original MongoDB ID created from a Sphinx ID. Only works if
+    # the ID was created from a MongoDB ID before!
+    #
+    # Parameters:
+    #
+    # [multi] Sphinx "multi" attribute to translate back
+
+    def self.decode(multi)
+      offset = 0
+      multi = multi.split(',') if not multi.kind_of? Array
+
+      return multi.collect {|x| (x.to_i-(offset+=0x0100)).chr}.to_s
+    end
+  end
+end

data/lib/mongoid_sphinx/version.rb

@@ -0,0 +1,3 @@
+module MongoidSphinx #:nodoc
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification 
+name: mongoid-sphinx
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Matt Hodgson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-21 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 62196421
+        segments: 
+        - 2
+        - 0
+        - 0
+        - beta
+        - 19
+        version: 2.0.0.beta.19
+  requirement: *id001
+  type: :runtime
+  name: mongoid
+  prerelease: false
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 19
+        segments: 
+        - 1
+        - 1
+        - 0
+        version: 1.1.0
+  requirement: *id002
+  type: :runtime
+  name: riddle
+  prerelease: false
+description: A full text indexing extension for MongoDB using Sphinx and Mongoid.
+email: 
+- mhodgson@scenario4.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/mongoid_sphinx/configuration.rb
+- lib/mongoid_sphinx/indexer.rb
+- lib/mongoid_sphinx/mongoid/identity.rb
+- lib/mongoid_sphinx/mongoid/sphinx.rb
+- lib/mongoid_sphinx/multi_attribute.rb
+- lib/mongoid_sphinx/version.rb
+- lib/mongoid_sphinx.rb
+- README.markdown
+has_rdoc: true
+homepage: http://github.com/mhodgson/mongoid-sphinx
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A full text indexing extension for MongoDB using Sphinx and Mongoid.
+test_files: []
+