xapit 0.2.7 → 0.3.0

data/lib/xapit/client/facet.rb

@@ -0,0 +1,11 @@
+module Xapit
+  module Client
+    class Facet
+      attr_reader :name, :options
+      def initialize(attribute, options, applied_facets = [])
+        @name = attribute.to_s.gsub("_", " ").gsub(/\b([a-z])/) { $1.to_s.upcase }
+        @options = options.map { |option| FacetOption.new(attribute, option, applied_facets) }
+      end
+    end
+  end
+end

data/lib/xapit/client/facet_option.rb

@@ -0,0 +1,29 @@
+module Xapit
+  module Client
+    class FacetOption
+      attr_reader :count, :attribute
+      def initialize(attribute, option, applied_facets = [])
+        @attribute = attribute
+        @value = option[:value]
+        @count = option[:count].to_i
+        @applied_facets = applied_facets
+      end
+
+      def identifier
+        Xapit.facet_identifier(@attribute, @value)
+      end
+
+      def name
+        @value
+      end
+
+      def to_param
+        if @applied_facets.include? identifier
+          (@applied_facets - [identifier]).join('-')
+        else
+          (@applied_facets + [identifier]).join("-")
+        end
+      end
+    end
+  end
+end

data/lib/xapit/client/index_builder.rb

@@ -0,0 +1,67 @@
+module Xapit
+  module Client
+    class IndexBuilder
+      attr_reader :attributes
+      def initialize
+        @attributes = {}
+      end
+
+      def text(*args, &block)
+        add_attribute(:text, *args, &block)
+      end
+
+      def field(*args, &block)
+        add_attribute(:field, *args, &block)
+      end
+
+      def sortable(*args, &block)
+        add_attribute(:sortable, *args, &block)
+      end
+
+      def facet(name, custom_name = nil, &block)
+        options = {}
+        options[:name] = custom_name if custom_name
+        add_attribute(:facet, name, options, &block)
+      end
+
+      def add_document(member)
+        Xapit.database.add_document(document_data(member))
+      end
+
+      def remove_document(member)
+        Xapit.database.remove_document(document_data(member))
+      end
+
+      def update_document(member)
+        Xapit.database.update_document(document_data(member))
+      end
+
+      def document_data(member)
+        data = {:class => member.class.name, :id => member.id, :attributes => {}}
+        attributes.each do |name, options|
+          value = member.send(name)
+          value = options[:_block].call(value) if options[:_block]
+          data[:attributes][name] = options.merge(:value => value)
+        end
+        data
+      end
+
+      def facets
+        attributes.keys.select do |attribute|
+          attributes[attribute][:facet]
+        end
+      end
+
+      private
+
+      def add_attribute(type, *args, &block)
+        options = args.last.kind_of?(Hash) ? args.pop : {}
+        args.each do |attribute|
+          @attributes[attribute] ||= {}
+          @attributes[attribute][type] = options
+          @attributes[attribute][:_block] = block if block
+        end
+      end
+    end
+  end
+end

data/lib/xapit/client/membership.rb

@@ -0,0 +1,46 @@
+module Xapit
+  module Client
+    module Membership
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      module ClassMethods
+        def xapit(&block)
+          @xapit_index_builder = IndexBuilder.new
+          @xapit_index_builder.instance_eval(&block)
+          include AdditionalMethods unless include?(AdditionalMethods)
+        end
+      end
+
+      module AdditionalMethods
+        def self.included(base)
+          base.extend ClassMethods
+          base.xapit_model_adapter.setup
+        end
+
+        module ClassMethods
+          def xapit_model_adapter
+            @xapit_model_adapter ||= Xapit::Client::AbstractModelAdapter.adapter_class(self).new(self)
+          end
+
+          def xapit_index_builder
+            @xapit_index_builder
+          end
+
+          def xapit_search(*args)
+            Collection.new.in_classes(self).include_facets(*xapit_index_builder.facets).search(*args)
+          end
+
+          def search(*args)
+            xapit_search(*args)
+          end
+        end
+
+        def search_similar(*args)
+          self.class.search(*args).similar_to(self)
+        end
+      end
+    end
+  end
+end

data/lib/xapit/client/model_adapters/abstract_model_adapter.rb

@@ -0,0 +1,30 @@
+module Xapit
+  module Client
+    class AbstractModelAdapter
+      def self.inherited(subclass)
+        @@subclasses ||= []
+        @@subclasses << subclass
+      end
+
+      def self.adapter_class(model_class)
+        @@subclasses.detect { |subclass| subclass.for_class?(model_class) } || DefaultModelAdapter
+      end
+
+      def self.for_class?(model_class)
+        false # override in subclass
+      end
+
+      def setup
+        # override in subclass
+      end
+
+      def index_all
+        # override in subclass
+      end
+
+      def initialize(model_class)
+        @model_class = model_class
+      end
+    end
+  end
+end

data/lib/xapit/client/model_adapters/active_record_adapter.rb

@@ -0,0 +1,27 @@
+module Xapit
+  module Client
+    class ActiveRecordAdapter < AbstractModelAdapter
+      def self.for_class?(model_class)
+        model_class <= ActiveRecord::Base
+      end
+
+      def setup
+        @model_class.after_create do |member|
+          member.class.xapit_index_builder.add_document(member) if Xapit.config[:enabled]
+        end
+        @model_class.after_update do |member|
+          member.class.xapit_index_builder.update_document(member) if Xapit.config[:enabled]
+        end
+        @model_class.after_destroy do |member|
+          member.class.xapit_index_builder.remove_document(member) if Xapit.config[:enabled]
+        end
+      end
+
+      def index_all
+        @model_class.find_each do |member|
+          member.class.xapit_index_builder.add_document(member)
+        end
+      end
+    end
+  end
+end

data/lib/xapit/client/model_adapters/default_model_adapter.rb

@@ -0,0 +1,7 @@
+module Xapit
+  module Client
+    class DefaultModelAdapter < AbstractModelAdapter
+      # This adapter is used when no matching adapter is found
+    end
+  end
+end

data/lib/xapit/client/railtie.rb

@@ -0,0 +1,18 @@
+module Xapit
+  module Client
+    class Railtie < Rails::Railtie
+      initializer "xapit.config" do
+        path = Rails.root.join("config/xapit.yml")
+        Xapit.load_config(path, Rails.env) if path.exist?
+      end
+
+      initializer "xapit.membership" do
+        ActiveRecord::Base.send(:include, Xapit::Client::Membership) if defined? ActiveRecord
+      end
+
+      rake_tasks do
+        load File.expand_path("../tasks.rb", __FILE__)
+      end
+    end
+  end
+end

data/lib/xapit/client/remote_database.rb

@@ -0,0 +1,21 @@
+module Xapit
+  module Client
+    class RemoteDatabase
+      def initialize(url)
+        @url = url
+      end
+
+      Xapit::Server::Database::COMMANDS.each do |command|
+        define_method(command) do |options|
+          request(command, options)
+        end
+      end
+
+      def request(command, options)
+        uri = URI.parse("#{@url}/xapit/#{command}")
+        response = Net::HTTP.start(uri.host, uri.port) { |http| http.request_post(uri.path, options.to_json) }
+        Xapit.symbolize_keys(JSON.parse("[#{response.body}]").first) # terrible hack for handling simple objects
+      end
+    end
+  end
+end

data/lib/xapit/client/tasks.rb

@@ -0,0 +1,18 @@
+require "rack"
+require "fileutils"
+
+namespace :xapit do
+  desc "Index all models for Xapit search"
+  task :index => :environment do
+    raise "No Xapian database specified in config." if Xapit.config[:database_path].blank?
+    FileUtils.rm_rf("tmp/xapit") if File.exist? "tmp/xapit"
+    FileUtils.mv(Xapit.config[:database_path], "tmp/xapit") if File.exist? Xapit.config[:database_path]
+    models = ActiveRecord::Base.subclasses
+    Dir[Rails.root.join("app", "models", "**", "*.rb")].each do |file|
+      # I hate to rescue nil, maybe there's a better way to handle unknown constants
+      models << File.basename(file, ".*").classify.constantize rescue nil
+    end
+    xapit_models = models.compact.uniq.select { |m| m.respond_to? :xapit_model_adapter }
+    Xapit.index(*xapit_models)
+  end
+end

data/lib/xapit/server/app.rb

@@ -0,0 +1,27 @@
+module Xapit
+  module Server
+    class App
+      def call(env)
+        request = Rack::Request.new(env)
+        command = request.path[%r</xapit/(.+)>, 1]
+        if Database::COMMANDS.include? command
+          action(command, request.body.gets)
+        else
+          render :status => 404
+        end
+      end
+
+      def action(command, json)
+        data = Xapit.symbolize_keys(JSON.parse(json))
+        render :content => Xapit.database.send(command, data).to_json
+      end
+
+      def render(options = {})
+        options[:status] ||= 200
+        options[:content] ||= ""
+        options[:content_type] ||= "text/html"
+        [options[:status], {"Content-Type" => options[:content_type]}, [options[:content]]]
+      end
+    end
+  end
+end

data/lib/xapit/server/database.rb

@@ -0,0 +1,47 @@
+module Xapit
+  module Server
+    class Database
+      COMMANDS = %w[query add_document remove_document update_document spelling_suggestion]
+
+      def initialize(path)
+        @path = path
+      end
+
+      def xapian_database
+        @xapian_database ||= load_database
+      end
+
+      def add_document(data)
+        xapian_database.add_document(Indexer.new(data).document)
+      end
+
+      def remove_document(data)
+        xapian_database.delete_document(Indexer.new(data).id_term)
+      end
+
+      def update_document(data)
+        indexer = Indexer.new(data)
+        xapian_database.replace_document(indexer.id_term, indexer.document)
+      end
+
+      def query(data)
+        Query.new(data).data
+      end
+
+      def spelling_suggestion(data)
+        Query.new(data).spelling_suggestion
+      end
+
+      private
+
+      def load_database
+        if @path
+          FileUtils.mkdir_p(File.dirname(@path)) unless File.exist?(File.dirname(@path))
+          Xapian::WritableDatabase.new(@path, Xapian::DB_CREATE_OR_OPEN)
+        else
+          Xapian.inmemory_open
+        end
+      end
+    end
+  end
+end

data/lib/xapit/server/indexer.rb

@@ -0,0 +1,138 @@
+module Xapit
+  module Server
+    class Indexer
+      def initialize(data)
+        @data = data
+      end
+
+      def database
+        Xapit.database.xapian_database
+      end
+
+      def document
+        document = Xapian::Document.new
+        document.data = id
+        terms.each { |term, weight| document.add_term(term, weight) }
+        text_terms.each { |term, weight| database.add_spelling(term, weight) } if Xapit.config[:spelling]
+        values.each { |index, value| document.add_value(index, value) }
+        save_facets
+        document
+      end
+
+      def id
+        "#{@data[:class]}-#{@data[:id]}"
+      end
+
+      def id_term
+        "Q#{id}"
+      end
+
+      def terms
+        base_terms + text_terms + stemmed_text_terms + field_terms + facet_terms
+      end
+
+      def values
+        values = {}
+        each_value do |index, value|
+          if values[index]
+            values[index] += "\3#{value}" # multiple values are split back out on the query side
+          else
+            values[index] = value
+          end
+        end
+        values
+      end
+
+      def text_terms
+        each_attribute(:text) do |name, value, options|
+          value.to_s.split(/\s+/u).map { |w| w.gsub(/[^\w]/u, "") }.map(&:downcase).map do |term|
+            [term, options[:weight] || 1]
+          end
+        end.flatten(1)
+      end
+
+      def stemmed_text_terms
+        if stemmer
+          each_attribute(:text) do |name, value, options|
+            value.to_s.split(/\s+/u).map { |w| w.gsub(/[^\w]/u, "") }.map(&:downcase).map do |term|
+              ["Z#{stemmer.call(term)}", options[:weight] || 1]
+            end
+          end.flatten(1)
+        else
+          []
+        end
+      end
+
+      def field_terms
+        each_attribute(:field) do |name, value, options|
+          ["X#{name}-#{parse_field(value)}", 1]
+        end
+      end
+
+      def facet_terms
+        each_attribute(:facet) do |name, value, options|
+          ["F#{Xapit.facet_identifier(name, value)}", 1]
+        end
+      end
+
+      def save_facets
+        each_attribute(:facet) do |name, value, options|
+          id = Xapit.facet_identifier(name, value)
+          unless database.term_exists("Xid-#{id}")
+            document = Xapian::Document.new
+            document.data = "#{name}|||#{value}"
+            document.add_term("CFacetOption")
+            document.add_term("Xid-#{id}")
+            database.add_document(document)
+          end
+        end
+      end
+
+      private
+
+      def stemmer
+        @stemmer ||= Xapian::Stem.new(Xapit.config[:stemming]) if Xapit.config[:stemming]
+      end
+
+      def base_terms
+        [["C#{@data[:class]}", 1], [id_term, 1]]
+      end
+
+      def parse_field(value)
+        if value.kind_of? Time
+          value.to_i
+        else
+          value.to_s.downcase
+        end
+      end
+
+      def each_value
+        each_attribute(:field) do |name, value, options|
+          yield(Xapit.value_index(:field, name), Xapit.serialize_value(value))
+        end
+        each_attribute(:sortable) do |name, value, options|
+          yield(Xapit.value_index(:sortable, name), Xapit.serialize_value(value))
+        end
+        each_attribute(:facet) do |name, value, options|
+          yield(Xapit.value_index(:facet, name), value)
+        end
+      end
+
+      def each_attribute(type)
+        if @data[:attributes]
+          @data[:attributes].map do |name, options|
+            if options.has_key? type
+              if options[:value].kind_of? Array
+                options[:value].map { |value| yield(name, value, options[type]) }
+              else
+                [yield(name, options[:value], options[type])]
+              end
+            end
+          end.compact.flatten(1)
+        else
+          []
+        end
+      end
+    end
+  end
+end