search_do 0.2.0

data/CHANGELOG

@@ -0,0 +1 @@
+ - added html_snippet (attr_accessor :html_snippet)

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2005 Rick Olson
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,63 @@
+= search_do
+* Library for fulltext search integration with active record.
+* Build to support multiple search Backends
+* acts_as_searchable Successor
+
+
+== Pre-requisites
+A working Hyper Estraier instance, setup instructions:
+
+* Setup Instructions: http://pragmatig.wordpress.com/2008/05/06/getting-started-with-acts_as_searchable-on-ubuntu/
+* In-depth Documentation: http://hyperestraier.sourceforge.net/nguide-en.html
+* rake search:node:create MODEL=User RAILS_ENV=production - for every model/environment you use
+
+
+== Install
+* script/plugin install git://github.com/grosser/search_do.git
+* install will_paginate to use "paginate_by_fulltext_search" (Instruction: http://github.com/mislav/will_paginate/wikis/installation)
+
+
+== Usage
+
+  #MODEL
+  class User < ActiveRecord::Base
+    acts_as_searchable(
+      #fields the will be found in fulltext search
+      :searchable_fields => [:name,:website,:city,:about],
+      #fields used for attribute search/ordering
+      :attributes => {:name=>nil,:city=>nil,:country=>nil,:age=>nil}
+    )
+    attr_accessor :html_snippet #add this to get html snippets on your results (see below)
+  end
+
+  #SEARCH
+  Users who:
+   - contain 'hello' in any of their searchable fields
+   - whose website attribute contains 'www' (contains search for strings)
+   - whose age is 1 (exact match for numbers/dates)
+   - sorted by age ASC
+  @results = User.paginate_by_fulltext_search('hello',:attributes=>{:website=>'www',:age=>1},:order=>'age ASC',:page=>1,:per_page=>20)
+
+  (Same can be done without pagination: User.fulltext_search)
+
+  #SNIPPETS
+  Each record found with a fulltext-search (not a attribute-only search) contains a snippet
+  of the surrounding where the phrase was found.
+
+  User.fulltext_search('hello') => user.html_snippet == "id like to say <b>hello</b> to my fellow students"
+
+  NOTE: html_snippet will not contain HTML except for the <b>, so there is no need to escape it.
+
+
+== Hyperestraier Features
+ - Phrase search, regular expressions, attribute search, and similarity search
+ - Snippet retrival
+ - UTF8 support
+ - Web interface
+ - Built in P2P clustering of index servers
+
+
+== Origin
+Original is written by scoop see
+* http://github.com/scoop/acts_as_searchable/tree/master
+* http://poocs.net/2006/4/6/introducing-acts-as-searchable

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rake/rdoctask'
+require 'spec'
+
+desc 'Default: run specs_all.'
+task :default => :spec_all
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new {|t| t.spec_opts = ['--color']}
+
+desc "Run specs both AR-latest and AR-2.0.x"
+task :spec_all do
+  ar20xs = (::Gem.source_index.find_name("activerecord", "<2.1") & \
+            ::Gem.source_index.find_name("activerecord", ">=2.0"))
+  if ar20xs.empty?
+    Rake::Task[:spec].invoke
+  else
+    ar20 = ar20xs.sort_by(&:version).last
+    system("rake spec")
+    system("rake spec AR=#{ar20.version}")
+  end
+end
+
+desc 'Generate documentation for the acts_as_searchable plugin.'
+Rake::RDocTask.new(:rdoc) do |doc|
+  doc.rdoc_dir = 'rdoc'
+  doc.title    = 'SearchDo'
+  doc.options << '--line-numbers' << '--inline-source'
+  doc.rdoc_files.include('README.rdoc')
+  doc.rdoc_files.include('lib/**/*.rb')
+end
+
+begin
+  require 'jeweler'
+  project_name = 'search_do'
+
+  Jeweler::Tasks.new do |gem|
+    gem.name = project_name
+    gem.summary = "AR: Hyperestraier integration"
+    gem.email = "moronatural@gmail.com"
+    gem.homepage = "http://github.com/grosser/#{project_name}"
+    gem.authors = ["MOROHASHI Kyosuke"]
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/TESTING

@@ -0,0 +1,6 @@
+If you have test errors by HyperEstraier, change cache settings:
+
+edit '${estraier_index_dir}/_conf' and set 'cachernum' parameter to 0.
+(Ubuntu: /var/lib/hyperestraier/estmaster/_conf)
+then restart hyperestraier
+(Ubuntu: sudo /etc/init.d/hyperestraier restart)

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/examples/he_search.rb

@@ -0,0 +1,13 @@
+# a module to include in your controllers
+# params:
+# q => search word
+# search => {name=>something}
+# order => name DESC
+# define the constant PER_PAGE in your controller so it can be used here
+module Modules
+  module HeSearch
+    def current_objects
+      @current_objects ||= current_model.fulltext_results(params[:q],:page=>params[:page],:per_page=>self.class.const_get('PER_PAGE'),:attributes=>params[:search],:order=>params[:order])
+    end
+  end
+end

data/examples/person.rb

@@ -0,0 +1,20 @@
+# this is a person indexed by hyperestraier
+# it should have name/city/country:string about:text online:boolean
+class Person < ActiveRecord::Base
+  acts_as_searchable(
+    :ignore_timestamp=>true, #do not add timestamps to search_backend
+    :searchable_fields => [:name,:website,:city,:about],
+    :attributes => {"@title"=>:name,:name=>nil,:city=>nil,:country=>nil}#attribute results
+    #add @title for higher search weight on this attribute
+  )
+  attr_accessor :html_snippet #so we get nice html snippets for our search results...
+
+  #customize the length/widths of the fetched snippets
+  search_backend.connection.set_snippet_width(150,20,20)#total,beginning(>0!),around
+
+  #only index a person when it is online
+  def add_to_index_with_online_check
+    add_to_index_without_online_check if online?
+  end
+  alias_method_chain :add_to_index, :online_check
+end

data/init.rb

@@ -0,0 +1 @@
+require 'search_do'

data/lib/estraier_admin.rb

@@ -0,0 +1,47 @@
+require 'net/http'
+require 'webrick/httputils'
+
+class EstraierAdmin
+  RequestFailed = Class.new(StandardError)
+  CREATE_NODE_ACTION = 8
+  DELETE_NODE_ACTION = 9
+  DEFAULT_CONFIG = {
+    :host=> "localhost",
+    :port=>1978,
+    :user=>"admin",
+    :password=>"admin"
+  }.freeze
+
+  # requires host 
+  def initialize(config={})
+    @config = DEFAULT_CONFIG.dup
+    config.each { |k, v| @config[k.to_sym] = v }
+  end
+
+  def create_node(name, label = nil)
+    label ||= name
+    request_or_raise(:name=>name, :action=>CREATE_NODE_ACTION, :label=>label)
+    return true
+  end
+
+  def delete_node(name)
+    request_or_raise(:name=>name, :action=>DELETE_NODE_ACTION, :sure=>1)
+    return true
+  end
+
+  private
+  def request_or_raise(params, path="/master_ui")
+    req = Net::HTTP::Post.new(path)
+    req.basic_auth(@config[:user], @config[:password])
+    res = Net::HTTP.start(@config[:host], @config[:port]) do |http|
+            http.request(req, build_body(params))
+          end
+    raise if res.code.to_i >= 400 
+    return true
+  end
+
+  def build_body(params={})
+    params.map{|k,v| "#{k}=#{WEBrick::HTTPUtils.escape_form(v.to_s)}" }.join('&')
+  end
+end
+

data/lib/search_do/backends/hyper_estraier/estraier_pure_extention.rb

@@ -0,0 +1,61 @@
+require 'search_do/backends/hyper_estraier'
+
+module SearchDo::Backends
+  module HyperEstraier::EstraierPureExtention
+    def self.included(base)
+      [Node,Condition,NodeResult,ResultDocument].each do |klas|
+        basic_class_name = klas.to_s.split('::').last
+        base.const_get(basic_class_name).send(:include, klas)
+      end
+    end
+
+    module Node
+      def list
+        return false unless @url
+        turl = @url + "/list"
+        reqheads = [ "Content-Type: application/x-www-form-urlencoded" ]
+        reqheads.push("Authorization: Basic " + Utility::base_encode(@auth)) if @auth
+        reqbody = ""
+        resbody = StringIO::new
+        rv = Utility::shuttle_url(turl, @pxhost, @pxport, @timeout, reqheads, reqbody, nil, resbody)
+        @status = rv
+        return nil if rv != 200
+        lines = resbody.string.split(/\n/)
+        lines.collect { |l| val = l.split(/\t/) and { :id => val[0], :uri => val[1], :digest => val[2] } }
+      end
+    end
+
+    module Condition
+      def to_s
+        "phrase: %s, attrs: %s, max: %s, options: %s, order: %s, skip: %s" % [ phrase, attrs * ', ', max, options, order, skip ]
+      end
+    end
+
+    module NodeResult
+      include Enumerable
+      def each(&block)
+        (0...doc_num).each{|i| yield get_doc(i) }
+      end
+
+      def docs; map{|e| e } ; end
+
+      def first_doc; get_doc(0); end
+    end
+
+    module ResultDocument
+      #wacky snippet string parsed into a nice array of lines
+      #where the found word (here: a search for bob) is marked to be highlighted
+      #[["hallo my name is ",false],["bob",true],[" butcher",false]]
+      #output does not contain HTML
+      def snippet_a
+        snip = @snippet.sub(/^\.+/,"\n")#first ... can be understood as \n
+        snip.split("\n").reject{|x|x==''}.map do |part|
+          part.include?("\t") ? [part.split("\t")[0],true] : [part,false]
+        end
+      end
+    end
+  end
+end
+
+::EstraierPure.send(:include, SearchDo::Backends::HyperEstraier::EstraierPureExtention)
+

data/lib/search_do/backends/hyper_estraier.rb

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+require 'vendor/estraierpure'
+require 'search_do/utils'
+
+module SearchDo
+  module Backends
+    class HyperEstraier
+      SYSTEM_ATTRIBUTES = %w( uri digest cdate mdate adate title author type lang genre size weight misc )
+
+      attr_reader :connection
+      attr_accessor :node_name
+
+      DEFAULT_CONFIG = {
+        'host' => 'localhost',
+        'port' => 1978,
+        'user' => 'admin',
+        'password' => 'admin',
+        'backend' => 'hyper_estraier',
+      }.freeze
+
+      # FIXME use URI
+      def initialize(ar_class, config = {})
+        @ar_class = ar_class
+        config = DEFAULT_CONFIG.merge(config)
+        self.node_name = calculate_node_name(config)
+
+        @connection = EstraierPure::Node.new
+        @connection.set_url("http://#{config['host']}:#{config['port']}/node/#{self.node_name}")
+        @connection.set_auth(config['user'], config['password'])
+      end
+
+      def index
+        cond = EstraierPure::Condition::new
+        cond.add_attr("db_id NUMGT 0")
+        result = raw_search(cond, 1)
+        result ? result.docs : []
+      end
+
+      def search_by_db_id(id)
+        cond = EstraierPure::Condition::new
+        cond.set_options(EstraierPure::Condition::SIMPLE | EstraierPure::Condition::USUAL)
+        cond.add_attr("db_id NUMEQ #{id}")
+
+        result = raw_search(cond, 1)
+        return nil if result.nil? || result.doc_num.zero?
+        result.first_doc
+      end
+
+      def count(query, options={})
+        cond = build_fulltext_condition(query, options.merge(:count=>true))
+        benchmark("  #{@ar_class.to_s} count fulltext, Cond: #{cond.to_s}") do
+          r = raw_search(cond, 1);
+          r.doc_num rescue 0
+        end
+      end
+
+      def search_all(query, options = {})
+        cond = build_fulltext_condition(query, options)
+
+        benchmark("  #{@ar_class.to_s} fulltext search, Cond: #{cond.to_s}") do
+          result = raw_search(cond, 1);
+          result ? result.docs : []
+        end
+      end
+
+      def search_all_ids(query, options ={})
+        search_all_ids_and_raw(query,options).map {|row|row[0]}
+      end
+      
+      def search_all_ids_and_raw(query, options ={})
+        search_all(query, options).map{|doc| [doc.attr("db_id").to_i,doc] }
+      end
+
+      def add_to_index(texts, attrs)
+        doc = EstraierPure::Document::new
+        texts.reject(&:blank?).each{|t| doc.add_text(textise(t)) }
+        attrs.reject{|k,v| v.blank?}.each{|k,v| doc.add_attr(attribute_name(k), textise(v)) }
+
+        log = "  #{@ar_class.name} [##{attrs["db_id"]}] Adding to index"
+        benchmark(log){ @connection.put_doc(doc) }
+      end
+
+      def remove_from_index(db_id)
+        return unless doc = search_by_db_id(db_id)
+        log = "  #{@ar_class.name} [##{db_id}] Removing from index"
+        benchmark(log){ delete_from_index(doc) }
+      end
+
+      def clear_index!
+        benchmark("  Deleting all index"){ index.each { |d| delete_from_index(d) } }
+      end
+
+      def raw(id)
+        condition = build_fulltext_condition
+        add_attributes_to "db_id STREQ #{id}", condition
+        result = connection.search(condition, 1)
+        return unless result and result.doc_num > 0
+        result.docs[0]
+      end
+
+    private
+
+      def raw_search(cond, num)
+        @connection.search(cond, num)
+      end
+
+      def textise(obj) # :nodoc:
+        case obj
+        when Time then obj.iso8601
+        when Date, DateTime then obj.to_s # Date#to_s equals iso8601
+        else obj.to_s
+        end
+      end
+
+      def build_fulltext_condition(query='', options = {})
+        options = {:limit => 100, :offset => 0}.merge(options)
+        # options.assert_valid_keys(VALID_FULLTEXT_OPTIONS)
+
+        cond = EstraierPure::Condition::new
+        cond.set_options(EstraierPure::Condition::SIMPLE | EstraierPure::Condition::USUAL)
+
+        cond.set_phrase Utils.cleanup_query(query)
+
+        #add a always-true condition to trigger find all
+        add_attributes_to("db_id NUMGT 0",cond) if query.blank?
+        add_attributes_to(options[:attributes],cond)
+        cond.set_max   options[:limit] unless options[:count]
+        cond.set_skip  options[:offset]
+        cond.set_order translate_order_to_he(options[:order]) unless options[:order].blank?
+        return cond
+      end
+      
+      def add_attributes_to(attributes,condition)
+        case attributes
+          when String,nil then condition.add_attr attributes unless attributes.blank?
+          when Array then
+            attributes.reject(&:blank?).each do |attr|
+              condition.add_attr attr
+            end
+          when Hash then
+            attributes.each do |attribute,value|
+              next if value.blank? or attribute.blank?
+              search_type = search_type_for_attribute(attribute)
+              attribute = translate_attribute_name_to_he(attribute)
+              condition.add_attr "#{attribute} #{search_type} #{value}"
+            end
+          else raise
+        end
+      end
+
+      def search_type_for_attribute(attribute)
+        column_type_of(attribute) == :numeric ? 'NUMEQ' : 'iSTRINC'
+      end
+
+      def translate_order_to_he(order)
+        order_parts = order.to_s.downcase.strip.split(' ') 
+        return order if order_parts.size > 2
+        return order unless a_or_d(order_parts[1])
+        
+        translated = translate_attribute_name_to_he(order_parts[0])
+        sort_word = (column_type_of(order_parts[0])==:numeric) ? 'NUM' : 'STR'
+        "#{translated} #{sort_word}#{a_or_d(order_parts[1])}"
+      end
+
+      #is the column numeric(numbers/dates) or string(else) ?
+      def column_type_of(attribute)
+        column = @ar_class.columns_hash[attribute.to_s]
+        return :string unless column
+        return :numeric if column.number? or [:datetime,:time,:date,:timestamp].include?(column.type)
+        return :string
+      end
+      
+      def translate_attribute_name_to_he(name)
+        case name.to_s
+          when 'updated_at','updated_on' then "@mdate"
+          when 'created_at','created_on' then "@cdate"
+          when 'id' then "db_id"
+          else name
+        end
+      end
+
+      #pre: string is downcased & stripped
+      def a_or_d(order_end)
+        case order_end
+          when 'asc' then "A"
+          when 'desc','',nil then "D"
+          else nil
+        end
+      end
+
+      def delete_from_index(document)
+        @connection.out_doc(document.attr('@id'))
+      end
+
+      def benchmark(log, &block)
+        @ar_class.benchmark(log, &block)
+      end
+
+      def calculate_node_name(config)
+        node_prefix = config['node_prefix'] || config['node'] || RAILS_ENV
+        "#{node_prefix}_#{@ar_class.table_name}"
+      end
+
+      def attribute_name(attribute)
+        SYSTEM_ATTRIBUTES.include?(attribute.to_s) ? "@#{attribute}" : "#{attribute}"
+      end
+    end
+  end
+end
+
+# call after creating namespace SearchDo::Backends::HyperEstraier
+require 'search_do/backends/hyper_estraier/estraier_pure_extention'
+

data/lib/search_do/backends.rb

@@ -0,0 +1,17 @@
+require 'search_do/backends/hyper_estraier'
+
+module SearchDo
+  module Backends
+    def connect(model_klass, config)
+      backend = config['backend'] || "hyper_estraier"
+
+      case backend
+      when "hyper_estraier", nil # default
+        Backends::HyperEstraier.new(model_klass, config)
+      else
+        raise NotImplementedError.new("#{backend} backend is not supported")
+      end
+    end
+    module_function :connect
+  end
+end

data/lib/search_do/dirty_tracking/bridge.rb

@@ -0,0 +1,22 @@
+module SearchDo
+  module DirtyTracking
+    module Bridge
+
+      def need_update_index?(attr_name = nil)
+        return false unless changed?
+        cs = changed_attributes.keys
+        if attr_name
+          cs.include?(attr_name)
+        else
+          search_indexer.observing_fields.any?{|t| cs.include?(t) }
+        end
+      end
+
+      private
+      def clear_changed_attributes
+        changed_attributes.clear
+      end
+    end
+  end
+end
+

data/lib/search_do/dirty_tracking/self_made.rb

@@ -0,0 +1,36 @@
+
+module SearchDo
+  module DirtyTracking
+    module SelfMade
+      def self.included(base)
+        base.class_eval do
+          attr_accessor :changed_attributes
+
+          search_indexer.observing_fields.each do |attr_name|
+            define_method("#{attr_name}=") do |value|
+              write_changed_attribute attr_name, value
+            end
+          end
+        end
+      end
+
+      # If called with no parameters, gets whether the current model has changed and needs to updated in the index.
+      # If called with a single parameter, gets whether the parameter has changed.
+      def need_update_index?(attr_name = nil)
+        changed_attributes and (attr_name.nil? ?
+          (not changed_attributes.length.zero?) : (changed_attributes.include?(attr_name.to_s)) )
+      end
+
+      private
+      def clear_changed_attributes #:nodoc:
+        self.changed_attributes = []
+      end
+
+      def write_changed_attribute(attr_name, attr_value) #:nodoc:
+        (self.changed_attributes ||= []) << attr_name.to_s unless self.need_update_index?(attr_name) or self.send(attr_name) == attr_value
+        write_attribute(attr_name.to_s, attr_value)
+      end
+    end
+  end
+end
+

data/lib/search_do/dirty_tracking.rb

@@ -0,0 +1,15 @@
+require 'search_do/dirty_tracking/self_made'
+require 'search_do/dirty_tracking/bridge'
+
+module SearchDo
+  module DirtyTracking
+    def self.included(base)
+      mod = if defined?(ActiveRecord::Dirty) && base.included_modules.include?(ActiveRecord::Dirty)
+              DirtyTracking::Bridge
+            else
+              DirtyTracking::SelfMade
+            end
+      base.send(:include, mod)
+    end
+  end
+end

data/lib/search_do/indexer.rb

@@ -0,0 +1,65 @@
+require 'set'
+
+module SearchDo
+  class Indexer
+    attr_reader :searchable_fields, :if_changed, :attributes_to_store
+    def initialize(base, configuration)
+      @base = base
+      @configuration = configuration
+    end
+
+    def searchable_fields=(fields)
+      expire_observing_fields_cache!
+      @searchable_fields = fields
+    end
+
+    def if_changed=(fields)
+      expire_observing_fields_cache!
+      @if_changed = fields
+    end
+
+    def attributes_to_store=(attrs)
+      expire_observing_fields_cache!
+      @attributes_to_store = attrs.stringify_keys
+    end
+
+    def observing_fields(update = false)
+      expire_observing_fields_cache! if update
+      @observing_fields ||=
+        Set.new((if_changed + searchable_fields + attributes_to_store.values).map(&:to_s))
+    end
+
+    def record_timestamps!
+      begin
+        detect_col = lambda{|candidate_col| @base.column_names.include?(candidate_col) }
+        @attributes_to_store[backend_vocabulary :create_timestamp] =
+          %w(created_at created_on).detect(&detect_col)
+        @attributes_to_store[backend_vocabulary :update_timestamp] =
+          %w(updated_at updated_on).detect(&detect_col)
+        expire_observing_fields_cache!
+      rescue
+        #allow db-operations like schema loading etc to work without this crashing
+        puts "Your database is non-existent or in a very bad state -- From:#{__FILE__}:#{__LINE__}"
+      end
+    end
+
+    def add_callbacks!
+      @base.after_update(:update_index)
+      @base.after_create(:add_to_index)
+      @base.after_destroy(:remove_from_index)
+      @base.after_save(:clear_changed_attributes)
+    end
+
+    private
+    def expire_observing_fields_cache!
+      @observing_fields = nil
+    end
+    
+    # TODO
+    def backend_vocabulary(type)
+      { :update_timestamp => 'mdate',
+        :create_timestamp => 'cdate',
+      }[type]
+    end
+  end
+end

data/lib/search_do/utils.rb

@@ -0,0 +1,11 @@
+
+module SearchDo
+  class Utils
+    MULTIBYTE_SPACE = [0x3000].pack("U")
+
+    def self.cleanup_query(query)
+      query.to_s.gsub(MULTIBYTE_SPACE,' ')
+    end
+  end
+end
+