search_do 0.2.0

data/spec/search_do_spec.rb

@@ -0,0 +1,335 @@
+require File.expand_path("spec_helper", File.dirname(__FILE__))
+
+require 'digest/sha1'
+
+describe Story, "extended by acts_as_searchable_enhance" do
+  it "should respond_to :fulltext_search" do
+    Story.should respond_to(:fulltext_search)
+  end
+
+  it "should have callbak :update_index" do
+    Story.after_update.should include(:update_index)
+  end
+
+  it "should have callbak :add_to_index" do
+    Story.after_create.should include(:add_to_index)
+  end
+
+  describe "separate node by model classname" do
+    before(:all) do
+      OtherKlass = Class.new(ActiveRecord::Base)
+      OtherKlass.class_eval do
+        acts_as_searchable :ignore_timestamp => true
+      end
+    end
+
+    it "OtherKlass.search_backend.node_name.should == 'aas_e_test_other_klasses'" do
+      OtherKlass.search_backend.node_name.should == 'aas_e_test_other_klasses'
+    end
+
+    it "search_backend.node_name should == 'aas_e_test_stories'" do
+      Story.search_backend.node_name.should == 'aas_e_test_stories'
+    end
+  end
+
+  describe "matched_ids" do
+    fixtures :stories
+    before do
+      @story_ids = Story.find(:all, :limit=>2).map(&:id)
+
+      @mock_results = @story_ids.map{|id| mock("ResultDocument_#{id}", :attr => id) }
+      nres = EstraierPure::NodeResult.new(@mock_results, {})
+      Story.search_backend.connection.stub!(:search).and_return(nres)
+    end
+
+    it "finds all story ids" do
+      Story.matched_ids("hoge").should == @story_ids
+    end
+
+    it "calls EstraierPure::Node#search()" do
+      Story.search_backend.connection.should_receive(:search)
+      Story.matched_ids("hoge")
+    end
+  end
+
+  describe "remove from index" do
+    fixtures :stories
+    before do
+      stories = Story.find(:all, :limit=>2)
+      @story = stories.first
+      mock_results = stories.map{|s| mock("ResultDocument_#{s.id}", :attr => s.id) }
+
+      nres = EstraierPure::NodeResult.new(mock_results, {})
+      Story.search_backend.connection.stub!(:search).and_return(nres)
+    end
+
+    it "should call EstraierPure::Node#delete_from_index" do
+      Story.search_backend.connection.should_receive(:out_doc)
+      @story.remove_from_index
+    end
+  end
+  
+  describe "add_snippets" do
+    before :each do
+      @stories = Story.find :all
+      count = 0
+      @ids_and_raw = @stories.map {|story| [story.id,fake_raw(count+=1)]}
+    end
+    
+    it "adds snippets to objects" do
+      Story.send(:add_snippets,@stories,@ids_and_raw)
+      @stories.map(&:snippet).should == ['snip1','snip2']
+    end
+    
+    it "adds snippets to right objects" do
+      Story.send(:add_snippets,@stories.reverse,@ids_and_raw)
+      @stories.map(&:snippet).should == ['snip1','snip2']
+    end
+  end
+
+  describe :snippet_to_html do
+    it "surrounds snippets with bold" do
+      Story.send(:snippet_to_html,[['x',true],['y',false]]).should == "<b>x</b>y"
+    end
+
+    it "removes tags html" do#since they would get broken up
+      Story.send(:snippet_to_html,[['ab</b>ab',true],['y',false]]).should == "<b>abab</b>y"
+      Story.send(:snippet_to_html,[['ab<br>ab',true],['y',false]]).should == "<b>abab</b>y"
+    end
+
+    it "strips tags" do
+      Story.send(:snippet_to_html,[['ab<a>x</a>ab',true],['y',false]]).should == "<b>abxab</b>y"
+    end
+  end
+
+  describe "fulltext_search" do
+    #matched_ids_and_raw => [:id,raw] and find_option=>{:condition => 'id = :id'}
+    
+    fixtures :stories
+    before do
+      stories = Story.find(:all)
+      @story = stories.first
+      fake_results = stories.map{|story| [story.id,fake_raw]}
+      Story.stub!(:matched_ids_and_raw).and_return fake_results
+    end
+
+    def fulltext_search(query='hoge')
+      finder_opt = {:conditions => ["id = ?", @story.id]}
+      Story.fulltext_search(query, :find => finder_opt)
+    end
+
+    it "finds story" do
+      fulltext_search.should == [@story]
+    end
+
+    it "adds snippets" do
+      fulltext_search[0].snippet.should == 'snip'
+    end
+    
+    it "does not add snippets if query is empty" do
+      fulltext_search('')[0].snippet.should be_nil
+    end
+    
+    it "does not add snippet when object does not respond to snippet=" do
+      @story.stub!(:respond_to?).with(:html_snippet=)
+      @story.should_receive(:respond_to?).with(:snippet=).and_return false
+      Story.should_receive(:find).and_return [@story]
+      fulltext_search[0].snippet.should be_blank
+    end
+    
+    it "calls matched_ids_and_raw" do
+      Story.should_receive(:matched_ids_and_raw).and_return([[@story.id,fake_raw]])
+      fulltext_search
+    end
+  end
+  
+  describe "paginate_by_fulltext_search" do
+    before do
+      Story.stub!(:fulltext_search).and_return [Story.first]
+      Story.stub!(:count_fulltext).and_return 1
+    end
+    
+    it "has total_entries" do
+      Story.paginate_by_fulltext_search('',:page=>1,:per_page=>1).total_entries.should == 1
+    end
+    
+    it "translates paginate terms to limit and offset and removes page/per_page" do
+      Story.should_receive(:fulltext_search).with('x',{:limit=>3,:offset=>6}).and_return []
+      Story.paginate_by_fulltext_search('x',:page=>3,:per_page=>3)
+    end
+    
+    it "uses search word and attributes for count query" do
+      Story.should_receive(:count_fulltext).with('x',:attributes=>'something').and_return 1
+      Story.paginate_by_fulltext_search('x',:page=>1,:per_page=>1,:attributes=>'something')
+    end
+    
+    it "calculates total_entries from search results" do
+      Story.should_receive(:count_fulltext).never
+      Story.paginate_by_fulltext_search('',:page=>1,:per_page=>2).total_entries.should == 1
+    end
+  end
+  
+  describe "new interface Model.find_fulltext(query, options={})" do
+    fixtures :stories
+    
+    before do
+      Story.stub!(:matched_ids).and_return([102, 101, 110])
+    end
+
+    it "find_fulltext('hoge', :order=>'updated_at DESC') should == [stories(:sanshiro), stories(:neko)]" do
+      Story.find_fulltext('hoge', :order=>"updated_at DESC").should == Story.find([102,101]).reverse
+    end
+  end
+
+  describe "search using real HyperEstraier (End-to-End test)" do
+    fixtures :stories
+    before(:all) do
+      Story.clear_index!
+      #Story.delete_all
+      Story.create!(:title=>"むかしむかし", :body=>"あるところにおじいさんとおばあさんが")
+      Story.create!(:title=>"i am so blue", :body=>"testing makes me happy",:non_column=>'non column value')
+      Story.reindex!
+      # waiting Estraier sync index, adjust 'cachernum' in ${estraier}/_conf if need
+      sleep 1
+    end
+
+    before do
+      @story = Story.find_by_title("むかしむかし")
+    end
+
+    it "finds a indexed object" do
+      Story.fulltext_search('むかしむかし').should == [@story]
+    end
+
+    it "counts correctly using count_fulltext" do
+      Story.count_fulltext('むかしむかし').should == 1
+    end
+    
+    it "finds all object when searching for ''" do
+      Story.fulltext_search('').size.should == Story.count
+    end
+    
+    it "finds using long strings" do
+      Story.fulltext_search("i am so blue").size.should == 1
+    end
+    
+    it "finds using attributes" do
+      #FIXME this fails if iSTRAND is used, why?
+      Story.fulltext_search('',:attributes=>{:body=>'るとこ'}).size.should == 1
+    end
+
+    it "finds using long strings in attributes" do
+      Story.fulltext_search('',:attributes=>{:body=>'testing makes'}).size.should == 1
+    end
+
+    it "finds using long strings using and in attributes" do
+      pending do
+        #FIXME this works if iSTRAND is used, but then "finds using attributes" fails
+        Story.fulltext_search('',:attributes=>{:body=>'testing makes happy'}).size.should == 1
+      end
+    end
+
+    it "finds using upper or lowercase attributes" do
+      Story.fulltext_search('',:attributes=>{:body=>'MakeS'}).size.should == 1
+    end
+
+    it "finds using non_column attributes" do
+      pending do
+        Story.fulltext_search('',:attributes=>{:non_column=>'non column value'}).size.should == 1
+      end
+    end
+
+    # asserts HE raw_match order
+    it "finds in correct order(descending)" do
+      Story.matched_ids('記憶', :order => "@mdate NUMD").should == [101, 102]
+    end
+
+    it "finds in correct order(ascending)" do
+      Story.matched_ids('記憶', :order => "@mdate NUMA").should == [102, 101]
+    end
+    
+    it "preserves order of found objects" do
+      Story.fulltext_search('記憶', :order => "@mdate NUMA").map(&:id).should == [102, 101]
+    end
+    
+    it "preservers order if scope is given" do
+      pending
+    end
+    
+    it "has all objects in index" do
+      Story.search_backend.index.size.should == Story.count
+    end
+  end
+
+  describe "partial updating" do
+    fixtures :stories
+    before do
+      @story = Story.find(:first)
+      @story.stub!(:record_timestamps).and_return(false)
+    end
+
+    it "updates when changing indexed column" do
+      Story.search_backend.should_receive(:add_to_index).once
+      @story.title = "new title"
+      @story.save
+    end
+
+    it "does not updates when changing unindexed column" do
+      Story.search_backend.should_not_receive(:add_to_index)
+      @story.popularity += 10
+      @story.save
+    end
+  end
+  
+  def fake_raw(num=nil)
+    mock("Raw",:snippet=>"snip#{num}")
+  end
+end
+
+describe "StoryWithoutAutoUpdate" do
+  before(:all) do
+    class StoryWithoutAutoUpdate < ActiveRecord::Base
+      set_table_name :stories
+      acts_as_searchable :searchable_fields=>[:title, :body], :auto_update=>false
+    end
+  end
+
+  it "should not have callback :update_index" do
+    StoryWithoutAutoUpdate.after_update.should_not include(:update_index)
+  end
+
+  it "should not have callback :add_to_index" do
+    StoryWithoutAutoUpdate.after_create.should_not include(:add_to_index)
+  end
+  
+  it "should not call add_to_index" do
+    story = StoryWithoutAutoUpdate.new
+    StoryWithoutAutoUpdate.search_backend.should_not_receive(:add_to_index)
+    story.popularity = 20
+    story.save
+  end
+end
+
+describe SearchDo::Utils do
+  describe "tokenize_query" do
+    it "converts nil to empty string" do
+      SearchDo::Utils.cleanup_query(nil).should == ''
+    end
+    
+    it "does not convert empty strings to nil" do
+      SearchDo::Utils.cleanup_query('').should == ''
+    end
+
+    it "does not alter search terms'" do
+      SearchDo::Utils.cleanup_query('ruby vim').should == 'ruby vim'
+    end
+
+    it "leaves in quotes" do
+      SearchDo::Utils.cleanup_query('"ruby on rails" vim').should == '"ruby on rails" vim'
+    end
+
+    it "converts long unicode spaces" do
+      SearchDo::Utils.cleanup_query('rails　vim').should == 'rails vim'
+    end
+  end
+end

data/spec/setup_test_model.rb

@@ -0,0 +1,38 @@
+require 'rubygems'
+require 'active_record'
+require 'active_record/fixtures'
+
+#activate will_paginate for models
+require 'will_paginate'
+WillPaginate.enable_activerecord
+
+#create model table
+ActiveRecord::Schema.define(:version => 1) do
+  create_table "stories" do |t|
+    t.date     "written_on"
+    t.datetime "read_at"
+    t.string   "title"
+    t.text     "body"
+    t.integer  "popularity", :default =>0
+    t.timestamps
+  end
+end
+
+#create model
+class Story < ActiveRecord::Base
+  attr_accessor :snippet
+  acts_as_searchable :searchable_fields=>[:title, :body],:attributes=>{:non_column=>nil,:body=>nil}
+  
+  def non_column
+    @non_column
+  end
+  
+  def non_column=(val)
+    @non_column=val
+  end
+end
+
+#create node
+require File.expand_path("../lib/estraier_admin", File.dirname(__FILE__))
+admin = EstraierAdmin.new(ActiveRecord::Base.configurations["test"][:estraier])
+admin.create_node(Story.search_backend.node_name)

data/spec/spec_helper.rb

@@ -0,0 +1,52 @@
+# ---- requirements
+$KCODE = 'u' #activate regex unicode
+require 'rubygems'
+require 'spec'
+$LOAD_PATH << File.expand_path("../lib", File.dirname(__FILE__))
+
+
+# ---- bugfix
+#`exit?': undefined method `run?' for Test::Unit:Module (NoMethodError)
+#can be solved with require test/unit but this will result in extra test-output
+unless defined? Test::Unit
+  module Test
+    module Unit
+      def self.run?
+        true
+      end
+    end
+  end
+end
+
+
+# ---- load active record
+#gem 'activerecord', '2.0.2'
+if ENV["AR"]
+  gem 'activerecord', ENV["AR"]
+  $stderr.puts("Using ActiveRecord #{ENV["AR"]}")
+end
+require 'active_record'
+
+require File.expand_path("../init", File.dirname(__FILE__))
+
+RAILS_ENV = "test"
+ActiveRecord::Base.configurations = {"test" => {
+  :adapter => "sqlite3",
+  :database => ":memory:",
+  :estraier => {:host=> "localhost", :node=>"aas_e_test", :user=>"admin", :password=>"admin"}
+}.with_indifferent_access}
+
+ActiveRecord::Base.logger = Logger.new(File.directory?("log") ? "log/#{RAILS_ENV}.log" : "/dev/null")
+ActiveRecord::Base.establish_connection(:test)
+
+load File.expand_path("setup_test_model.rb", File.dirname(__FILE__))
+
+
+# ---- fixtures
+Spec::Example::ExampleGroupMethods.module_eval do
+  def fixtures(*tables)
+    dir = File.expand_path("fixtures", File.dirname(__FILE__))
+    tables.each{|table| Fixtures.create_fixtures(dir, table.to_s) }
+  end
+end
+

data/tasks/acts_as_searchable_tasks.rake

@@ -0,0 +1,70 @@
+require File.expand_path("../lib/estraier_admin", File.dirname(__FILE__))
+
+namespace :search do
+  desc "Clears HE Index"
+  task :clear => :environment do
+    raise "Pass a searchable model with MODEL=" unless ENV['MODEL']
+    ENV['MODEL'].constantize.clear_index!
+  end
+
+  desc "Reindexes all model attributes"
+  task :reindex => :environment do
+    raise "Pass a searchable model with MODEL=" unless ENV['MODEL']
+    model_class = ENV['MODEL'].constantize
+    reindex = lambda { model_class.reindex! }
+    if ENV['INCLUDE']
+      model_class.with_scope :find => { :include => ENV['INCLUDE'].split(',').collect { |i| i.strip.to_sym } } do
+        reindex.call
+      end
+    else
+      reindex.call
+    end
+  end
+
+  namespace :node do
+    desc "Create HE node"
+    task :create => :environment do
+      raise "Pass a searchable model with MODEL=" unless ENV['MODEL']
+      model_class = ENV['MODEL'].constantize
+      admin.create_node(model_class.search_backend.node_name)
+    end
+
+    desc "Delete HE node"
+    task :delete => :environment do
+      raise "Pass a searchable model with MODEL=" unless ENV['MODEL']
+      model_class = ENV['MODEL'].constantize
+      admin.delete_node(model_class.search_backend.node_name)
+    end
+
+    def admin
+      EstraierAdmin.new(ActiveRecord::Base.configurations[RAILS_ENV]["estraier"])
+    end
+  end
+
+  namespace :server do
+    desc "initialize HyperEstraier index directory on localhost"
+    task :init do
+      system(estmaster, 'init', index_dir) unless File.directory?(index_dir)
+    end
+
+    desc "start HyperEstraier server on loaclhost"
+    task :start => [:init] do
+      $stderr.puts "starting HyperEstraier server..."
+      system(estmaster, 'start', '-bg', index_dir)
+    end
+
+    desc "stop HyperEstraier server on localhost"
+    task :stop => [:init] do
+      $stderr.puts "stopping HyperEstraier server..."
+      system(estmaster, 'stop', index_dir)
+    end
+
+    def estmaster
+      ENV['COMMAND'] || 'estmaster'
+    end
+
+    def index_dir
+      ENV['SEARCH_INDEX_DIR'] || File.expand_path("tmp/fulltext_index", Dir.pwd)
+    end
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification 
+name: search_do
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- MOROHASHI Kyosuke
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-12-24 00:00:00 +01:00
+default_executable: 
+dependencies: []
+
+description: 
+email: moronatural@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- CHANGELOG
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- TESTING
+- VERSION
+- examples/he_search.rb
+- examples/person.rb
+- init.rb
+- lib/estraier_admin.rb
+- lib/search_do.rb
+- lib/search_do/backends.rb
+- lib/search_do/backends/hyper_estraier.rb
+- lib/search_do/backends/hyper_estraier/estraier_pure_extention.rb
+- lib/search_do/dirty_tracking.rb
+- lib/search_do/dirty_tracking/bridge.rb
+- lib/search_do/dirty_tracking/self_made.rb
+- lib/search_do/indexer.rb
+- lib/search_do/utils.rb
+- lib/vendor/estraierpure.rb
+- lib/vendor/overview
+- recipes/mode_maintenance.rb
+- spec/backends/hyper_estraier_spec.rb
+- spec/backends/result_document_spec.rb
+- spec/dirty_tracking/bridge_spec.rb
+- spec/estraier_admin_spec.rb
+- spec/fixtures/stories.yml
+- spec/indexer_spec.rb
+- spec/search_do_spec.rb
+- spec/setup_test_model.rb
+- spec/spec_helper.rb
+- tasks/acts_as_searchable_tasks.rake
+has_rdoc: true
+homepage: http://github.com/grosser/search_do
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: "AR: Hyperestraier integration"
+test_files: 
+- spec/spec_helper.rb
+- spec/backends/result_document_spec.rb
+- spec/backends/hyper_estraier_spec.rb
+- spec/indexer_spec.rb
+- spec/setup_test_model.rb
+- spec/estraier_admin_spec.rb
+- spec/search_do_spec.rb
+- spec/dirty_tracking/bridge_spec.rb
+- examples/person.rb
+- examples/he_search.rb