search_do 0.2.0

data/lib/vendor/overview

@@ -0,0 +1,100 @@
+= Pure Ruby Interface of Hyper Estraier
+
+Hyper Estraier is a full-text search system for communities.
+
+== Introduction
+
+This is a package implementing the node API of {Hyper Estraier}[http://hyperestraier.sourceforge.net/].  This is a pure ruby package.  So, it works on Linux, Mac OS X, Windows, and so on.  It does not depend on the core library of Hyper Estraier.  Applications are implemented as clients of node servers running on local or remote machines.
+
+Though Hyper Estraier itself is released under the terms of the GNU LGPL, this package is released under the terms of a BSD-style license.
+
+== Setting
+
+Get the package of the latest version of Hyper Estraier.
+
+Extract the package and enter the sub directory `rubypure' and perform installation.
+
+ cd rubypure
+ ./configure
+ make
+ su
+ make install
+
+The package `estraierpure' should be required in each source file of application programs and include the module `EstraierPure' at pleasure.
+
+== Example of Gatherer
+
+The following is the simplest implementation of a gatherer.
+
+ require "estraierpure"
+ include EstraierPure
+
+ # create and configure the node connecton object
+ node = Node::new
+ node.set_url("http://localhost:1978/node/test1")
+ node.set_auth("admin", "admin")
+
+ # create a document object
+ doc = Document::new
+
+ # add attributes to the document object
+ doc.add_attr("@uri", "http://estraier.gov/example.txt")
+ doc.add_attr("@title", "Over the Rainbow")
+
+ # add the body text to the document object
+ doc.add_text("Somewhere over the rainbow.  Way up high.")
+ doc.add_text("There's a land that I heard of once in a lullaby.")
+
+ # register the document object to the node
+ unless node.put_doc(doc)
+   STDERR.printf("error: %d\n", node.status)
+ end
+
+==Example of Searcher
+
+The following is the simplest implementation of a searcher.
+
+ require "estraierpure"
+ include EstraierPure
+ 
+ # create and configure the node connecton object
+ node = Node::new
+ node.set_url("http://localhost:1978/node/test1")
+
+ # create a search condition object
+ cond = Condition::new
+
+ # set the search phrase to the search condition object
+ cond.set_phrase("rainbow AND lullaby")
+
+ # get the result of search
+ nres = node.search(cond, 0);
+ if nres
+   # for each document in the result
+   for i in 0...nres.doc_num
+     # get a result document object
+     rdoc = nres.get_doc(i)
+     # display attributes
+     value = rdoc.attr("@uri")
+     printf("URI: %s\n", value) if value
+     value = rdoc.attr("@title")
+     printf("Title: %s\n", value) if value
+     # display the snippet text */
+     printf("%s", rdoc.snippet)
+   end
+ else
+   STDERR.printf("error: %d\n", node.status)
+ end
+
+== License
+
+ Copyright (C) 2004-2006 Mikio Hirabayashi
+ All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+- Neither the name of Mikio Hirabayashi nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/recipes/mode_maintenance.rb

@@ -0,0 +1,52 @@
+namespace :search do
+
+  desc "Reindexes all model attributes"
+  task :reindex do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:reindex MODEL=#{ENV['MODEL']}"
+  end
+
+  namespace :node do
+
+    desc "Create HE node"
+    task :create do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:node:create MODEL=#{ENV['MODEL']}"
+    end
+
+    desc "Delete HE node"
+    task :delete do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:node:delete MODEL=#{ENV['MODEL']}"
+    end
+  end
+end
+namespace :search do
+
+  desc "Reindexes all model attributes"
+  task :reindex do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:reindex MODEL=#{ENV['MODEL']}"
+  end
+
+  namespace :node do
+
+    desc "Create HE node"
+    task :create do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:node:create MODEL=#{ENV['MODEL']}"
+    end
+
+    desc "Delete HE node"
+    task :delete do
+      rake = fetch(:rake, "rake")
+      rails_env = fetch(:rails_env, "production")
+      run "cd #{current_path}; #{rake} RAILS_ENV=#{rails_env} search:node:delete MODEL=#{ENV['MODEL']}"
+    end
+  end
+end

data/spec/backends/hyper_estraier_spec.rb

@@ -0,0 +1,220 @@
+require File.expand_path("../spec_helper", File.dirname(__FILE__))
+
+describe SearchDo::Backends::HyperEstraier do
+  before do
+    @backend = SearchDo::Backends::HyperEstraier.new(Story, ActiveRecord::Base.configurations["test"]["estraier"])
+  end
+
+  describe "without index" do
+    before do
+      @backend.connection.should_receive(:search).and_return(nil)
+    end
+
+    it "should be empty" do
+      @backend.index.should == []
+    end
+  end
+
+  describe "include nil to indexed text" do
+    it "should not raise error" do
+      lambda{ 
+        @backend.add_to_index(['foo', nil], {})
+      }.should_not raise_error
+    end
+  end
+
+  describe "date searching" do
+    before do
+      @backend.clear_index!
+      @time = Time.local(2008,9,17)
+      @backend.add_to_index([@time], 'db_id' => "1", '@uri' => "/Story/1")
+    end
+
+    it "should searchable with time in iso format" do
+      @backend.count(@time.iso8601).should > 0
+    end
+
+    it "is ordered through dates" do
+      @backend.add_to_index([], 'db_id' => "1", 'read_at'=>Time.now, '@uri' => "/Story/1")
+      @backend.add_to_index([], 'db_id' => "2", 'read_at'=>Time.now+1.day, '@uri' => "/Story/2")
+      @backend.add_to_index([], 'db_id' => "3", 'read_at'=>Time.now-1.day, '@uri' => "/Story/3")
+      @backend.search_all_ids('',:order=>"read_at ASC").should == [3,1,2]
+    end
+  end
+  
+  describe "raw" do
+    before do
+      @backend.add_to_index([], 'db_id' => "1", '@uri' => "/Story/1")
+    end
+    
+    it "finds a raw document" do
+      @backend.raw(1).should_not be_nil
+      @backend.raw(1).attr('db_id').should == '1'
+    end
+  end
+
+  describe :build_fulltext_condition do
+    it "does not use limit for counting" do
+      @backend.send(:build_fulltext_condition,'',:count=>true).max.should == -1
+    end
+    
+    describe 'parsing attributes' do
+      def condition(options)
+        @backend.send(:build_fulltext_condition,'something',options)
+      end
+      
+      it 'raises on unknown' do
+        lambda{condition(:attributes=>1)}.should raise_error(RuntimeError)
+      end
+      
+      it "removes blanks" do
+        condition(:attributes=>['','  ','a']).attrs.should == ['a']
+      end
+      
+      it 'ignores empty' do
+        condition(:attributes=>nil).attrs.should == []
+        condition(:attributes=>'').attrs.should == []
+      end
+      
+      it "adds a always-true condition when search is blank" do
+        @backend.send(:build_fulltext_condition,'  ').attrs.should_not be_blank
+      end
+      
+      it "parses a string" do
+        condition(:attributes=>'x y z').attrs.should == ['x y z']
+      end
+      
+      it "parses an array" do
+        condition(:attributes=>['a b c','d e f']).attrs.should == ['a b c','d e f']
+      end
+      
+      
+      describe 'parsing a hash' do
+        before :all do
+          Story.columns_hash['popularity'].should be_number
+          Story.columns_hash['title'].should be_text
+        end
+
+        it "parses a simple hash" do
+          condition(:attributes=>{:a=>'b'}).attrs.should == ['a iSTRINC b']
+        end
+        
+        it "parses a blank hash" do
+          condition(:attributes=>{:a=>''}).attrs.should == []
+        end
+        
+        it "parses a keyless hash" do
+          condition(:attributes=>{''=>'wtf'}).attrs.should == []
+        end
+        
+        it "parses a number column to number search" do
+          condition(:attributes=>{'popularity'=>12}).attrs.should == ['popularity NUMEQ 12']
+        end
+        
+        it "parses a string column to string search" do
+          condition(:attributes=>{'title'=>12}).attrs.should == ['title iSTRINC 12']
+        end
+        
+        it "translates columns" do
+          condition(:attributes=>{'id'=>1}).attrs.should == ['db_id NUMEQ 1']
+        end
+        
+        it "parses a unknown column to string search" do
+          condition(:attributes=>{'xxx'=>'x'}).attrs.should == ['xxx iSTRINC x']
+        end
+        
+        it "parses a date or time" do
+          pending do
+            #raw output: "@mdate" "2008-09-21T09:51:27+02:00"
+            raise
+          end
+        end
+      end
+    end
+    
+    describe "translate order" do
+      def translated_order(order)
+        @backend.send(:build_fulltext_condition,'',:order=>order).order
+      end
+
+      it "translates nil to nil" do
+        translated_order(nil).should == nil
+      end
+
+      it "translates blank to nil" do
+        translated_order("").should == nil
+      end
+
+      it "translates strings to STRA" do
+        translated_order('title ASC').should == 'title STRA'
+        translated_order('title asc').should == 'title STRA'
+      end
+
+      it "translates strings to STRD" do
+        translated_order('title').should == 'title STRD'
+        translated_order('title DESC').should == 'title STRD'
+        translated_order('title desc').should == 'title STRD'
+      end
+
+      it "translates dates to NUMD" do
+        translated_order(:written_on).should == "written_on NUMD"
+      end
+      
+      it "translates datetimes to NUM" do
+        translated_order("read_at ASC").should == "read_at NUMA"
+      end
+      
+      it "translates numbers to NUMD" do
+        translated_order(:popularity).should == "popularity NUMD"
+      end
+      
+      it "translates numbers with asc to NUMA" do
+        translated_order("popularity ASC").should == "popularity NUMA"
+      end
+      
+      it "translate non-columns to string" do
+        translated_order("paid_at ASC").should == "paid_at STRA"
+      end
+      
+      describe "translating rails-terms" do
+        before :all do
+          class FakeColumn
+            def number?;false;end
+            def type;:date;end
+          end
+          Story.columns_hash["created_on"] = FakeColumn.new
+          Story.columns_hash["updated_on"] = FakeColumn.new
+        end
+
+        after :all do
+          Story.reset_column_information
+        end
+        
+        #symbols and desc <-> DESC only need testing once, to see if order values get normalized
+        ['updated_at','updated_on',:updated_at,'updated_at DESC','updated_at desc'].each do |order|
+          it "translates #{order}" do
+            translated_order(order).should == "@mdate NUMD"
+          end
+        end
+        ['created_at','created_on','created_at DESC'].each do |order|
+          it "translates #{order}" do
+            translated_order(order).should == "@cdate NUMD"
+          end
+        end
+        ['id','id DESC'].each do |order|
+          it "translates #{order}" do
+            translated_order(order).should == "db_id NUMD"
+          end
+        end
+        
+        it "does not translate strange things" do
+          translated_order('id strange').should == 'id strange'
+        end
+        
+        it "does not translate long orders" do
+          translated_order('id desc wtf').should == 'id desc wtf'
+        end
+      end
+    end
+  end
+end

data/spec/backends/result_document_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path("../spec_helper", File.dirname(__FILE__))
+require 'search_do/backends/hyper_estraier/estraier_pure_extention'
+
+
+describe EstraierPure::ResultDocument do
+  describe :snippet_a do
+    before do
+      #result of a search for "i am so blue"
+      snip = "...i\ti\n \nam\tam\n \nso\tso\n \nblue\tblue\n test\ni\ti\nng makes me happy\n\n"
+      @res = EstraierPure::ResultDocument.new('',{},snip,'')
+    end
+
+    it "returns a snippet_a" do
+      @res.snippet_a.should_not be_nil
+    end
+
+    it "transform the snippet to a array of lines" do
+      @res.snippet_a.size.should == 10
+    end
+
+    it "highlights the found words" do
+      #                                      i    ' '   am   ' '   so   ' '   blue   ' test'  i    ng makes...
+      @res.snippet_a.map{|x|x[1]}.should == [true,false,true,false,true,false,true,  false,   true,false]
+    end
+  end
+end

data/spec/dirty_tracking/bridge_spec.rb

@@ -0,0 +1,33 @@
+require File.expand_path("../spec_helper", File.dirname(__FILE__))
+
+describe :need_update_index? do
+  def needs
+    @story.send(:need_update_index?)
+  end
+  
+  before do
+    @story = Story.new
+  end
+  
+  it "does not need update when object is new" do
+    needs.should be_false
+  end
+  
+  it "needs update when column changes" do
+    @story.title = "new value"
+    needs.should be_true
+  end
+  
+  it "needs update when a indexed non-column changes" do
+    pending do
+      @story.non_column = "new value"
+      needs.should be_true
+    end
+  end
+  
+  it "does not need update after changes have been cleared" do
+    @story.title = "new value"
+    @story.send(:clear_changed_attributes)
+    needs.should be_false
+  end
+end

data/spec/estraier_admin_spec.rb

@@ -0,0 +1,26 @@
+require File.expand_path("../lib/estraier_admin", File.dirname(__FILE__))
+
+describe EstraierAdmin do 
+  before do
+    @admin = EstraierAdmin.new(:user=>"hoge",:password=>"foo")
+  end
+
+  it "@config[:user] should == 'hoge'" do
+    @admin.instance_eval{@config}[:user].should == 'hoge'
+  end
+
+  it "@config[:password] should == 'foo'" do
+    @admin.instance_eval{@config}[:password].should == 'foo'
+  end
+
+  it "should receive :request_or_raise, with {:name=>'piyo', :label=>'piyo', :action=>8} when create_node()" do
+    @admin.should_receive(:request_or_raise).with(:name=>'piyo', :label=>'piyo', :action=>8)
+    @admin.create_node('piyo')
+  end
+
+  it "should receive :request_or_raise, with {:name=>'piyo', :action=>9, :sure=>1} when delete_node()" do
+    @admin.should_receive(:request_or_raise).with(:name=>'piyo', :action=>9, :sure=>1)
+    @admin.delete_node('piyo')
+  end
+end
+

data/spec/fixtures/stories.yml

@@ -0,0 +1,27 @@
+sanshiro:
+  id: 101
+  title: "三四郎"
+  body: "
+うとうととして目がさめると女はいつのまにか、隣のじいさんと話を始めている。
+このじいさんはたしかに前の前の駅から乗ったいなか者である。発車まぎわに頓狂
+（とんきょう）な声を出して駆け込んで来て、いきなり肌（はだ）をぬいだと
+思ったら背中にお灸（きゅう）のあとがいっぱいあったので、三四郎（さんしろう）
+の記憶に残っている。じいさんが汗をふいて、肌を入れて、女の隣に腰をかけたまで
+よく注意して見ていたくらいである。"
+  popularity: 10
+  updated_at: 2008-04-05 12:34:56
+  created_at: 2008-04-05 12:34:56
+
+neko:
+  id: 102
+  title: "吾輩は猫である"
+  body: "
+吾輩（わがはい）は猫である。名前はまだ無い。
+どこで生れたかとんと見当（けんとう）がつかぬ。何でも薄暗いじめじめした所でニャ
+ーニャー泣いていた事だけは記憶している。吾輩はここで始めて人間というものを見た
+。しかもあとで聞くとそれは書生という人間中で一番獰悪（どうあく）な種族であった
+そうだ。"
+  popularity: 15
+  updated_at: 2008-04-01 12:34:56
+  created_at: 2008-04-01 12:34:56
+

data/spec/indexer_spec.rb

@@ -0,0 +1,59 @@
+require File.expand_path("spec_helper", File.dirname(__FILE__))
+require 'search_do/indexer'
+
+describe SearchDo::Indexer,"new(AR[created_at updated_at]) / and set some attrs" do
+  before do
+    @ar_klass = mock("ar_klass")
+    @ar_klass.stub!(:column_names).and_return(%[created_at updated_at])
+
+    @indexer = SearchDo::Indexer.new(@ar_klass, {})
+    @indexer.searchable_fields = %w[title body]
+    @indexer.attributes_to_store = {:user_id => :user_id}
+    @indexer.if_changed = %w[popularity]
+  end
+
+  it "#searchable_fields should == %w[title body]" do
+    @indexer.searchable_fields.should == %w[title body]
+  end
+
+  it "#attributes_to_store['user_id'].should == :user_id" do
+    @indexer.attributes_to_store['user_id'].should == :user_id
+  end
+
+  it "#observing_fields.should == Set.new(%w[title body user_id popularity])" do
+    @indexer.observing_fields.should == Set.new(%w[title body user_id popularity])
+  end
+
+  describe "call '.record_timestamps!'" do
+    before do
+      @indexer.record_timestamps!
+    end
+
+    it "#attributes_to_store['cdate'] should == 'created_at'" do
+      @indexer.attributes_to_store['cdate'].should == 'created_at'
+    end
+
+    it "#attributes_to_store['mdate'] should == 'updated_at'" do
+      @indexer.attributes_to_store['mdate'].should == 'updated_at'
+    end
+
+    it "#observing_fields.should include 'updated_at' and 'created_at'" do
+      @indexer.observing_fields.should include('created_at')
+      @indexer.observing_fields.should include('updated_at')
+    end
+  end
+
+  describe "#add_callbacks!() " do
+    before do
+      @ar_klass.should_receive(:after_update).with(:update_index)
+      @ar_klass.should_receive(:after_create).with(:add_to_index)
+      @ar_klass.should_receive(:after_save).with(:clear_changed_attributes)
+      @ar_klass.should_receive(:after_destroy).with(:remove_from_index)
+    end
+
+    it "adds appropriate callbacks" do
+      @indexer.add_callbacks!
+    end
+  end
+end
+