rsi 0.4

data/tests/suite_all.rb

@@ -0,0 +1,14 @@
+#
+# Test suite for running all RSI tests
+#
+require 'test/unit'
+require 'tests/t_index'
+require 'tests/t_index_multi'
+require 'tests/t_dictionary'
+require 'tests/t_analysis'
+
+if $0 == __FILE__
+  require 'rsi/logmanager'
+  fh = File.open( "/dev/null", "a" )
+  RSI::LogManager.instance().root_fh = fh
+end

data/tests/t_analysis.rb

@@ -0,0 +1,43 @@
+#!/usr/local/bin/ruby -w
+require 'fileutils'
+require 'test/unit'
+require 'tmpdir'
+require 'rsi/analysis'
+require 'rsi/index' # for FIELD_TYPE_TEXT
+
+class AnalTest < Test::Unit::TestCase
+
+  DOC_A = "Weebles wobble but they don't fall down"
+  DOC_B = "The boot is a whale, then?"
+  
+  def setup()
+    @a = RSI::DefaultTextAnalyzer.new()    
+  end
+
+  def teardown();  end
+  
+  def test_types()
+    t = @a.get_field_types()
+    assert_equal( 1, t.size(), "Only one type" )
+    assert_equal( RSI::FIELD_TYPE_TEXT, t['text'], "Text type dictionary" )
+  end
+
+  def test_text()
+    t = @a.tokenize( DOC_A )
+    #puts t['text'].join(":")
+    assert_equal( 5, t['text'].size(), "Phrase contains 5 interesting terms" )
+    assert( t['text'].include?( "DONT" ), "DONT should be in termlist" )
+    t = @a.tokenize( DOC_B )
+    assert_equal( 3, t['text'].size(), "Phrase contains 3 non-stopwords" )
+    assert( ! t['text'].include?( "THE" ), "The is a stopword" )
+  end
+
+  def test_stopwords()
+    source = "dog dogs buggy buggies child children The an a stop"
+    t = @a.tokenize( source )
+    #puts t['text'].join(":")
+    assert_equal( 5, t['text'].size(), "porter.rb doesn't stem 'children' right" )
+    assert( !t['text'].include?( "THE" ), "The is a stopword" )
+  end
+
+end

data/tests/t_dictionary.rb

@@ -0,0 +1,76 @@
+# -*- ruby -*-
+# 
+# Tests for rsi/dictionary.rb
+#
+
+require 'test/unit'
+require 'fileutils'
+require 'tmpdir'
+require 'rsi/dictionary'
+require 'rsi/serializers'
+require 'rsi/logmanager'
+
+class DictTest < Test::Unit::TestCase
+  
+  def setup() 
+    @failed = false
+    @tmp = Dir::tmpdir()
+    fn = "d_#{$$}-#{rand(65536)}"
+    @root = File.join( @tmp, fn )
+  end
+
+  def add_failure( msg, bt )
+    super
+    @failed = true
+  end
+
+  # Delete the temp dir for the index tests
+  def teardown()
+    if @failed
+      print "Test case failed, not cleaning up #@root\n";
+    else
+      FileUtils::rm_rf( @root )
+    end
+  end
+
+  def test_create()
+    create_dictionary()
+  end
+
+  def test_reopen()
+    create_dictionary()
+    assert( FileTest.exists?( @root ), "Test dict root should exist" )
+    d = RSI::Dictionary.new( @root )
+    d.serializer = RSI::YAMLSerializer.new()
+    d.open()
+
+    id2 = d.get_termid_for( "STRAWBERRY" )
+    assert_equal( 1, id2, "Should get same termid both times")
+    el = d.get_entry_list( id2 )
+    assert_equal( 1, el.length(), "Should be one entry" )
+    assert_equal( "poo.txt", el[0].docid )
+    assert_equal( 1, el[0].freq )
+    assert_equal( 1, el[0].pos_list.length() )   
+  end    
+
+  def create_dictionary()
+    assert( !FileTest.exists?( @root ), "Test dict root should not exist" )
+    d = RSI::Dictionary.new( @root )
+    d.serializer = RSI::YAMLSerializer.new()
+    d.open()
+    assert( FileTest.directory?( @root ), "Root should exist" )
+
+    id = d.get_termid_for( "STRAWBERRY", true )
+    d.add_term_entries( "poo.txt", id, [0] )
+    d.store()
+
+    id2 = d.get_termid_for( "STRAWBERRY" )
+    assert_equal( 1, id2, "Should get one termid")
+    el = d.get_entry_list( id2 )
+    assert_equal( 1, el.length(), "Should be one entry" )
+    assert_equal( "poo.txt", el[0].docid )
+    assert_equal( 1, el[0].freq )
+    assert_equal( 1, el[0].pos_list.length() )
+  end
+
+end

data/tests/t_index.rb

@@ -0,0 +1,78 @@
+#!/usr/local/bin/ruby -w
+require 'fileutils'
+require 'test/unit'
+require 'tmpdir'
+require 'rsi'
+require 'rsi/logmanager'
+
+DOC_A = "The quick brown fox jumped over the lazy dogs"
+DOC_B = "Every dog has his day"
+DOC_C = "Bjork, bjork, were you brought by the stork, or were you" +
+  " created from bubbles and cork?"
+
+class IdxTest < Test::Unit::TestCase
+
+  # Create a temp dir for the indexes in the test
+  def setup()
+    @tmp = Dir::tmpdir()
+    @root = File.join( @tmp, "searchtest.#{$$}.#{rand(65535)}" )
+    Dir.mkdir( @root ) # will raise if root exists, which is what I want
+    @failed = false
+  end
+
+  def add_failure( msg, bt )
+    super
+    @failed = true
+  end
+
+  # Delete the temp dir for the index tests
+  def teardown()
+    if @failed
+      print "Test case failed, not cleaning up #@root\n";
+    else
+      FileUtils::rm_rf( @root )
+    end
+  end
+
+  # Create an index, assert that it contains the right docs and terms
+  def test_basic()
+    indexer = RSI::Indexer.new( @root )
+    indexer.serializer = RSI::YAMLSerializer.new()
+    indexer.open()
+    indexer.add_document( "DOC_A", DOC_A )
+    indexer.add_document( "DOC_B", DOC_B )
+    indexer.add_document( "DOC_C", DOC_C )
+    indexer.flush()
+    assert_finds( indexer )
+  end
+
+  # Creates an index, then asserts that it can be re-opened
+  def __SKIP__test_reopen()
+    test_basic() # build index with 3 docs
+    indexer = RSI::Indexer.new( @root )
+    indexer.serializer = RSI::YAMLSerializer.new()
+    indexer.open()
+    assert_finds( indexer )
+  end
+
+  # Assert that the given index contains the right documents/terms
+  def assert_finds( indexer )
+    a = indexer.find_all( "Bjork" )
+    assert_equal( 1, a.length(), "Should have one hit (DOC_C)" )
+    assert_equal( "DOC_C", a[0], "Should find DOC_C" )
+
+    a = indexer.find_all( "bjork" )
+    assert_equal( 1, a.length(), "(+i) Should have one hit (DOC_C)" )
+    assert_equal( "DOC_C", a[0], "(+i) Should find DOC_C" )
+
+    a = indexer.find_all( "dog" )
+    assert_equal( 2, a.length(), "Should have two hits (DOC_A, DOC_B)" )
+  end
+
+end
+
+if $0 == __FILE__
+  RSI::LogManager.instance.root = "."
+  RSI::LogManager.instance.log_filename = "t_index.log"
+end
+

data/tests/t_index_multi.rb

@@ -0,0 +1,71 @@
+#!/usr/local/bin/ruby -w
+require 'fileutils'
+require 'tmpdir'
+require 'test/unit'
+require 'rsi'
+
+module RSI
+
+  class TestX2SampleAnalyzer
+    attr_accessor :base
+    def initialize( base=RSI::DefaultTextAnalyzer.new() )
+      @base = base
+    end
+
+    def get_field_types()
+      return { "text" => RSI::FIELD_TYPE_TEXT,
+          "subject" => RSI::FIELD_TYPE_TEXT }
+    end
+
+    def tokenize( content )
+      r = {}
+      content =~ /^(.*?)\n/
+      r['subject'] = @base.tokenize_text( $1 )
+      r['text'] = @base.tokenize_text( content )
+      return r
+    end
+
+  end
+end
+
+class X2SampleAnalyzerTest < Test::Unit::TestCase
+  
+  DOC_A = "The subject was a long and pointless one\n" +
+    "\n"+
+    "Also, so was the story.\n\n";
+
+  def setup()
+    @tmp = Dir::tmpdir()
+    @root = File.join( @tmp, "searchtest.#{$$}.#{rand(65535)}" )
+    Dir.mkdir( @root )
+    @failed = false
+  end
+
+  def add_failure( msg, bt )
+    super
+    @failed = true
+  end
+
+  # Delete the temp dir for the index tests
+  def teardown()
+    if @failed
+      print "Test case failed, not cleaning up #@root\n";
+    else
+      FileUtils::rm_rf( @root )
+    end
+  end
+
+  def test_sample()
+    indexer = RSI::Indexer.new( @root )
+    indexer.serializer = RSI::YAMLSerializer.new()
+    indexer.analyzer = RSI::TestX2SampleAnalyzer.new()
+    indexer.open()
+    indexer.add_document( "DOC_A", DOC_A )
+    indexer.flush()
+
+    # TODO: queryanalyzers
+    a = indexer.find_all( "pointless subject" )
+  end
+
+end
+    

data/version.release

@@ -0,0 +1 @@
+0.4

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.4
+specification_version: 1
+name: rsi
+version: !ruby/object:Gem::Version 
+  version: "0.4"
+date: 2005-02-04
+summary: RSI (Ruby Simple Indexer) is a simple full text index
+require_paths: 
+  - lib
+email: gdf@rubyforge.org
+homepage: http://rsi.rubyforge.org
+rubyforge_project: rsi
+description: "RSI is a simple full text search engine implementation in Ruby.  It aims to be
+  easily useful within other programs: simple to set up, simple to use.  An
+  emphasis has been placed on getting functionality out the door, rather than
+  heavy optimization (that can come later).  It still appears to be reasonably
+  fast and efficient (while admitting to have not been heavily profiled...)."
+autorequire: rsi
+default_executable: rsi_search.rb
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+authors: 
+  - Greg Fast
+files: 
+  - LICENSE
+  - Makefile
+  - Manifest
+  - README
+  - TODO
+  - rsi.gemspec
+  - setup.rb
+  - version.release
+  - bin/rsi_search.rb
+  - bin/search_bench.rb
+  - docs/ATTRIB
+  - docs/Changes
+  - docs/Roadmap
+  - lib/rsi.rb
+  - lib/rsi/analysis.rb
+  - lib/rsi/compressed_serializers.rb
+  - lib/rsi/dictionary.rb
+  - lib/rsi/index.rb
+  - lib/rsi/logmanager.rb
+  - lib/rsi/porter.rb
+  - lib/rsi/query.rb
+  - lib/rsi/rsi_intro.rb
+  - lib/rsi/serializers.rb
+  - lib/rsi/stoplist.rb
+  - lib/rsi/stoplist.txt
+  - tests/suite_all.rb
+  - tests/t_analysis.rb
+  - tests/t_index.rb
+  - tests/t_index_multi.rb
+  - tests/t_dictionary.rb
+test_files: 
+  - tests/suite_all.rb
+rdoc_options: []
+extra_rdoc_files: []
+executables: 
+  - rsi_search.rb
+extensions: []
+requirements: []
+dependencies: []