rsi 0.4

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2005, Gregory D. Fast
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/Makefile

@@ -0,0 +1,24 @@
+
+PKGNAME = rsi
+VERSION = $(shell cat version.release)
+RELEASE = $(PKGNAME)-$(VERSION)
+
+PHONY: check.manifest dist
+
+dist: check.manifest version.release
+	mkdir $(RELEASE)
+	tar cf - `cat Manifest` | tar xvf - -C $(RELEASE)
+	tar zcvf $(RELEASE).tar.gz $(RELEASE)
+	rm -rf $(RELEASE)
+
+check.manifest: Manifest
+	for i in `cat Manifest`; do test -e $$i; done
+
+test:
+	ruby -Ilib tests/suite_all.rb
+
+
+#   ;; default ruby-mode has tabs on?  ugh.
+#   (add-hook 'ruby-mode-hook
+#   	  (lambda ()
+#   	    (setq indent-tabs-mode nil)))

data/Manifest

@@ -0,0 +1,30 @@
+LICENSE
+Makefile
+Manifest
+README
+TODO
+rsi.gemspec
+setup.rb
+version.release
+bin/rsi_search.rb
+bin/search_bench.rb
+docs/ATTRIB
+docs/Changes
+docs/Roadmap
+lib/rsi.rb
+lib/rsi/analysis.rb
+lib/rsi/compressed_serializers.rb
+lib/rsi/dictionary.rb
+lib/rsi/index.rb
+lib/rsi/logmanager.rb
+lib/rsi/porter.rb
+lib/rsi/query.rb
+lib/rsi/rsi_intro.rb
+lib/rsi/serializers.rb
+lib/rsi/stoplist.rb
+lib/rsi/stoplist.txt
+tests/suite_all.rb
+tests/t_analysis.rb
+tests/t_index.rb
+tests/t_index_multi.rb
+tests/t_dictionary.rb

data/README

@@ -0,0 +1,49 @@
+RSI README v0.4
+===============
+
+  RSI(Ruby Simple Indexer (or perhaps Really Simple Indexer)) is a
+  simple full text index.
+
+  RSI is a simple full text search engine implementation in Ruby.  It
+  aims to be easily useful within other programs: simple to set up,
+  simple to use.
+  
+  An emphasis has been placed on getting functionality out the door,
+  rather than heavy optimization (that can come later).  It still
+  appears to be reasonably fast and efficient (while admitting to have
+  not been heavily profiled...).
+
+Requirements
+------------
+
+  * Ruby 1.8
+
+Install
+-------
+
+  De-compress archive and enter its top directory.
+  Then type:
+
+   ($ su)
+    # ruby setup.rb
+
+  These simple step installs this program under the default
+  location of Ruby libraries.  You can also install files into
+  your favorite directory by supplying setup.rb some options.
+  Try "ruby setup.rb --help".
+
+
+Usage
+-----
+
+  See rsi_intro.rb .
+
+
+License
+-------
+
+  This is free software.
+  See the LICENSE file included in this distribution for terms.
+
+Copyright 2005, Greg Fast <gdf@speakeasy.net>
+

data/TODO

@@ -0,0 +1,30 @@
+#perf - reduce index size (more!)
+#func - allow fields (date=x, text=x) etc
+func - allow range queries
+#stable - store info on module versions, serialization format, etc in db root
+stable - still need version info stored in root...  icky.
+#stable - tests, benchmarking
+longterm - index efficiency
+longterm - benchmarking
+
+# dist - name project - rsi?
+# dist - setup.rb config
+dist - gem config - tests, docs
+# dist - license
+# dist2 - docs
+
+stable - threadsafe
+
+#func - make bz2 optional
+
+# dist - finish ATTRIB
+# dist2 - organize file structure (eg, porter.rb!@#)
+# dist - no tabs!@  ARGH!
+
+tests - more unit/regression tests
+# idx - format store for multiple indexes
+stable - do something reasonable with logs
+
+#dist - whoops.  use zlib.rb instead of bz2...
+
+dist - include version numbers in source

data/bin/rsi_search.rb

@@ -0,0 +1,50 @@
+#! /opt/ruby-1.8.1/bin/ruby -w
+#
+# Usage:
+#  search.rb [--db /path/to/index/db] --index-dir dir/to/index
+#  search.rb [--db /path/to/index/db] term1 term2 term3
+#
+# (Not yet implemented:)
+#  search.rb [--db /path/to/index/db] --index-files file file file
+#
+require 'getoptlong'
+require 'rsi'
+
+gopt = GetoptLong.new()
+gopt.set_options(
+  [ '--index-dir', '-i', GetoptLong::REQUIRED_ARGUMENT],
+  [ '--db', '-d', GetoptLong::REQUIRED_ARGUMENT]
+)
+# lame...
+OPT = { '--db' => "/var/tmp/search" }
+gopt.each_option {|opt, value| OPT[opt] = value }
+
+indexer = RSI::Indexer.new( OPT['--db'] )
+#indexer.serializer = RSI::YAMLSerializer.new()
+#indexer.serializer = RSI::CompressedSerializer.new()
+indexer.serializer = RSI::NativeSerializer.new() # default
+indexer.open()
+
+if OPT.has_key?( '--index-dir' )
+  to_index = OPT['--index-dir']
+  puts "Indexing #{to_index}..."
+  Dir.foreach( to_index ) do |filename|
+    next if filename =~ /^\./
+    next if FileTest.directory?(filename)
+    next if filename =~ /~$/
+
+    fullpath = File.expand_path( File.join( to_index, filename ) )
+    puts "...#{fullpath}"
+    uri = fullpath
+    contents = File.read( fullpath )
+    indexer.add_document( uri, contents )
+  end
+  puts "Synching..."
+  indexer.flush()
+else
+  query = ARGV.join(" ")
+  puts "Query: #{query}"
+  puts indexer.find_all( query )
+end
+
+

data/bin/search_bench.rb

@@ -0,0 +1,47 @@
+#! /opt/ruby-1.8.1/bin/ruby
+#
+# Benchmark the serializers
+#
+require 'getoptlong'
+require 'benchmark'
+require 'rsi'
+
+testdocs = "./testdocs"
+
+dbs = { 
+  "/var/tmp/search.yaml" => RSI::YAMLSerializer.new(),
+  "/var/tmp/search.marshal" => RSI::NativeSerializer.new(),
+  "/var/tmp/search.bz2" => RSI::CompressedSerializer.new(),
+};
+
+
+Benchmark.bm do |benchmarker|
+  dbs.each do |db, serializer|
+    benchmarker.report(db) do
+
+      if FileTest.exists?( db )
+	raise "DB location #{db} exists"
+      end
+      
+      indexer = RSI::Indexer.new( db )
+      indexer.dictionary.serializer = serializer
+      
+      #puts "Indexing #{testdocs}..."
+      Dir.foreach( testdocs ) do |filename|
+	next if filename =~ /^\./
+	next if FileTest.directory?(filename)
+	next if filename =~ /~$/
+	
+	fullpath = File.expand_path( File.join( testdocs, filename ) )
+	#puts "...#{fullpath}"
+	uri = fullpath
+	contents = File.read( fullpath )
+	indexer.add_document( uri, contents )
+      end
+      #puts "Synching..."
+      indexer.finish()
+      
+    end
+  end
+end
+

data/docs/ATTRIB

@@ -0,0 +1,14 @@
+Misc Attributions
+-----------------
+
+`porter.rb` is Ray Pereda's implementation of the Porter Stemming
+Algorithm (updated by Dave Thomas).  The original, and more
+information, is available at http://www.tartarus.org/~martin/PorterStemmer .
+
+`stoplist.rb` was generated from a stoplist found at
+http://www.cs.utep.edu/nigel/nlp/ir/stoplist.txt, and which in turn was
+attributed as "from Manning and Schutze, 1999, pg 533".
+
+`setup.rb` is (of course) from Minero Aoki's project at
+http://i.loveruby.net/en/prog/setup.html .
+

data/docs/Changes

@@ -0,0 +1,25 @@
+0.1 - 11 Jan 2005
+ - Initial public release.  Basic functions work, lots of nice stuff missing.
+
+0.2 - 12 Jan 2005
+ - Doc release.  Basic documentation, code rearrangement.
+
+0.3 - 18 Jan 2005
+ - Added some unit tests
+ - Refactored dictionary.rb, analysis.rb, towards multiple index support
+ - API: index.finish() is now index.flush()
+ - API: protected internal methods in index, dictionary
+
+0.4 - 04 Feb 2005
+ - Improved logging (see rsi/logmanager.rb)
+ - Improved gemspec to a releasable state
+ - API: multi-field indexes supported
+ - API: structured queries supported
+ - Index metadata is stored and used on open, eliminating need to
+   remember what kind of Serializers, etc, the index was created with.
+ - CompressedSerializer moved into its own file, and now uses zlib
+   (which is part of Ruby 1.8 stdlib) rather than bz2.  The bz2
+   serializer is still included, and raises a sensible exception if
+   used without BZ2 loaded.
+ - More tests
+

data/docs/Roadmap

@@ -0,0 +1,41 @@
+Ye Olde Release Planne
+----------------------
+
+[11 Jan 2005 - released]
+0.1 - initial public release, basic impl
+
+[12 Jan 2005 -released]
+0.2 - docs - api, usage/tutorial, roadmap
+    - make sure source has no tabs (argh!)
+    - source layout, module structure
+
+[18 Jan 2005 - released]
+0.3 - gem dist, basic stability issues
+    - support multiple indexes (req internal api change) [forward from 0.5]
+
+[04 Feb 2005]
+0.4 - support simple field-based queries, including range queries
+      [forward from 0.5]
+    - store index db metainfo in metadata, verify on Index#open()
+      [pushed back from 0.3]
+
+0.45 - doc update
+
+0.5 - build range index impl [pushed back from 0.4]
+    - build compound analyzers
+    - (?) build email, html analyzers
+    - arch/rationale docs? [pushed back from 0.3]
+
+[extreme fuzziness follows]
+
+0.6 - (?) update/migration mechanism for index schema changes
+
+0.7 - reasonably complete test, benchmarking suites
+
+0.8 - support index updates, deletion
+
+1.0 - threadsafe
+    - reduce index size as much as possible
+    - make searches swift
+
+1.1 - interesting query analyzer 

data/lib/rsi.rb

@@ -0,0 +1,40 @@
+#
+# Simple minimal fulltext indexer/search engine.
+#
+# = Synopsis
+#
+#   require 'rsi'
+#   indexer = RSI::Index.new( "/var/db/index" )
+#   uri = "file:///Users/gregfast/poo.txt"
+#   content = File.read( uri )
+#   indexer.add_document( uri, content )
+#   indexer.finish()
+#
+#   docs = indexer.find_all( "quick brown fox" )
+#
+# = Intro
+#
+# See link:files/lib/rsi/rsi_intro_rb.html .
+#
+# = Author
+# 
+# Greg Fast, gdf@speakeasy.net
+# 
+# = Copyright
+# 
+# Copyright 2005 Greg Fast <gdf@speakeasy.net>
+# See LICENSE file distributed with this software.
+#
+#--
+# $Id: rsi.rb 48 2005-01-14 21:57:07Z gdf $ 
+#
+require 'logger'
+require 'yaml'
+require 'bz2'
+require 'rsi/porter'
+require 'rsi/stoplist'
+require 'rsi/serializers'
+require 'rsi/dictionary'
+require 'rsi/index'
+require 'rsi/analysis'
+

data/lib/rsi/analysis.rb

@@ -0,0 +1,79 @@
+#
+# Content/query tokenization classes
+#
+require 'rsi/stoplist'
+require 'rsi/query'
+require 'rsi/logmanager'
+
+module RSI
+
+  # Constants for field/dictionary types.
+  FIELD_TYPE_TEXT = :FIELD_TYPE_TEXT
+  FIELD_TYPE_DATE = :FIELD_TYPE_DATE
+
+  class DefaultTextAnalyzer
+    include Loggable
+
+    attr_accessor :stoplist
+
+    def initialize()
+      @stoplist = nil
+    end
+
+    # Returns a map of fields to field types, for each field returned 
+    # by this analyzer's tokenize() method.
+    # Field names should be safe to be used as file path components.
+    def get_field_types()
+      return { "text" => RSI::FIELD_TYPE_TEXT }
+    end
+
+    # Given a chunk of text content, returns a list of indexable
+    # terms contained in that content.
+    # The content may not be a complete document.
+    # The terms returned may not be a unique set.
+    # The terms returned will all be set to field 'text'.
+    def tokenize( content ) # -> { field, [terms...] }..
+      return { "text" => tokenize_text(content) }
+    end
+
+    def tokenize_query( query )
+      q = RSI::ANDQuery.new()
+      tokenize_text( query ).each do |t|
+        q.add_subquery( RSI::TermQuery.new( 'text', t ) )
+      end
+      return q
+    end
+
+    def tokenize_text( content )
+      initialize_stoplist()
+      c = content.dup.to_s #copy
+      c.gsub!( /\'s\b/, "s" ) # normalize contractions
+      c.gsub!( /n\'t\b/, "nt" )
+      c.tr!( "^a-zA-Z0-9", " " ) # thunk non-wordy chars to ws
+      a = c.split() # split on whitespace
+      a.collect! { |x| x.length<3 ? nil : x } # remove short terms
+      a.compact!
+      a.collect! { |x| x.stem } # stem terms
+      a.compact!
+      a.collect! { |x| x.upcase }
+      a.compact!
+      a.collect! { |x| @stoplist.has_key?(x) ? nil : x } # remove stops
+      a.uniq!
+      a.compact!
+      return a
+    end
+
+    protected
+
+    # done lazily, to account for changing analyzer
+    def initialize_stoplist()
+      return unless @stoplist.nil?
+      @stoplist = {}
+      tokenize_text( RSI::STOPLIST_s ).each do |x|
+        @stoplist[ x ] = 1 if x.length > 0
+      end
+    end
+
+  end
+
+end