moonstone 0.6.0

data/Rakefile

@@ -0,0 +1,61 @@
+$:.unshift "#{here = File.dirname(__FILE__)}/lib"
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+
+deps = %w{ rspec }
+
+task(:install_gems) {
+  deps.each { |g|
+    system "jruby -S gem install #{g}"
+  }
+}
+
+spec = Gem::Specification.new { |s|
+  s.platform = Gem::Platform::RUBY
+
+  s.authors = "Matthew King", "Jason Rush", "Jay Donnell", "Dan Yoder"
+  s.email = "self@automatthew.com"
+  s.files = Dir["{lib,doc,bin,ext}/**/*"].delete_if {|f|
+    /\/rdoc(\/|$)/i.match f
+  } + %w(Rakefile)
+  s.require_path = 'lib'
+  s.has_rdoc = true
+  s.extra_rdoc_files = Dir['doc/*'].select(&File.method(:file?))
+  s.extensions << 'ext/extconf.rb' if File.exist? 'ext/extconf.rb'
+  Dir['bin/*'].map(&File.method(:basename)).map(&s.executables.method(:<<))
+
+  s.name = 'moonstone'
+  s.summary = "Moonstone Agile Search Framework"
+  deps.each &s.method(:add_dependency)
+  s.version = '0.6.0'
+}
+
+Rake::GemPackageTask.new(spec) { |pkg|
+  pkg.need_tar_bz2 = true
+}
+
+task(:uninstall) {
+  system "sudo jruby -S gem uninstall -aIx #{spec.name}"
+}
+
+task(:install => [:uninstall, :package]) {
+  g = "pkg/#{spec.name}-#{spec.version}.gem"
+  system "sudo jruby -S gem install --local #{g}"
+}
+
+task(:uninstall_no_sudo) {
+  system "jruby -S gem uninstall -aIx #{spec.name}"
+}
+
+task(:install_no_sudo => [:uninstall_no_sudo, :package]) {
+  g = "pkg/#{spec.name}-#{spec.version}.gem"
+  system "jruby -S gem install -l #{g}"
+}
+
+desc "run some tests"
+task :test do
+  options = ENV['options']
+  files = FileList['test/**/*.rb'].exclude('test/helpers.rb')
+  puts cmd = "jruby #{options} -I lib -S spec -c #{  files.join(' ') }"
+  system cmd 
+end

data/lib/lucene/analysis.rb

@@ -0,0 +1,58 @@
+module Lucene
+  module Analysis
+    include_package "org.apache.lucene.analysis"
+    
+    module Standard
+      include_package "org.apache.lucene.analysis.standard"
+      [
+        StandardAnalyzer,
+        StandardFilter,
+        StandardTokenizer,
+      ]
+    end
+    include Standard
+    
+    TokenStream.module_eval do
+      include Enumerable
+      def each
+        token = Token.new
+        while token = self.next(token) do
+          yield token
+        end
+      end
+    end
+    
+    Analyzer.module_eval do
+      def tokenize(field, text)
+        token_stream(field, java.io.StringReader.new(text)).map { |token| token.term_text }
+      end
+    end
+    
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [ 
+      CachingTokenFilter,
+      CharTokenizer,
+      ISOLatin1AccentFilter,
+      KeywordAnalyzer,
+      KeywordTokenizer,
+      LengthFilter,
+      LetterTokenizer,
+      LowerCaseFilter,
+      LowerCaseTokenizer,
+      PerFieldAnalyzerWrapper,
+      PorterStemFilter,
+      PorterStemmer,
+      SimpleAnalyzer,
+      SinkTokenizer,
+      StopAnalyzer,
+      StopFilter,
+      TeeTokenFilter,
+      Token,
+      TokenFilter,
+      Tokenizer,
+      WhitespaceAnalyzer,
+      WhitespaceTokenizer,
+      WordlistLoader
+      ]
+  end
+end

data/lib/lucene/document.rb

@@ -0,0 +1,149 @@
+module Lucene
+  module Document
+    include_package "org.apache.lucene.document"
+    
+    # avoid naming problems with Lucene::Document::Document
+    Doc = Lucene::Document::Document
+    
+    # I spit on final class
+    Doc.module_eval do
+      attr_accessor :score, :id, :tokens, :explanation
+      
+      self::Field = Lucene::Document::Field
+      
+      @@field_store = {
+        nil => Field::Store::YES,
+        false => Field::Store::NO,
+        :NO => Field::Store::NO,
+        :no => Field::Store::NO,
+        true => Field::Store::YES,
+        :YES => Field::Store::YES,
+        :yes => Field::Store::YES,
+        :compress => Field::Store::COMPRESS,
+        :COMPRESS => Field::Store::COMPRESS
+      }
+      @@field_index = {
+        nil => Field::Index::ANALYZED,
+        false => Field::Index::NO,
+        :NO => Field::Index::NO,
+        :no => Field::Index::NO,
+        true => Field::Index::ANALYZED,
+        :analyzed => Field::Index::ANALYZED,
+        :ANALYZED => Field::Index::ANALYZED,
+        :not_analyzed => Field::Index::NOT_ANALYZED,
+        :NOT_ANALYZED => Field::Index::NOT_ANALYZED,
+        :analyzed_no_norms => Field::Index::ANALYZED_NO_NORMS,
+        :ANALYZED_NO_NORMS => Field::Index::ANALYZED_NO_NORMS,
+        :not_analyzed_no_norms => Field::Index::NOT_ANALYZED_NO_NORMS,
+        :NOT_ANALYZED_NO_NORMS => Field::Index::NOT_ANALYZED_NO_NORMS
+      }
+      
+      @@field_term_vector = {
+        nil => Field::TermVector::NO,
+        :NO => Field::TermVector::NO,
+        :no => Field::TermVector::NO,
+        false => Field::TermVector::NO,
+        :YES => Field::TermVector::YES,
+        :yes => Field::TermVector::YES,
+        true => Field::TermVector::YES,
+        :WITH_POSITIONS => Field::TermVector::WITH_POSITIONS,
+        :with_positions => Field::TermVector::WITH_POSITIONS,
+        :WITH_OFFSETS => Field::TermVector::WITH_OFFSETS,
+        :with_offsets => Field::TermVector::WITH_OFFSETS,
+        :WITH_POSITIONS_OFFSETS => Field::TermVector::WITH_POSITIONS_OFFSETS,
+        :with_positions_offsets => Field::TermVector::WITH_POSITIONS_OFFSETS
+      }
+      
+      def self.new
+        doc = super()
+        yield doc if block_given?
+        doc
+      end
+      
+      def self.create(fields)
+        doc = self.new
+        fields.each { |field| doc.add_field(*field) }
+        doc
+      end
+      
+      def add_field(name, value, options={})
+        field = if value.is_a? java.io.Reader
+          Field.new(name, value, @@field_term_vector[options[:term_vector]])
+        else
+          store = @@field_store[options[:store]]
+          index = @@field_index[options[:index]]
+          term_vector = @@field_term_vector[options[:term_vector]]
+          params = [name, value, store, index]
+          params << term_vector if term_vector
+          Field.new(*params)
+        end
+        add(field)
+      end
+      
+      # specialty field adders
+      def stored(name, value)
+        add_field(name, value, :store => true, :index => false)
+      end
+      
+      def analyzed(name, value)
+        add_field(name, value, :store => true, :index => :tokenized)
+      end
+      
+      def unanalyzed(name, value)
+        add_field(name, value, :store => true, :index => :not_analyzed)
+      end
+      
+      alias_method :[], :get
+      
+      def get_all(field_name)
+        fields.select { |f| f.name == field_name }.map { |f| f.string_value }
+      end
+      
+      def field_names
+        fields.map { |f| f.name }.uniq
+      end
+      
+      alias_method :keys, :field_names
+      
+      def to_hash
+        hash = {}
+        hash["id"] = @id if @id
+        hash["score"] = @score if @score
+        hash["explanation"] = @explanation.toString(1) if @explanation
+        fields = {}
+        hash["fields"] = fields
+        keys.each do|k|
+          values = self.get_all(k)
+          # fields[k] = values.size == 1 ? values.first : values
+          fields[k] = values
+        end
+        hash["tokens"] = @tokens if @tokens
+        hash
+      end
+      
+      def to_json
+        to_hash.to_json
+      end
+              
+    end
+    
+    Field.module_eval do
+      
+      alias_method :stored?, :is_stored
+      alias_method :indexed?, :is_indexed
+      alias_method :tokenized?, :is_tokenized
+      alias_method :analyzed?, :is_tokenized
+      alias_method :compressed?, :is_compressed
+      
+      def unanalyzed?; indexed? && !analyzed?; end
+      def unindexed?; stored? && !indexed?; end
+      
+    end
+  
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [
+      DateField,
+      DateTools
+      ]
+  end
+end

data/lib/lucene/function.rb

@@ -0,0 +1,9 @@
+module Lucene
+  module Search
+    module Function
+      include_package 'org.apache.lucene.search.function'
+      
+      [FieldScoreQuery, CustomScoreQuery]
+    end
+  end
+end

data/lib/lucene/index.rb

@@ -0,0 +1,48 @@
+module Lucene
+  module Index
+    include_package "org.apache.lucene.index"
+    
+    IndexWriter.module_eval do
+      MaxFieldLength = self::MaxFieldLength
+      
+      def self.open(*args)
+        args << MaxFieldLength::UNLIMITED unless args.last.is_a? MaxFieldLength
+        writer = new(*args)
+        begin
+          result = yield(writer)
+        ensure
+          writer.close
+        end
+        result
+      end
+      
+      def add_documents(docs)
+        docs.each { |doc| add_document(doc) }
+      end
+      
+      
+    end
+    
+    TermEnum.module_eval do
+      include Enumerable
+      
+      def each
+        while self.next do
+          yield term
+        end
+      end
+      
+      def for_field(field_name)
+        select { |t| t.field == field_name }.map { |t| t.text }
+      end
+      
+    end
+    
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [
+      IndexReader,
+      Payload,
+      Term,
+      ]
+  end
+end

data/lib/lucene/query_parser.rb

@@ -0,0 +1,14 @@
+module Lucene
+  module QueryParser
+    include_package "org.apache.lucene.queryParser"
+    
+    # avoid problems with Lucene::QueryParser::QueryParser
+    Parser = org.apache.lucene.queryParser.QueryParser
+    
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [
+      MultiFieldQueryParser,
+      Token
+      ]
+  end
+end

data/lib/lucene/search.rb

@@ -0,0 +1,195 @@
+module Lucene
+  module Search
+    include_package "org.apache.lucene.search"
+    
+    [ SortField, Sort ]
+    
+    Hits.module_eval do
+      include Enumerable
+      def each
+        i = 0
+        while i < length do
+          yield doc(i)
+          i += 1
+        end
+      end
+      
+      def to_a
+        map
+      end
+      
+      alias_method :size, :length
+    end
+    
+    TopDocs.module_eval do
+      attr_accessor :query
+      include Enumerable
+            
+      def each(searcher=nil)
+        initialize_docs(searcher) if searcher && documents.empty? #Do we ever want to reinitialize the documents list?
+        documents.each { |doc| yield doc }
+      end
+      
+      def initialize_docs(searcher)
+        @offset ||= 0
+        self.scoreDocs.each_with_index do |sd, i|
+          #For pagination, only init the docs that fit the offset
+          if i >= @offset
+            doc = searcher.doc(sd.doc)
+            doc.score = sd.score
+            doc.id = sd.doc
+            documents << doc
+          end
+        end
+      end
+      
+      #Remove docs that precede the offset
+      def offset!(offset)
+        @offset = offset || 0
+        self
+      end
+      
+      def offset
+        @offset ||= 0
+      end
+      
+      def [](index)
+        documents[index]
+      end
+      
+      def first
+        documents[0]
+      end
+      
+      def last
+        to_a.last
+      end
+      
+      def length
+        self.scoreDocs.length - (@offset || 0)
+      end
+      
+      alias_method :size, :length
+      
+      def empty?
+        self.length == 0
+      end
+      
+      def to_hash
+        {
+          :query => self.query,
+          :total_hits => self.totalHits,
+          :documents => self.to_a
+        }
+      end
+      
+      def to_json
+        to_hash.to_json
+      end
+      
+    private
+      def documents
+        @documents ||= []
+      end
+    end
+    
+    Hit.module_eval do
+      alias_method :[], :get
+    end
+    
+    IndexSearcher.module_eval do
+      def self.open(*args)
+        searcher = new(*args)
+        begin
+          result = yield(searcher)
+        ensure
+          searcher.close
+        end
+        result
+      end
+    end
+    
+    BooleanQuery.module_eval do
+      
+      def self.and(*queries)
+        q = self.new
+        queries.each { |query| q.add(query, BooleanClause::Occur::MUST) }
+        q
+      end
+      
+      def self.or(*queries)
+        q = self.new
+        queries.each { |query| q.add(query, BooleanClause::Occur::SHOULD) }
+        q
+      end
+      
+      def self.not(*queries)
+        q = self.new
+        queries.each { |query| q.add(query, BooleanClause::Occur::MUST_NOT) }
+        q
+      end
+      
+      def and(*queries)
+        queries.each { |query| add(query, BooleanClause::Occur::MUST) }
+        self
+      end
+      
+      def or(*queries)
+        queries.each { |query| add(query, BooleanClause::Occur::SHOULD) }
+        self
+      end
+      
+      def not(*queries)
+        queries.each { |query| add(query, BooleanClause::Occur::MUST_NOT) }
+        self
+      end
+            
+    end
+    
+    TermQuery.module_eval do
+      
+      def self.new(*args)
+        term = args.first.is_a?(Lucene::Index::Term) ? args.first : Lucene::Index::Term.new(*args)
+        super(term)
+      end
+      
+    end
+    
+    module Spell
+      include_package 'org.apache.lucene.search.spell'
+      [PlainTextDictionary]
+    end
+    
+    PhraseQuery.module_eval do
+      def self.create(field, phrase)
+        raise "I need an array" unless phrase.is_a? Array
+        query = self.new
+        phrase.each do |word|
+          query.add(Index::Term.new(field, word))
+        end
+        query
+      end
+    end
+    
+    
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [
+      Explanation,
+      FilteredQuery,
+      FuzzyQuery,
+      HitIterator,
+      MultiPhraseQuery,
+      PrefixQuery,
+      Query,
+      RangeQuery,
+      ScoreDoc,
+      Searcher,
+      Similarity,
+      TopDocCollector,
+      TopFieldDocCollector,
+      TopFieldDocs,
+      Weight,
+      WildcardQuery
+      ]
+  end
+end

data/lib/lucene/store.rb

@@ -0,0 +1,12 @@
+module Lucene
+  module Store
+    include_package "org.apache.lucene.store"
+    
+    # Biggie Smalls, Biggie Smalls, Biggie Smalls
+    [
+      Directory,
+      FSDirectory,
+      RAMDirectory
+      ]
+  end
+end

data/lib/moonstone/analyzer.rb

@@ -0,0 +1,23 @@
+module Moonstone
+  class Analyzer < Lucene::Analysis::Analyzer
+    attr_accessor :filter_chain
+    # Moonstone::Analyzer.new(WhitespaceTokenizer, StandardFilter, StemFilter)
+    # FIXME:  Why don't we explicitly require a tokenizer + *filters ?
+    def self.new(*classes)
+      analyzer = super()
+      analyzer.filter_chain = classes
+      analyzer
+    end
+    
+    def tokenStream(field_name, reader)
+      tokenizer, *args = @filter_chain[0]
+      stream = tokenizer.new(reader, *args)
+      @filter_chain.slice(1..-1).each do |filter|
+        klass, *args = filter
+        stream = klass.new(stream, *args)
+      end
+      stream
+    end
+    
+  end
+end

data/lib/moonstone/engine.rb

@@ -0,0 +1,186 @@
+module Moonstone
+  class Engine
+    include Lucene::Index
+    include Lucene::Search
+    
+    attr_reader :store, :similarity
+    
+    # :store should be a String or some kind of Lucene::Store::Directory
+    def initialize(options = {})
+      @store = options[:store] || Lucene::Store::RAMDirectory.new
+      @inspect = options[:inspect]
+    end
+    
+    # The source should be enumerable.
+    def index(source, optimize=true)
+      IndexWriter.open(@store, analyzer) do |writer|
+        writer.set_similarity(@similarity.new) if @similarity
+        
+        source.each_with_index do |record, i|
+          doc = doc_from(record)
+          writer.add_document(doc) if doc
+          Moonstone::Logger.info "Indexed #{i+1} records" if (i+1)%1000 == 0
+        end
+        writer.optimize if optimize
+        yield writer if block_given? #For post-processing stuff where you still need access to the writer
+      end
+      refresh_searcher
+    end
+    
+    def stamp_metadata
+      metadata = Lucene::Document::Doc.new
+      metadata.add_field 'metadata', 'index', :index => :not_analyzed
+      metadata.add_field 'build_date', Date.today.strftime("%Y-%m-%d"), :index => false
+      metadata.add_field 'engine_name', self.class.name, :index => false
+      metadata.add_field 'engine_version', `git show-ref -s --abbrev HEAD`.chomp, :index => false
+      metadata.add_field 'query_conditions', ENV['query_conditions'].to_s, :index => false
+      writer do |w|
+        w.add_document(metadata)
+      end
+    end
+    
+    def index_metadata
+      query = TermQuery.new 'metadata', 'index'
+      @index_metadata ||= search(query).last
+    end
+    
+    def doc_count
+      @reader ||= IndexReader.open(@store)
+      @reader.max_doc
+    end
+    
+    def document(id)
+      @reader ||= IndexReader.open(@store)
+      if id < @reader.max_doc
+        doc = @reader.document(id) 
+        doc.tokens = tokens_for_doc(id)
+        doc.id = id
+        doc
+      end
+    end
+    
+    # Adds docs to index.  docs must be an enumerable set of such objects that doc_from can turn into a document
+    def insert_documents(source, optimize=false)
+      index(source, optimize)
+      refresh_searcher
+    end
+    
+    def insert_document(source, optimize=false)
+      insert_documents([source], optimize)
+    end
+    
+    # docs must be enumerable set of hashes, with fields
+    # :field, :value, :document 
+    # (where field and value combine to make a term to match documents to replace)
+    def update_documents(docs)
+      IndexWriter.open(@store, analyzer) do |writer|
+        writer.set_similarity(@similarity.new) if @similarity
+        docs.each do |doc|
+          raise "Invalid arguments" unless doc[:field] && doc[:value] && doc[:document]
+          term = Term.new(doc[:field], doc[:value])
+          document = doc_from(doc[:document])
+          writer.updateDocument(term, document)
+        end
+      end
+      refresh_searcher
+    end
+    
+    def update_document(doc)
+      update_documents([doc])
+    end
+    
+    # terms should be an enumerable set of hashes, with fields
+    # :field and :value, which combine to make a term to match documents to delete
+    def delete_documents(terms)
+      IndexWriter.open(@store, analyzer) do |writer|
+        terms.each do |t|
+          term = Term.new(t[:field], t[:value])
+          writer.deleteDocuments(term)
+        end
+      end
+      refresh_searcher
+    end
+    
+    def delete_document(term)
+      delete_documents([term])
+    end
+    
+    # Takes any kind of input object parsable by your #create_query method.  Quack.  
+    # Options patterns (see javadoc for org.apache.lucene.search.Searcher):
+    # Returns a TopDocs object
+    # Note that Hits is deprecated so the versions of search() returning a Hits object are not implemented
+    def search(input, options = {})
+      query = input.kind_of?(Lucene::Search::Query) ? input : create_query(input)
+      @searcher ||= IndexSearcher.new(@store)
+      top_docs = if (hit_collector = options[:hit_collector])
+        args = [ options[:filter], hit_collector ].compact
+        @searcher.search(query, *args)
+        hit_collector.topDocs
+      else
+        options[:limit] ||= 25
+        options[:offset] ||= 0
+        args = [ options[:filter], (options[:limit] + options[:offset]) ]  #Always include both of these, even if nil
+        args << options[:sort] if options[:sort]
+        @searcher.search(query, *args).offset!(options[:offset])
+      end
+      top_docs.each(@searcher) do |doc| 
+        doc.tokens = self.tokens_for_doc(doc) if inspect_mode?
+        yield doc if block_given?
+      end
+      top_docs
+    end
+    
+    #Reopen the searcher (used when the index has changed)
+    def refresh_searcher
+      @searcher = IndexSearcher.new(@store) if @searcher  #If it's nil, it'll get lazy loaded
+    end
+    
+    def close
+      @searcher.close if @searcher
+      @reader.close if @reader
+    end
+    
+    # Returns an instance of the Analyzer class defined within 
+    # this class's namespace.
+    def analyzer
+      @analyzer ||= self.class::Analyzer.new
+    end
+    
+    # Opens an IndexWriter for the duration of the block.
+    #   engine.writer { |w| w.add_document(doc) }
+    def writer
+      IndexWriter.open(@store, self.class::Analyzer.new) do |writer|
+        writer.set_similarity(@similarity.new) if @similarity
+        yield writer
+      end
+    end
+
+    # Opens an IndexSearcher for the duration of the block.
+    #   engine.searcher { |s| s.search(query_object) }
+    def searcher
+      IndexSearcher.open(@store) do |searcher|
+        searcher.set_similarity(@similarity.new) if @similarity
+        yield searcher
+      end
+    end
+
+    # Opens an IndexReader for the duration of the block.
+    #   engine.reader { |r| r.terms }
+    def reader
+      reader = IndexReader.open(@store)
+        yield reader
+      reader.close
+    end
+    
+    
+    def parser(field, analyzer = nil)
+      @parser ||= {}
+      @parser[field.to_sym] ||= Lucene::QueryParser::Parser.new(field, analyzer || self.analyzer)
+    end
+    
+    def inspect_mode?
+      @inspect
+    end
+        
+  end
+end

data/lib/moonstone/filter.rb

@@ -0,0 +1,30 @@
+module Moonstone
+  class Filter < Lucene::Analysis::TokenFilter
+    
+    def initialize(stream)
+      if block_given?
+        self.class.module_eval do
+          define_method :process do |token|
+            yield token
+          end
+        end
+      end
+      super
+      @stream = stream
+    end
+    
+    def next(token=nil)
+      if token = (token ? @stream.next(token) : @stream.next)
+        text = process(token.term_text)
+        # skip a token if its text is empty
+        if text.empty?
+          token = self.next(token)
+        else
+          token.term_text = text
+          token
+        end
+      end
+    end
+    
+  end
+end

data/lib/moonstone/filters/synonymer.rb

@@ -0,0 +1,26 @@
+module Moonstone
+    
+  module Filters
+    
+    class Synonymer < Moonstone::QueuedFilter
+      
+      def initialize(stream, synonym_hash)
+        @synonym_hash = synonym_hash
+        super(stream)
+      end
+
+      def process(text)
+        if syns = @synonym_hash[text]
+          if syns.is_a?(String)
+            [text, syns]
+          elsif syns.is_a?(Array)
+            [text].concat syns
+          end
+        else
+          text
+        end
+      end
+      
+    end
+  end
+end

data/lib/moonstone/index_inspection.rb

@@ -0,0 +1,38 @@
+#Methods to assist in index analysis
+module Moonstone
+  class Engine
+    
+    #Return a hash of tokens, keyed on field name, for the given doc.
+    # Doc can be either a Document, or the integer document id.
+    # Note that if it is a Document, doc.id cannot be nil
+    def tokens_for_doc(doc, fields = nil)
+      tokens = {}
+      self.reader do |reader|
+        unless doc.kind_of?(Lucene::Document::Doc)
+          doc_id = doc
+          doc = reader.document(doc)
+          doc.id = doc_id
+        end
+        fields = doc.keys if fields.nil?
+        fields.each do |field|
+          tokens[field] = []
+          tfv = reader.getTermFreqVector(doc.id, field)
+          if tfv && tfv.size > 0 && tfv.respond_to?(:getTermPositions)
+            tv = tfv.getTerms
+            tv.length.times do |i|
+              positions = tfv.getTermPositions(i) || []
+              positions.each { |pos| tokens[field][pos] = tv[i]}
+            end
+          end 
+        end
+      end
+      tokens
+    end
+    
+    #Helper, delegates to tokens_for_doc
+    def tokens_for_field(doc, field)
+      tokens_for_doc(doc, [field])[field]
+    end
+                
+  end
+end

data/lib/moonstone/multi_analyzer.rb

@@ -0,0 +1,25 @@
+module Moonstone
+  class MultiAnalyzer < Lucene::Analysis::Analyzer
+    attr_accessor :fields
+    
+    # Moonstone::MultiAnalyzer.new  :name => [KeywordTokenizer, SynonymFilter], 
+    #                               :categories => [WhitespaceTokenizer, SynonymFilter, StemFilter]
+    def self.new(hash={})
+      analyzer = super()
+      analyzer.fields = hash
+      analyzer
+    end
+    
+    def tokenStream(field_name, reader)
+      filter_chain = @fields[field_name.to_sym] || @fields[true]
+      tokenizer, *args = filter_chain[0]
+      stream = tokenizer.new(reader, *args)
+      filter_chain.slice(1..-1).each do |filter|
+        klass, *args = filter
+        stream = klass.new(stream, *args)
+      end
+      stream
+    end
+    
+  end
+end

data/lib/moonstone/queued_filter.rb

@@ -0,0 +1,45 @@
+module Moonstone
+  
+  class QueuedFilter < Moonstone::Filter
+    
+    def initialize(stream)
+      @buffer = []
+      super
+    end
+    
+    def read_buffer(token=nil)
+      if item = @buffer.shift
+        if item.is_a? String
+          token ||= Lucene::Analysis::Token.new
+          token.term_text = item
+          token
+        else
+          raise "What have you done?"
+        end
+      end
+    end
+    
+    def next(token=nil)
+      if t = read_buffer(token)
+        t
+      elsif token = (token ? @stream.next(token) : @stream.next)
+        results = process(token.term_text)
+        if results.is_a? Array
+          text = results.shift
+          results.each { |t| @buffer << t }
+        else
+          text = results
+        end
+        # skip a token if its text is empty
+        if text && text.empty?
+          token = self.next(token)
+        else
+          token.term_text = text
+          token
+        end
+      end
+    end
+    
+  end
+  
+end

data/lib/moonstone/racker/basic_search.rb

@@ -0,0 +1,28 @@
+require 'moonstone/racker'
+module Moonstone
+  module Racker
+    module BasicSearch
+      include Moonstone::Racker
+      
+      # GET /search.html?input=happiness
+      def html_GET_search(request)
+        results = search(request.params['input'], search_options(request))
+        results.join("\n<br>")
+      end
+
+      # GET /search.json?input=happiness
+      def json_GET_search(request)
+        results = search(request.params['input'], search_options(request))
+        results.to_json
+      end
+
+      # POST /search.json
+      def json_POST_search(request)
+        options = search_options(request)
+        data = request.env['rack.input'].read
+        JSON.parse(data).map { |input| search(input, options) }.to_json
+      end
+      
+    end
+  end
+end

data/lib/moonstone/racker/local_search.rb

@@ -0,0 +1,33 @@
+require 'moonstone/racker'
+module Moonstone
+  module Racker
+    module LocalSearch
+      include Moonstone::Racker
+    
+      def json_GET_search(request)
+        args = request.params.values_at('input', 'lat', 'lon')
+        options = search_options(request)
+        args << options
+        t = Time.now
+        results = search(*args).to_hash
+        results[:time] = Time.now - t
+        results.to_json
+      end
+    
+      # JSON body should contain an array of 3-element arrays (topic, lat, lon)
+      #  curl -i -X POST -d '[ ["plumbers", "", ""], ["burgers", "", ""] ]' \
+      #    http://localhost:9292/search.json
+      def json_POST_search(request)
+        options = search_options(request)
+        data = request.env['rack.input'].read
+        JSON.parse(data).map do |input, lat, lon|
+          t = Time.now
+          results = search(input, lat, lon, options).to_hash
+          results[:time] = Time.now - t
+          results
+        end.to_json
+      end
+    
+    end
+  end
+end

data/lib/moonstone/racker.rb

@@ -0,0 +1,84 @@
+require 'rack'
+require 'json'
+module Moonstone
+  # include me in a Moonstone::Engine, maybe?
+  module Racker
+    
+    PathMatcher = %r{^/([\w_]+)\.([\w_]+)$}
+    
+    def call(env)
+      request, response = Rack::Request.new(env), Rack::Response.new
+      # Determine (or possibly fake) an HTTP method
+      real = request.request_method.upcase
+      http_method = if (real == 'POST') && (fake = request.params['_method'])
+        fake.upcase
+      else
+        real
+      end
+      # Match against a very limited species of URI path.
+      whole, action, ext = request.path_info.match(PathMatcher).to_a
+      # Poor man's content negotiation
+      content_type = case ext
+      when 'json'
+        'application/json'
+      end
+      response['Content-Type'] = content_type if content_type
+      # Poor man's routing
+      method_name = action ? "#{ext || 'html'}_#{http_method}_#{action}" : nil
+      if method_name && respond_to?(method_name)
+        response.body = send(method_name, request).to_s
+      else
+        response.status, response.body = 404, "404"
+      end
+      response.finish
+    end
+    
+    # helper for action methods
+    def search_options(request)
+      params = request.params
+      limit = params['limit']
+      offset = params['offset']
+      options = {}
+      options[:limit] = limit.to_i if limit
+      options[:offset] = offset.to_i if offset
+      options
+    end
+    
+    def json_GET_engine_version(request)
+      { :name => self.class.name, 
+        :version => `git show-ref -h -s --abbrev HEAD`.chomp.split.first
+      }.to_json
+    end
+    
+    def json_GET_index_info(request)
+      md = index_metadata || {}
+      {   :build_date => md["build_date"],
+          :build_engine => {  :name => md["engine_name"], 
+                              :version => md["engine_version"]},
+          :query_conditions => md["query_conditions"],
+          :doc_count => doc_count
+      }.to_json
+    end
+    
+    def json_GET_document(request)
+      document(request.params['id'].to_i).to_json
+    end
+    
+    def self.generate_rackup_file(engine, store)      
+      rackup = <<RACKUP
+options[:Port] = 9293
+#{yield}
+require 'moonstone/racker/local_search'
+#{engine}.module_eval do
+  include Moonstone::Racker::LocalSearch
+end
+run #{engine}.new(:store => "#{File.expand_path store}")
+RACKUP
+
+      File.open "#{File.dirname(store)}/config.ru", "w" do |f|
+        f.puts rackup
+      end
+    end
+    
+  end
+end

data/lib/moonstone/tokenizer.rb

@@ -0,0 +1,19 @@
+module Moonstone
+  class Tokenizer < Lucene::Analysis::Tokenizer
+   
+    include Lucene::Analysis
+    
+    def initialize(reader)
+      @reader = java.io.BufferedReader.new(reader)
+    end
+    
+    # No, this is not terribly useful.  Subclass me already.
+    def next(token=nil)
+      token = (token ? token.clear :  Token.new)
+      token.set_term_text @reader.read_line
+      token.set_start_offset 1
+      token.set_end_offset 1
+    end
+   
+  end
+end

data/lib/moonstone.rb

@@ -0,0 +1,28 @@
+require 'java'
+require 'logger'
+require 'json'
+
+$:.unshift(here = File.dirname(__FILE__))
+
+Dir["#{here}/jar/*.jar"].each { |jar| require jar }
+require 'lucene/analysis'
+require 'lucene/document'
+require 'lucene/function'
+require 'lucene/index'
+require 'lucene/query_parser'
+require 'lucene/search'
+require 'lucene/store'
+
+require 'moonstone/engine'
+require 'moonstone/tokenizer'
+require 'moonstone/filter'
+require 'moonstone/queued_filter'
+require 'moonstone/analyzer'
+require 'moonstone/multi_analyzer'
+require 'moonstone/index_inspection'
+
+require 'moonstone/filters/synonymer.rb'
+
+require 'moonstone/racker'
+
+Moonstone::Logger = Logger.new($stderr) unless defined? Moonstone::Logger

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification 
+extensions: []
+
+homepage: 
+executables: []
+
+version: !ruby/object:Gem::Version 
+  version: 0.6.0
+post_install_message: 
+date: 2009-06-16 07:00:00 +00:00
+files: 
+- lib/jar
+- lib/lucene
+- lib/moonstone
+- lib/moonstone.rb
+- lib/jar/lucene-core-2.4.0.jar
+- lib/jar/lucene-spellchecker-2.4-dev.jar
+- lib/lucene/analysis.rb
+- lib/lucene/document.rb
+- lib/lucene/function.rb
+- lib/lucene/index.rb
+- lib/lucene/query_parser.rb
+- lib/lucene/search.rb
+- lib/lucene/store.rb
+- lib/moonstone/analyzer.rb
+- lib/moonstone/engine.rb
+- lib/moonstone/filter.rb
+- lib/moonstone/filters
+- lib/moonstone/index_inspection.rb
+- lib/moonstone/multi_analyzer.rb
+- lib/moonstone/queued_filter.rb
+- lib/moonstone/racker
+- lib/moonstone/racker.rb
+- lib/moonstone/tokenizer.rb
+- lib/moonstone/filters/synonymer.rb
+- lib/moonstone/racker/basic_search.rb
+- lib/moonstone/racker/local_search.rb
+- Rakefile
+rubygems_version: 1.3.1
+rdoc_options: []
+
+signing_key: 
+cert_chain: []
+
+name: moonstone
+has_rdoc: true
+platform: ruby
+summary: Moonstone Agile Search Framework
+default_executable: 
+bindir: bin
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  version: 
+  requirements: 
+  - - '>='
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement 
+  version: 
+  requirements: 
+  - - '>='
+    - !ruby/object:Gem::Version 
+      version: "0"
+require_paths: 
+- lib
+specification_version: 2
+test_files: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  type: :runtime
+  name: rspec
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    version: 
+    requirements: 
+    - - '>='
+      - !ruby/object:Gem::Version 
+        version: "0"
+description: 
+email: self@automatthew.com
+authors: 
+- Matthew King
+- Jason Rush
+- Jay Donnell
+- Dan Yoder
+extra_rdoc_files: []
+
+requirements: []
+
+rubyforge_project: 
+autorequire: