stellr 0.1.0

data/History.txt

@@ -0,0 +1,4 @@
+== 0.1.0 / 2008-08-05
+
+* Initial release
+

data/Manifest.txt

@@ -0,0 +1,36 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/stellr
+bin/stellr-search
+config/stellr.yml
+lib/stellr.rb
+lib/stellr/client.rb
+lib/stellr/collections.rb
+lib/stellr/collections/base.rb
+lib/stellr/collections/searchable_collection.rb
+lib/stellr/collections/writeable_collection.rb
+lib/stellr/collections/rsync.rb
+lib/stellr/collections/static.rb
+lib/stellr/collections/multi_collection.rb
+lib/stellr/config.rb
+lib/stellr/search.rb
+lib/stellr/search/search_result.rb
+lib/stellr/search/search_results.rb
+lib/stellr/server.rb
+lib/stellr/strategies.rb
+lib/stellr/strategies/base.rb
+lib/stellr/strategies/blocking.rb
+lib/stellr/strategies/queueing.rb
+lib/stellr/utils.rb
+lib/stellr/utils/shutdown.rb
+lib/stellr/utils/observable.rb
+test/fixtures/movies.yml
+test/stellr_test.rb
+test/test_collections_base.rb
+test/test_helper.rb
+test/test_rsync_collection.rb
+test/test_server.rb
+test/test_static_collection.rb
+test/test_stellr.rb

data/README.txt

@@ -0,0 +1,109 @@
+stellr
+    by Jens Kraemer and Benjamin Krause
+    http://rubyforge.org/projects/stellr
+
+== DESCRIPTION:
+  
+Stellr is a Ferret based standalone search server featuring a DRB and (soon to come) an http frontend. It can handle multiple indexes, including multi-index searches. A client library and a simple command line query tool are included with the gem.
+
+== FEATURES:
+
+* DRb frontend
+* easy to use client library (see below)
+* multi index search
+* Index rotation
+  Stellr always keeps two versions of your index around - one is used in a
+  multi threaded, read only way to handle incoming search requests, while the
+  other one is written to when you index something.
+  Using the switch function you may decide when to switch over searching from 
+  the old index to the new one. Then, changes will be synced, and searches will
+  see the new or updated data from before the switch call.
+* Index synchronization
+  Two kinds of synchronization methods are supported for now: rsync, using rsync
+  two copy over the changes from one index to the other, and static, which will
+  completely replace the old index with the new one. While the latter is suitable
+  for indexes which you rebuild completely from time to time, the former is good
+  for large indexes that are updated frequently or that are too large for frequent
+  rebuilds.
+
+== SYNOPSIS:
+
+* start the server:
+
+  stellr -c /path/to/config.yml start
+
+* index something
+
+  require 'stellr/client'
+  stellr = Stellr::Client.new('druby://localhost:9010')
+  config = {
+    :collection => :static,         # static collections are rebuilt from scratch everytime changes occur
+    # :collection => :rsync,        # use the rsync collection for indexes that are updated frequently and/or are 
+                                    # too large to be rebuilt from scratch every time they're updated
+    :analyzer => 'My::Analyzer',
+    :fields => {
+      :title   => { :boost => 5, :store => :yes },
+      :content => { :store => :no }
+    }
+  }
+  collection = stellr.connect('my_collection', config)
+  collection.add_record(:id => 1, :title => 'Some Title', :content => 'Content')
+  collection.switch # 
+
+* command line search
+
+  stellr-search my_collection 'query string'
+
+  for now, this will display the first 10 hits only.
+
+* search via the client library
+
+  require 'stellr/client'
+  stellr = Stellr::Client.new('druby://localhost:9010')
+  collection = stellr.connect 'my_collection',
+                              :analyzer => 'My::Analyzer'     # the analyzer to use for query parsing
+  results = collection.search 'querystring',
+                              :page => 1, :per_page => 100,   # built in pagination support
+                              :fields => [:title, :content],  # the fields to search in
+                              :get_fields => [ :title ]       # the fields to fetch for result display
+
+* multi-collection search
+  pass an array of collection names to search multiple
+  collections at once:
+
+  collection = stellr.connect [ 'my_collection', 'another_collection' ],
+                              :analyzer => 'My::Analyzer'
+
+== REQUIREMENTS:
+
+* Ferret (gem install ferret)
+* daemons (gem install daemons)
+
+== INSTALL:
+
+* sudo gem install stellr
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2007 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'hoe'
+$:.unshift 'lib'
+require './lib/stellr.rb'
+
+class Hoe
+  def extra_deps
+    @extra_deps.reject do |x|
+      Array(x).first == 'hoe'
+    end
+  end
+end
+
+Hoe.new('stellr', Stellr::VERSION) do |p|
+  p.rubyforge_name = 'stellr'
+  p.author = [ 'Benjamin Krause', 'Jens Krämer' ]
+  p.email = [ 'bk@benjaminkrause.com', 'jk@jkraemer.net' ]
+  p.summary = 'Stellr is a Ferret based standalone search server.'
+  p.description = p.paragraphs_of('README.txt', 2..5).join("\n\n")
+  p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[1..-1]
+  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
+  p.extra_deps << [ 'ferret', '>= 0.11.6', 'daemons', '>= 1.0.10', 'fastthread', '>= 1.0' ]
+end
+
+desc "Release and publish documentation"
+task :repubdoc => [:release, :publish_docs]
+
+# vim:syntax=ruby

data/bin/stellr

@@ -0,0 +1,64 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'optparse'
+require 'daemons'
+require 'fileutils'
+
+begin
+  gem 'stellr' 
+rescue Gem::LoadError
+  $:.unshift(File.dirname(__FILE__), "../lib")
+end
+
+require 'stellr'
+
+include Stellr
+
+commands = %w( start stop restart status )
+
+options = OptionParser.new do |opts|
+  
+  opts.banner = "Usage: #{File.basename(__FILE__)} [options] #{commands.join('|')}"
+  
+  opts.on("-c", "--config CONFIG", "config file location") do |value|
+    @config_file = value
+  end
+  
+  opts.on("-h", "--help", "show this help") do
+    @help = true
+  end
+end
+
+cmdline = options.parse( ARGV )
+command = cmdline.last
+
+if @help or not commands.include?( command )
+  puts options.to_s
+else
+  config   = Stellr::Config.new @config_file
+  FileUtils.mkdir_p config.tmp_dir
+  FileUtils.mkdir_p config.log_dir
+  
+  Daemons.run_proc( "stellr", :dir        => config.tmp_dir, 
+                              :dir_mode   => :normal,
+                              :log_output => true,
+                              :ARGV       => [ command ] ) do
+    Stellr::start_server config
+  end
+
+  if command == "start"
+    sleep 2
+    pid = Daemons::PidFile.new( config.tmp_dir, 'stellr' )
+    if pid.exist? and Daemons::Pid.running?(pid.pid)
+      exit 0
+    else
+      puts "not started. Watch #{config.tmp_dir}/stellr.output for errors."
+      exit 1
+    end  
+  end
+
+  # FIXME how to correctly shut down the running server so indexes are closed
+  # and such?
+  # look for some hook provided by daemon tools, or trap SIGKILL
+end

data/bin/stellr-search

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'optparse'
+
+begin
+  gem 'stellr' 
+rescue Gem::LoadError
+  $:.unshift(File.dirname(__FILE__), "../lib")
+end
+
+require 'stellr'
+require 'stellr/client'
+
+options = OptionParser.new do |opts|
+  
+  opts.banner = "Usage: #{File.basename(__FILE__)} [options] collection query"
+  
+  opts.on("-c", "--config CONFIG", "config file location") do |value|
+    @config_file = value
+  end
+  
+  opts.on("-h", "--help", "show this help") do
+    @help = true
+  end
+end
+
+cmdline = options.parse( ARGV )
+query = cmdline.pop rescue nil
+collection_name = cmdline.pop
+
+if @help
+  puts options.to_s
+else
+  config = Stellr::Config.new @config_file
+  client = Stellr::Client.new config.drb_uri
+  collection = client.connect collection_name
+  puts "searching collection #{collection_name} for >#{query}< ..."
+  results = collection.search query, :limit => 10
+  if results.any?
+    puts "showing first 10 of #{results.total_hits} results"
+    results.each do |r| 
+      puts r[:id]
+    end
+  else
+    puts "no results for >#{query}<"
+  end
+end
+
+

data/config/stellr.yml

@@ -0,0 +1,8 @@
+:port:      9010
+:host:      'localhost'
+:base_dir:  '/var/stellr'
+:data_dir:  'data'
+:log_dir:   'log'
+:tmp_dir:   'tmp'
+:conf_dir:  'conf'
+:log_level: :warn

data/lib/stellr.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'thread'
+begin
+  require 'fastthread' 
+rescue LoadError
+  puts "couldn't load fastthread"
+end
+require 'drb'
+require 'monitor'
+require 'ferret'
+require 'stellr/utils'
+require 'stellr/server'
+require 'stellr/config'
+require 'stellr/collections'
+require 'stellr/strategies'
+require 'stellr/search'
+
+$SAFE = 1
+
+module Stellr
+  VERSION = '0.1.0'
+  
+  def self.start_server( config )
+    if config.script
+      begin
+        load config.script
+      rescue Exception => e
+        puts "\nerror loading script #{config.script}: #{e}\n#{e.backtrace.join("\n")}"
+        exit 1
+      end
+    end
+    stellr = Server.new config
+    server = DRb.start_service config.drb_uri, stellr
+    server.thread.join
+  end
+  
+end

data/lib/stellr/client.rb

@@ -0,0 +1,78 @@
+require 'drb'
+require 'stellr/search'
+
+module Stellr
+
+  # = Stellr client
+  #
+  # This class acts as a wrapper around the connection to a Stellr server.
+  # Typical usage looks like this:
+  #
+  #    stellr = Stellr::Client.new('druby://myserver.com:9000')
+  #    collection = stellr.connect('myindex', :fields => { :content => { :store => :yes } })
+  #    collection << { :content => 'lorem ipsum' }
+  #    collection.switch
+  #    results = collection.search('lorem', :page => 1, :per_page => 10)
+  #
+  class Client
+
+    def initialize( drb_uri )
+      @server = DRbObject.new(nil, drb_uri)
+    end
+
+    # connects to a remote collection and returns a stub that can be used to
+    # add records to the collection and to search for them.
+    #
+    # specify an array of collection names for the first argument to search
+    # multiple connections at once. No indexing is possible with such a
+    # MultiCollection.
+    def connect( collection_name, options = nil )
+      if Array === collection_name
+        multi_connect collection_name, options
+      else
+        @server.register collection_name, options
+        ClientCollection.new @server, collection_name
+      end
+    end
+
+    # Connects to multiple remote collections at once on order to run
+    # cross-collection searches.
+    def multi_connect( collection_names, options = {} )
+      MultiCollection.new @server, collection_names, options
+    end
+
+  end
+
+  # Wrapper around a remote collection.
+  #
+  # See the documentation of the collection class you use for more information.
+  class ClientCollection
+
+    def initialize( server, name )
+      @name   = name
+      @server = server
+    end
+
+    # Disconnects this collection from the server.
+    def disconnect
+      @server = nil
+    end
+
+    def method_missing( method, *args )
+      raise "use of disconnected collection" if @server.nil?
+      @server.send method, @name, *args
+    end
+
+  end
+
+  # This client collection class allows to search multiple server side
+  # collections at once.
+  class MultiCollection < ClientCollection
+    def initialize( server, names, options = {} )
+      @server = server
+      @name = @server.register_multi_collection names, options
+    end
+  end
+
+end
+

data/lib/stellr/collections.rb

@@ -0,0 +1,6 @@
+require 'stellr/collections/base'
+require 'stellr/collections/searchable_collection'
+require 'stellr/collections/writeable_collection'
+require 'stellr/collections/static'
+require 'stellr/collections/rsync'
+require 'stellr/collections/multi_collection'

data/lib/stellr/collections/base.rb

@@ -0,0 +1,79 @@
+module Stellr
+  module Collections
+
+    # Base class for collection implementations
+    class Base
+      include Ferret::Index
+      include Stellr::Utils::Shutdown
+      include Stellr::Utils::Observable
+      attr_reader :name
+
+      def self.create( name, options )
+        collection_class = collection_class_for_options options
+        collection       = collection_class.new( name, options )
+        if strategy_class = strategy_class_for_options( options )
+          strategy_class.new( collection, options )
+        else
+          collection
+        end
+      end
+
+      def initialize( name, options )
+        @logger = options[:logger] || (require 'logger'; Logger.new 'stellr.log')
+        @name = name
+        @options = options.dup
+      end
+
+      # called whenever the strategy thinks it's a good time do do something
+      # timeconsuming (like switching indexes, optimizing, flushing, ...)
+      def batch_finished
+      end
+
+      def on_shutdown( mode )
+        close
+      end
+
+      # close this collection
+      def close
+      end
+
+    protected
+      
+      
+      def collection_directory
+        @options[:path]
+      end
+      
+
+      def self.collection_class_for_options( options )
+        if (c = options.delete(:collection))
+          options[:collection_class] = collection_class_for_key c
+        end
+        Object.module_eval("::#{options[:collection_class] || 'Stellr::Collections::RSync'}", __FILE__, __LINE__)
+      end
+      def self.collection_class_for_key(key)
+        case key
+        when :static
+          'Stellr::Collections::Static'
+        when :rsync
+          'Stellr::Collections::RSync'
+        end
+      end
+
+      def self.strategy_class_for_options( options )
+        if (c = options.delete(:strategy))
+          options[:strategy_class] = strategy_class_for_key c
+        end
+        Object.module_eval("::#{options[:strategy_class]}", __FILE__, __LINE__) if options[:strategy_class]
+      end
+      def self.strategy_class_for_key(key)
+        case key
+        when :queueing
+          'Stellr::Strategies::Queueing'
+        end
+      end
+    end
+
+  end
+
+end