stellr 0.1.0

data/lib/stellr/config.rb

@@ -0,0 +1,88 @@
+require 'yaml'
+require 'erb'
+
+module Stellr
+  # = Config
+  #
+  # The +stellr+ and +stellr-search+ commands both rely on a YAML file
+  # to provide some basic configuration:
+  #
+  # +:port+      Port the server should listen to
+  # +:host+      Hostname or IP of the server
+  # +:script+    Optional ruby file to load during startup. This is the place to load custom code like self made Analyzers you intend to use.
+  # +:log_level+ Log level, default is +:debug+
+  # +:base_dir+  Base directory where the server will store index data, log
+  #              files and configuration data.
+  # +:data_dir+  Index directory, relative to +base_dir+. Defaults to +data+
+  # +:log_dir+   Log file directory, relative to +base_dir+. Defaults to +log+
+  # +:conf_dir+  Configuration directory, relative to +base_dir+. Defaults to +conf+. Here stellr will keep the configuration of registered collections, one YAML file per collection.
+  # +:tmp_dir+   Temp directory, relative to +base_dir+. Defaults to +tmp+
+  #
+  #
+  class Config
+    DEFAULTS = { :port           => 9010,
+                 :host           => 'localhost',
+                 :base_dir       => '/var/stellr',
+                 :data_dir       => 'data',
+                 :log_dir        => 'log',
+                 :tmp_dir        => 'tmp',
+                 :conf_dir       => 'conf',
+                 :log_level      => :warn }
+                 
+                 
+    # Configfile search order:
+    #  - argument
+    #  - /etc/stellr.yml
+    #  - +gem_directory+/config/stellr.yml 
+    #
+    def initialize( config_file = nil, extra_options = {} )
+      load_config config_file
+      @config.update extra_options
+      @config.each { |k,v| v.untaint } # we trust our config file
+    end
+    
+    def data_dir
+      resolve_directory_name( :data_dir )
+    end
+    
+    def log_dir
+      resolve_directory_name( :log_dir )
+    end
+    
+    def tmp_dir
+      resolve_directory_name( :tmp_dir ) 
+    end
+    
+    def conf_dir
+      resolve_directory_name( :conf_dir ) 
+    end
+    
+    def collection_dir
+    end
+
+    def drb_uri
+      "druby://#{host}:#{port}"
+    end
+    
+    protected
+
+    def method_missing( method_name, *args )
+      return @config[method_name] if @config.has_key?( method_name )
+      raise NameError.new( "unknown configuration key: #{method_name}" )
+    end
+
+    def resolve_directory_name( sub_dir )
+      raise NameError.new unless @config.has_key?( sub_dir )
+      File.join( base_dir, @config[sub_dir] )      
+    end
+            
+    def load_config( config_file )
+      config_file ||= "/etc/stellr.yml"
+      config_file = File.join( File.dirname(__FILE__), "../../config/stellr.yml" ) unless File.exists?( config_file )
+      
+      @config = DEFAULTS.merge(
+                  YAML.load( ERB.new( IO.read(config_file) ).result )
+                )
+    end
+  end
+end

data/lib/stellr/search.rb

@@ -0,0 +1,2 @@
+require 'stellr/search/search_result'
+require 'stellr/search/search_results'

data/lib/stellr/search/search_result.rb

@@ -0,0 +1,21 @@
+module Stellr
+  module Search
+  
+    # A single search result
+    class SearchResult
+      attr_reader :score, :doc_id
+
+      def initialize(id, score, field_data)
+        @doc_id = id
+        @score = score
+        @field_data = field_data
+      end
+
+      # retrieve contents of the field +name+
+      def field(name)
+        @field_data[name]
+      end
+      alias [] field
+    end
+  end
+end

data/lib/stellr/search/search_results.rb

@@ -0,0 +1,50 @@
+module Stellr
+  module Search
+
+    # Thin wrapper around an array of search results
+    class SearchResults
+      attr_accessor :total_hits, :current_page, :per_page
+      alias total_entries total_hits
+
+      def initialize
+        @results = []
+      end
+
+      def method_missing(symbol, *args, &block)
+        @results.send(symbol, *args, &block)
+      end
+
+      def respond_to?(name)
+        self.methods.include?(name) || @results.respond_to?(name)
+      end
+
+
+      # code from here on derived from will_paginate's collection.rb
+
+      # The total number of pages.
+      def total_pages
+        @total_pages ||= per_page > 0 ? (total_hits / per_page.to_f).ceil : 0
+      end
+      alias page_count total_pages
+
+      # Current offset of the paginated collection. If we're on the first page,
+      # it is always 0. If we're on the 2nd page and there are 30 entries per page,
+      # the offset is 30. This property is useful if you want to render ordinals
+      # besides your records: simply start with offset + 1.
+      #
+      def offset
+        (current_page - 1) * per_page
+      end
+
+      # current_page - 1 or nil if there is no previous page
+      def previous_page
+        current_page > 1 ? (current_page - 1) : nil
+      end
+
+      # current_page + 1 or nil if there is no next page
+      def next_page
+        current_page < page_count ? (current_page + 1) : nil
+      end
+    end
+  end
+end

data/lib/stellr/server.rb

@@ -0,0 +1,166 @@
+require 'logger'
+
+module Stellr
+  class Server
+    include Utils::Shutdown
+    protected :shutdown
+
+    attr_accessor :config
+    attr_reader   :collections
+
+    def initialize( config )
+      @config = config
+      create_directories
+      @logger = Logger.new File.join(config.log_dir, 'stellr.log')
+      @logger.level = Logger.const_get config.log_level.to_s.upcase
+      @collections = {}
+      @collections.extend MonitorMixin
+    end
+    
+    def add_record( collection_name, record, boost = nil )
+      collection( collection_name ).add_record record, boost
+    end
+    alias :<< :add_record
+    
+    def delete_record( collection_name, record )
+      collection( collection_name ).delete_record record
+    end
+    
+    def size( collection_name )
+      collection( collection_name ).size
+    end
+
+    # Initialize a collection.
+    #
+    # Before anything can be done with a collection, it has to be registered
+    # through this method. When called for a non-existing collection, this will
+    # also create the empty physical index. The given options are saved to a yml
+    # file so they can be loaded again later on.
+    #
+    # Calling register for an existing collection will update the saved index
+    # configuration from the options given, unless the options hash is nil.
+    # An already existing physical index won't be touched and used as is.
+    # Remember that changing Ferret options like analyzers or field
+    # configuration requires an index rebuild to be effective for existing
+    # content. Stellr doesn't take care of this.
+    #
+    # If you access the server through the Stellr::Client class you don't need
+    # to call +register+ explicitly as the client will do it when connecting.
+    #
+    # Name is the name of the collection to create.
+    #
+    # Options is a hash, consisting of:
+    # [+collection+]  The collection implementation to use, may be one of :static or :rsync (default)
+    # [+strategy+]    The strategy implementation to use (optional, atm there is only :queueing)
+    # [+fields+]            +hash+ (see http://ferret.davebalmain.com/api/classes/Ferret/Index/FieldInfo.html)
+    # [+recreate+]          Recreate the index (defaults to +false+). A true value will lead to the deletion of any already indexed data.
+    # [+analyzer+]          The class name (String) of the Analyzer to use for this collection. By default, Ferret's StandardAnalyzer will be used. 
+    # [+field_defaults+] Default setting for unconfigured fields
+    #
+    # Example
+    #   register 'standard_index', { :recreate => false,
+    #                                :fields   =>  { :author  => { :index       => :untokenized,
+    #                                                  :store       => :no,
+    #                                                  :term_vector => :with_offsets,
+    #                                                  :boost       => 2.0 },
+    #                                                :content => { :index       => :tokenized } }
+    #                              }
+    #
+    #
+    def register( name, options = {} )
+      @collections.synchronize do
+        @collections[name] ||= create_collection( name, options )
+      end
+    end
+
+    # Initializes a read-only virtual collection that may be used to search
+    # across multiple physical collections.
+    #
+    # Returns the name of the collection to be used with further calls.
+    def register_multi_collection( names, options = {} )
+      key = ".multi_#{names.join '_'}" # '.' is not allowed for regular collection names, so we are safe from name collisions
+      @collections.synchronize do
+        @collections[key] ||= create_multi_collection( key, names.map{ |name| collection(name) }, options )
+      end
+      return key
+    end
+
+    def collection( name )
+      @collections.synchronize do
+        if @collections.has_key?( name )
+          return @collections[name] 
+        else
+          @logger.info "trying to initialize collection #{name} from stored configuration..."
+          return @collections[name] = create_collection( name, nil )
+        end
+      end
+      raise "UnknownCollection #{name}"
+    end
+
+    protected
+    
+    # pass through commands to collection
+    def method_missing(method, *args)
+      if args.size >= 1
+        collection_name = args.shift
+        return collection( collection_name ).send( method, *args )
+      end
+      super
+    end
+
+    def create_multi_collection( name, collections, options = {} )
+      Stellr::Collections::MultiCollection.new name, collections, { :logger => @logger }.merge(options)
+    end
+
+    # initializes a new collection object
+    #
+    # if nil is given for options, the method tries to locate a previously
+    # saved collection configuration and restore from it.
+    def create_collection( name, options )
+      raise "invalid collection name >#{name}<, may only contain a-zA-Z0-9_-" unless name =~ /^([a-zA-Z0-9_-]+)$/
+      name.untaint
+      save_config = true
+      if options.nil?
+        options = load_collection_config name
+        save_config = false
+      end
+      raise "No options given for collection #{name} and no stored configuration found." if options.nil?
+
+      options[:path] = File.join( @config.data_dir, name )
+      save_collection_config name, options if save_config
+      return Collections::Base.create( name, {:logger => @logger}.merge(options) )
+    end
+
+    def save_collection_config( name, options )
+      path = collection_config_path name
+      ( File.open(path, 'w') << YAML.dump(options) ).close
+      @logger.info "wrote collection config to #{path}"
+    end
+
+    def load_collection_config( name )
+      path = collection_config_path name
+      conf = YAML.load( File.read(path) ) if File.readable?(path) rescue nil
+      @logger.info "loaded collection config from #{path}" unless conf.nil?
+      return conf
+    end
+
+    def collection_config_path( name )
+      File.join @config.conf_dir, "#{name}.yml"
+    end
+
+    # called by shutdown
+    def on_shutdown( mode )
+      @collections.synchronize do
+        @collections.values.each { |coll| coll.shutdown mode }
+      end
+    end
+
+    def create_directories
+      FileUtils.mkdir_p config.log_dir
+      FileUtils.mkdir_p config.tmp_dir
+      FileUtils.mkdir_p config.conf_dir
+      FileUtils.mkdir_p config.data_dir
+    end
+
+  end
+end

data/lib/stellr/strategies.rb

@@ -0,0 +1,4 @@
+require 'stellr/strategies/base'
+require 'stellr/strategies/blocking'
+require 'stellr/strategies/queueing'
+

data/lib/stellr/strategies/base.rb

@@ -0,0 +1,16 @@
+module Stellr
+  module Strategies
+    class Base
+      include Stellr::Utils::Shutdown
+
+      def initialize( collection, options )
+        @collection = collection
+        @options = options.dup
+      end
+
+      def method_missing(name, *args)
+        @collection.send name, *args
+      end
+    end
+  end
+end

data/lib/stellr/strategies/blocking.rb

@@ -0,0 +1,13 @@
+module Stellr
+
+  module Strategies
+
+    # Blocking strategy. Any index modifying methods will block until the change has
+    # been done.
+    class Blocking < Base
+
+    end
+
+  end
+end
+

data/lib/stellr/strategies/queueing.rb

@@ -0,0 +1,78 @@
+module Stellr
+
+  module Strategies
+
+    # Queueing strategy. Any index modifying methods return immediately, actions 
+    # are queued and executed asynchronously in order of arrival.
+    # 
+    class Queueing < Base
+
+      def initialize( collection, options )
+        super collection, options
+        @options[:max_batch_size] ||= 200
+        @queue = Queue.new
+        @thread = spawn_indexing_thread
+      end
+
+      def add_record( record, boost = nil )
+        enqueue :add, [record, boost]
+      end
+      
+      def delete_record( record )
+        enqueue :delete, record
+      end
+
+      protected
+
+      def on_shutdown( mode )
+        @queue << 'shutting down' # letztes queue item damit process_queue nicht haengt
+        @thread.join
+        # save_queue :TODO:
+        @collection.shutdown mode
+      end
+
+      # called by the indexer thread as long as the server runs
+      def process_queue
+        counter = 0
+        max_batch_size = @options[:max_batch_size]
+        begin
+          while record = @queue.deq and not shutting_down?( :abort )
+            process_record( *record )
+            break if ((counter += 1) > max_batch_size) or @queue.empty?
+          end
+          @collection.batch_finished
+        rescue Exception => e
+          puts "OH NO! #{e}\n#{e.backtrace.join "\n"}"
+        end
+      end
+
+      # process a single task from the queue
+      def process_record( action, record )
+        case action
+          when :add
+            @collection.add_record( *record )
+          when :delete
+            @collection.delete_record record
+          else
+            raise "UnknownAction"
+        end
+      end
+
+      # Spawns the thread executing the main loop
+      def spawn_indexing_thread
+        Thread.new do
+          process_queue while !shutting_down?
+        end
+      end
+
+      # add a task to the queue
+      def enqueue( action, record )
+        return false if shutting_down?
+        @queue << [ action, record ]
+        true
+      end
+
+    end
+
+  end
+end