stellr 0.1.0 → 0.1.1

data/History.txt

@@ -1,3 +1,7 @@
+== 0.1.1 / 2008-08-06
+
+* Add support for bulk indexing of multiple records
+
 == 0.1.0 / 2008-08-05
 
 * Initial release

data/README.txt

@@ -87,7 +87,7 @@ Stellr is a Ferret based standalone search server featuring a DRB and (soon to c
 
 (The MIT License)
 
-Copyright (c) 2007 FIX
+Copyright (c) 2008 Jens Kraemer, Benjamin Krause
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/stellr.rb

@@ -18,7 +18,7 @@ require 'stellr/search'
 $SAFE = 1
 
 module Stellr
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
   
   def self.start_server( config )
     if config.script

data/lib/stellr/collections/base.rb

@@ -6,12 +6,14 @@ module Stellr
       include Ferret::Index
       include Stellr::Utils::Shutdown
       include Stellr::Utils::Observable
-      attr_reader :name
+      attr_reader :name, :logger
 
       def self.create( name, options )
+        log = (options[:logger] ||= (require 'logger'; Logger.new 'stellr.log'))
         collection_class = collection_class_for_options options
         collection       = collection_class.new( name, options )
         if strategy_class = strategy_class_for_options( options )
+          log.debug "using strategy #{strategy_class}"
           strategy_class.new( collection, options )
         else
           collection
@@ -19,11 +21,11 @@ module Stellr
       end
 
       def initialize( name, options )
-        @logger = options[:logger] || (require 'logger'; Logger.new 'stellr.log')
+        @logger = options[:logger]
         @name = name
         @options = options.dup
       end
-
+      
       # called whenever the strategy thinks it's a good time do do something
       # timeconsuming (like switching indexes, optimizing, flushing, ...)
       def batch_finished

data/lib/stellr/collections/rsync.rb

@@ -29,7 +29,9 @@ module Stellr
       end
 
       def sync_indexes
+        logger.debug "syncing #{searching_directory} to #{indexing_directory} ..."
         system("rsync -r --delete #{searching_directory}/ #{indexing_directory}")
+        logger.debug "done."
       end
 
     end

data/lib/stellr/collections/writeable_collection.rb

@@ -15,32 +15,39 @@ module Stellr
       #
       # Record may be a hash, or a Ferret::Document instance
       def add_record( record, boost = nil )
-        raise ArgumentError.new("record must contain :id field") if record[:id].nil?
-        if boost
-          if Ferret::Document === record
-            record.boost = boost
-          else
-            hash, record = record, Ferret::Document.new( boost )
-            hash.each_pair do |k,v|
-              record[k] = v
-            end
-          end
+        add_records [ [ record, boost ] ]
+      end
+      alias :<< :add_record
+      
+      # adds multiple records at once
+      # records should be an array of hashes or of two-element arrays
+      # consisting of a hash and the record-specific boost value
+      def add_records(records)
+        return unless records.any?
+        records = if Hash === records.first
+          records.map{ |r| prepare_document r }
+        else
+          records.map{ |r, boost| prepare_document(r, boost) }
         end
         @writer_monitor.synchronize do
-          @processed_records += 1
           w = writer
-          w.delete :id, record[:id].to_s # ensure uniqueness by :id field
-          w << record
+          records.each do |record|
+            @processed_records += 1
+            w.delete :id, record[:id].to_s # ensure uniqueness by :id field
+            w << record
+          end
+          w.commit
         end
         true
       end
-      alias :<< :add_record
 
       def delete_record( record )
         raise ArgumentError.new("record must contain :id field") if record[:id].nil?
         @writer_monitor.synchronize do
+          w = writer
           @processed_records += 1
-          writer.delete :id, record[:id].to_s
+          w.delete :id, record[:id].to_s
+          w.commit
         end
         true
       end
@@ -80,6 +87,21 @@ module Stellr
       end
 
     protected
+
+      def prepare_document(record, boost = nil)
+        raise ArgumentError.new("record must contain :id field") if record[:id].nil?
+        unless boost.nil?
+          if Ferret::Document === record
+            record.boost = boost
+          else
+            hash, record = record, Ferret::Document.new( boost )
+            hash.each_pair do |k,v|
+              record[k] = v
+            end
+          end
+        end
+        return record
+      end
       
       # should open a writer and return it
       def open_writer

data/lib/stellr/server.rb

@@ -68,8 +68,12 @@ module Stellr
     #
     #
     def register( name, options = {} )
+      raise "invalid collection name >#{name}<, may only contain a-zA-Z0-9_-" unless name =~ /^([a-zA-Z0-9_-]+)$/
+      name.untaint
       @collections.synchronize do
-        @collections[name] ||= create_collection( name, options )
+        collection = (@collections[name] ||= create_collection( name, options ))
+        save_collection_config name, options unless options.nil? or options.empty?
+        collection
       end
     end
 
@@ -117,24 +121,19 @@ module Stellr
     # if nil is given for options, the method tries to locate a previously
     # saved collection configuration and restore from it.
     def create_collection( name, options )
-      raise "invalid collection name >#{name}<, may only contain a-zA-Z0-9_-" unless name =~ /^([a-zA-Z0-9_-]+)$/
-      name.untaint
-      save_config = true
-      if options.nil?
-        options = load_collection_config name
-        save_config = false
-      end
+      options ||= load_collection_config name
       raise "No options given for collection #{name} and no stored configuration found." if options.nil?
 
       options[:path] = File.join( @config.data_dir, name )
-      save_collection_config name, options if save_config
       return Collections::Base.create( name, {:logger => @logger}.merge(options) )
     end
 
+    # TODO move into collection?
     def save_collection_config( name, options )
       path = collection_config_path name
       ( File.open(path, 'w') << YAML.dump(options) ).close
       @logger.info "wrote collection config to #{path}"
+      @logger.debug "config is now:\n#{options.inspect}"
     end
 
     def load_collection_config( name )

data/lib/stellr/strategies/queueing.rb

@@ -4,7 +4,21 @@ module Stellr
 
     # Queueing strategy. Any index modifying methods return immediately, actions 
     # are queued and executed asynchronously in order of arrival.
-    # 
+    #
+    # Unless you're using the static collection type, indexes will be switched
+    # whenever options[:max_batch_size] (which defaults to 200) is reached,
+    # and when the queue is empty.
+    #
+    # with static collections manually calling switch is required, and this call
+    # will block until the switch is actually done. 
+    #
+    # However this does not mean that
+    # all records from the queue have been processed, the switch may also occur
+    # between processing of add_record or add_records calls.
+    #
+    # FIXME fix this: switch should be an operation that is enqueued just like
+    # add_record so it occurs at the point in time the client desires.
+    # Is implicit switching really that useful? Definitely not with static collections...
     class Queueing < Base
 
       def initialize( collection, options )
@@ -18,6 +32,10 @@ module Stellr
         enqueue :add, [record, boost]
       end
       
+      def add_records(records)
+        enqueue :bulk_add, records
+      end
+      
       def delete_record( record )
         enqueue :delete, record
       end
@@ -47,12 +65,15 @@ module Stellr
       end
 
       # process a single task from the queue
-      def process_record( action, record )
+      # TODO refacoring: rename to process_task
+      def process_record( action, data )
         case action
           when :add
-            @collection.add_record( *record )
+            @collection.add_record( *data )
+          when :bulk_add
+            @collection.add_records data
           when :delete
-            @collection.delete_record record
+            @collection.delete_record data
           else
             raise "UnknownAction"
         end

data/test/test_rsync_collection.rb

@@ -10,7 +10,7 @@ class RSyncCollectionTest < StellrTest
 
   def teardown
     super
-    @collection.close
+    @collection.close if @collection
   end
 
   def test_create
@@ -67,6 +67,7 @@ class RSyncCollectionTest < StellrTest
   
   def default_collection_options( options = {} )
     { :recreate => false,
-      :path     => INDEX_TMP_TEST_DIR }.update( options )
+      :path     => INDEX_TMP_TEST_DIR,
+      :logger => Logger.new('/tmp/stellr/test.log') }.update( options )
   end
 end

data/test/test_server.rb

@@ -56,6 +56,24 @@ class ServerTest < StellrTest
     assert_equal 2, @server.size( 'default' )
   end
   
+  def test_index_multiple_records_arrays
+    @server.register 'default'
+    @server.add_records 'default', 
+                        [ [ { :id => 1, :text => 'hello world' }, 2],
+                          [ { :id => 2, :text => 'hello world two' }, nil ] ]
+    @server.batch_finished 'default'
+    assert_equal 2, @server.size( 'default' )
+  end
+  
+  def test_index_multiple_records_hashes
+    @server.register 'default'
+    @server.add_records 'default', 
+                        [ { :id => 1, :text => 'hello world' },
+                          { :id => 2, :text => 'hello world two' } ]
+    @server.batch_finished 'default'
+    assert_equal 2, @server.size( 'default' )
+  end
+  
   def test_delete_data_queued
     coll = "del-queued"
     @server.register coll, :strategy => :queueing

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: stellr
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - Benjamin Krause
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-08-04 00:00:00 +02:00
+date: 2008-08-13 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency