picky 0.0.5 → 0.0.6

data/lib/bundling.rb

@@ -4,5 +4,5 @@ rescue LoadError => e
   require 'rubygems'
   require 'bundler'
 end
-Bundler.setup SEARCH_ENVIRONMENT
+Bundler.setup PICKY_ENVIRONMENT
 Bundler.require

data/lib/constants.rb

@@ -3,7 +3,7 @@
 
 # Use rack's environment for the search engine.
 #
-ENV['SEARCH_ENV'] ||= ENV['RACK_ENV']
+ENV['PICKY_ENV'] ||= ENV['RACK_ENV']
 
-SEARCH_ENVIRONMENT = ENV['SEARCH_ENV'] || 'development' unless defined? SEARCH_ENVIRONMENT
-SEARCH_ROOT        = Dir.pwd unless defined? SEARCH_ROOT
+PICKY_ENVIRONMENT = ENV['PICKY_ENV'] || 'development' unless defined? PICKY_ENVIRONMENT
+PICKY_ROOT        = Dir.pwd unless defined? PICKY_ROOT

data/lib/deployment.rb

@@ -13,12 +13,12 @@ module Picky
           # Executes a rake task on the server.
           #
           # Options:
-          #  * env: The SEARCH_ENV. Will not set if set explicitly to false. Default: production.
+          #  * env: The PICKY_ENV. Will not set if set explicitly to false. Default: production.
           #  * All other options get passed on to the Capistrano run task.
           #
           def execute_rake_task name, options = {}, &block
             env = options.delete :env
-            env = env == false ? '' : "SEARCH_ENV=#{env || 'production'}"
+            env = env == false ? '' : "PICKY_ENV=#{env || 'production'}"
             run "cd #{current_path}; rake #{name} #{env}", options, &block
           end
           

data/lib/picky/configuration/field.rb

@@ -33,12 +33,12 @@ module Configuration
     # TODO Move to type, and use in bundle from there.
     #
     def search_index_root
-      File.join SEARCH_ROOT, 'index'
+      File.join PICKY_ROOT, 'index'
     end
     # TODO Move to config. Duplicate Code in field.rb.
     #
     def cache_directory
-      File.join search_index_root, SEARCH_ENVIRONMENT, type.name.to_s
+      File.join search_index_root, PICKY_ENVIRONMENT, type.name.to_s
     end
     def search_index_file_name
       File.join cache_directory, "#{type.name}_#{name}_index.txt"

data/lib/picky/configuration/type.rb

@@ -45,5 +45,10 @@ module Configuration
       @indexer = Indexers::Solr.new self
       @indexer.index
     end
+    # TODO Spec!
+    #
+    def connect_backend
+      @source.connect_backend
+    end
   end
 end

data/lib/picky/index/bundle.rb

@@ -56,7 +56,7 @@ module Index
     # Point to category.
     #
     def search_index_root
-      File.join SEARCH_ROOT, 'index'
+      File.join PICKY_ROOT, 'index'
       # category.search_index_root
     end
     
@@ -145,7 +145,7 @@ module Index
     # TODO Move to config. Duplicate Code in field.rb.
     #
     def cache_directory
-      File.join search_index_root, SEARCH_ENVIRONMENT, type.name.to_s
+      File.join search_index_root, PICKY_ENVIRONMENT, type.name.to_s
     end
 
     # Generates a cache path.

data/lib/picky/indexers/base.rb

@@ -44,8 +44,10 @@ module Indexers
       comma   = ?,
       newline = ?\n
       
+      indexing_message
+      
       File.open(search_index_file_name, 'w:binary') do |file|
-        chunked do |indexed_id, text|
+        source.harvest(@type, @field) do |indexed_id, text|
           tokenizer.tokenize(text).each do |token_text|
             file.write indexed_id
             file.write comma
@@ -55,22 +57,9 @@ module Indexers
         end
       end
     end
-    # Split original data into chunks.
-    #
-    def chunked
-      (0..source.count(@type)).step(chunksize) do |offset|
-        indexing_message offset
-        data = source.harvest @type, @field, offset, chunksize
-        data.each do |indexed_id, text|
-          next unless text
-          text.force_encoding 'utf-8' # TODO Still needed?
-          yield indexed_id, text
-        end
-      end
-    end
     
-    def indexing_message offset
-      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name} beginning at #{offset}."
+    def indexing_message
+      puts "#{Time.now}: Indexing #{@type.name}:#{@field.name}:#{@field.indexed_name}."
     end
     
   end

data/lib/picky/indexers/field.rb

@@ -2,12 +2,8 @@ module Indexers
   # Base indexer for fields.
   #
   class Field < Base
-
-    # Override in subclasses.
-    #
-    def chunksize
-      25_000
-    end
+    
+    # TODO Still needed?
     
   end
 end

data/lib/picky/indexes.rb

@@ -20,7 +20,7 @@ module Indexes
     Cores.forked self.fields, :randomly => true do |field|
       # Reestablish DB connection.
       #
-      DB.connect # TODO Rewrite!
+      connect_backends
       field.index
       field.cache
     end
@@ -29,6 +29,14 @@ module Indexes
     configuration.index_solr
   end
   
+  # TODO Push into configuration.
+  #
+  def self.connect_backends
+    configuration.types.each do |type|
+      type.connect_backend
+    end
+  end
+  
   # Returns an array of fields.
   #
   # TODO Rewrite.

data/lib/picky/loader.rb

@@ -6,7 +6,7 @@ module Loader
   # First itself, then the app.
   #
   def self.reload
-    Dir.chdir(SEARCH_ROOT)
+    Dir.chdir(PICKY_ROOT)
     exclaim 'Reloading loader.'
     load_self
     exclaim 'Reloading framework.'
@@ -30,13 +30,13 @@ module Loader
   end
   
   def self.load_user filename
-    load File.join(SEARCH_ROOT, "#{filename}.rb")
+    load File.join(PICKY_ROOT, "#{filename}.rb")
   end
   def self.load_user_lib filename
     load_user File.join('lib', filename)
   end
   def self.load_all_user_in dirname
-    Dir[File.join(SEARCH_ROOT, dirname, '**', '*.rb')].each do |filename|
+    Dir[File.join(PICKY_ROOT, dirname, '**', '*.rb')].each do |filename|
       load filename
     end
   end
@@ -203,10 +203,7 @@ module Loader
     #
     load_relative 'sources/base'
     load_relative 'sources/db'
-    
-    # DB
-    #
-    load_relative 'db/configuration'
+    load_relative 'sources/csv'
     
     # Indexes.
     #

data/lib/picky/query/token.rb

@@ -178,13 +178,13 @@ module Query
     def to_solr
       blank? ? '' : (to_s + @@solr_fuzzy_mapping[@text.size].to_s)
     end
-
+    
     #
     #
     def to_result
       [@original, @text]
     end
-
+    
     # Displays the qualifier text and the text, joined.
     #
     # e.g. name:meier
@@ -192,9 +192,9 @@ module Query
     def to_s
       [@qualifier, @text].compact.join ':'
     end
-
+    
     private
-
+      
       # Splits text into a qualifier and text.
       #
       # Returns [qualifier, text].

data/lib/picky/solr/schema_generator.rb

@@ -52,7 +52,7 @@ module Solr
     #
     #
     def read_template
-      template_path = File.join SEARCH_ROOT, 'solr', 'conf', 'schema.xml.erb'
+      template_path = File.join PICKY_ROOT, 'solr', 'conf', 'schema.xml.erb'
       schema = ''
       File.open(template_path, 'r') do |f|
         schema = f.read
@@ -63,7 +63,7 @@ module Solr
     #
     #
     def write result
-      schema_path = File.join SEARCH_ROOT, 'solr', 'conf', 'schema.xml'
+      schema_path = File.join PICKY_ROOT, 'solr', 'conf', 'schema.xml'
       File.open(schema_path, 'w') do |f|
         f << result
       end

data/lib/picky/sources/base.rb

@@ -2,15 +2,15 @@ module Sources
   
   class Base
     
-    def take_snapshot type
+    def connect_backend
       
     end
     
-    def count type
+    def take_snapshot type
       
     end
     
-    def harvest offset
+    def harvest type, field
       
     end
     

data/lib/picky/sources/csv.rb

@@ -1,28 +1,41 @@
+require 'CSV'
+
 module Sources
   
+  class NoCSVFileGiven < StandardError; end
+  
   class CSV < Base
     
-    attr_reader :file_name
+    attr_reader :file_name, :field_names
+    
+    def initialize *field_names, options
+      @field_names = field_names
+      @file_name   = Hash === options && options[:file] || raise_no_file_given(field_names)
+    end
     
-    def initialize file_name, *field_names
-      @file_name = file_name
-      @field_names
+    #
+    #
+    def raise_no_file_given field_names
+      raise NoCSVFileGiven.new field_names.join(', ')
     end
     
-    # Counts all the entries that are used for the index.
+    # Harvests the data to index.
     #
-    def count type
-      `wc -l #{file_name}`
+    def harvest _, field
+      index = field_names.index field.name
+      get_data do |ary|
+        indexed_id = ary.shift.to_i
+        text       = ary[index]
+        next unless text
+        text.force_encoding 'utf-8' # TODO Still needed?
+        yield indexed_id, text
+      end
     end
     
-    # Harvests the data to index, chunked.
     #
-    # Subclasses should override harvest_statement to define how their data is found.
-    # Example:
-    #   "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
     #
-    def harvest offset
-      File.open file_name, 'r'
+    def get_data &block
+      ::CSV.foreach file_name, &block
     end
     
   end

data/lib/picky/sources/db.rb

@@ -2,17 +2,57 @@ module Sources
   
   class DB < Base
     
-    attr_reader :select_statement, :database
+    attr_reader :select_statement, :database, :connection_options
     
-    def initialize select_statement, database_adapter
+    def initialize select_statement, with_options = { :file => 'app/db.yml' }
       @select_statement = select_statement
-      @database         = database_adapter
+      @database         = create_database_adapter
+      configure with_options
+    end
+    
+    # Get a configured Database backend.
+    #
+    # Options:
+    #  Either
+    #  * file => 'some/filename.yml' # With an active record configuration.
+    #  Or
+    #  * The configuration as a hash.
+    #
+    def create_database_adapter
+      adapter_class = Class.new ActiveRecord::Base
+      adapter_class.abstract_class = true
+      adapter_class
+    end
+    
+    # Configure the backend.
+    #
+    # Options:
+    #  Either
+    #  * file => 'some/filename.yml' # With an active record configuration.
+    #  Or
+    #  * The configuration as a hash.
+    #
+    def configure options
+      @connection_options = if filename = options[:file]
+        File.open(File.join(PICKY_ROOT, filename)) { |f| YAML::load(f) }
+      else
+        options
+      end
+      self
+    end
+    
+    # Connect the backend.
+    #
+    def connect_backend
+      return if PICKY_ENVIRONMENT.to_s == 'test' # TODO Unclean.
+      raise "Database backend not configured" unless connection_options
+      database.establish_connection connection_options
     end
     
     # Take the snapshot.
     #
     def take_snapshot type
-      database.connect
+      connect_backend
       
       origin = snapshot_table_name type
 
@@ -29,6 +69,8 @@ module Sources
     # Counts all the entries that are used for the index.
     #
     def count type
+      connect_backend
+      
       database.connection.select_value("SELECT COUNT(id) FROM #{snapshot_table_name(type)}").to_i
     end
     
@@ -44,10 +86,28 @@ module Sources
     # Example:
     #   "SELECT indexed_id, value FROM bla_table st WHERE kind = 'bla'"
     #
-    def harvest type, field, offset, chunksize
-      database.connect
+    def harvest type, field
+      connect_backend
       
-      database.connection.execute harvest_statement_with_offset(type, field, offset, chunksize)
+      (0..count(type)).step(chunksize) do |offset|
+        get_data(type, field, offset).each do |indexed_id, text|
+          next unless text
+          text.force_encoding 'utf-8' # TODO Still needed?
+          yield indexed_id, text
+        end
+      end
+    end
+    
+    # Override in subclasses.
+    #
+    def chunksize
+      25_000
+    end
+    
+    # Gets database from the backend.
+    #
+    def get_data type, field, offset
+      database.connection.execute harvest_statement_with_offset(type, field, offset)
     end
     
     # Base harvest statement for dbs.
@@ -60,7 +120,7 @@ module Sources
     #
     # TODO Use the adapter for this.
     #
-    def harvest_statement_with_offset type, field, offset, chunksize
+    def harvest_statement_with_offset type, field, offset
       statement = harvest_statement type, field
       
       if statement.include? 'WHERE'

data/lib/picky.rb

@@ -15,4 +15,4 @@ require File.expand_path(File.join(File.dirname(__FILE__), 'picky', 'loader'))
 # Load the framework
 #
 Loader.load_framework
-puts "Loaded picky with environment '#{SEARCH_ENVIRONMENT}' in #{SEARCH_ROOT} on Ruby #{RUBY_VERSION}."
+puts "Loaded picky with environment '#{PICKY_ENVIRONMENT}' in #{PICKY_ROOT} on Ruby #{RUBY_VERSION}."

data/lib/tasks/server.rake

@@ -3,11 +3,11 @@
 namespace :server do
   
   def chdir_to_root
-    Dir.chdir SEARCH_ROOT
+    Dir.chdir PICKY_ROOT
   end
   
   def current_pid
-    pid = `cat #{File.join(SEARCH_ROOT, 'tmp/pids/unicorn.pid')}`
+    pid = `cat #{File.join(PICKY_ROOT, 'tmp/pids/unicorn.pid')}`
     pid.blank? ? nil : pid.chomp
   end
   
@@ -15,8 +15,8 @@ namespace :server do
   task :start => :framework do
     chdir_to_root
     # Rake::Task[:"solr:start"].invoke # TODO Move to better place.
-    daemonize = SEARCH_ENVIRONMENT == 'production' ? '-D' : ''
-    command = "export SEARCH_ENV=#{SEARCH_ENVIRONMENT}; unicorn -c unicorn.ru #{daemonize}".strip
+    daemonize = PICKY_ENVIRONMENT == 'production' ? '-D' : ''
+    command = "export PICKY_ENV=#{PICKY_ENVIRONMENT}; unicorn -c unicorn.ru #{daemonize}".strip
     puts "Running \`#{command}\`."
     exec command
   end

data/lib/tasks/solr.rake

@@ -18,7 +18,7 @@ namespace :solr do
   
   
   def action name
-    `sunspot-solr #{name} --solr-home=solr --data-directory=index/#{SEARCH_ENVIRONMENT}/solr --pid-dir=solr/pids --log-file=log/solr.log`
+    `sunspot-solr #{name} --solr-home=solr --data-directory=index/#{PICKY_ENVIRONMENT}/solr --pid-dir=solr/pids --log-file=log/solr.log`
   end
   task :start => :application do
     Rake::Task['solr:schema:generate'].invoke

data/lib/tasks/statistics.rake

@@ -2,12 +2,12 @@ namespace :statistics do
 
   desc "start the server"
   task :start => :application do
-    Statistics.start unless SEARCH_ENVIRONMENT == 'test'
+    Statistics.start unless PICKY_ENVIRONMENT == 'test'
   end
   
   desc "stop the server"
   task :stop => :application do
-    Statistics.stop unless SEARCH_ENVIRONMENT == 'test'
+    Statistics.stop unless PICKY_ENVIRONMENT == 'test'
   end
   
 end

data/lib/tasks/try.rake

@@ -15,7 +15,9 @@ namespace :try do
   task :query, [:text] => :application do |_, options|
     text = options.text
     
-    puts "\"#{text}\" is query tokenized as #{Tokenizers::Query.new.tokenize(text).to_a}"
+    # TODO tokenize destroys the original text...
+    #
+    puts "\"#{text}\" is query tokenized as #{Tokenizers::Query.new.tokenize(text.dup).to_a.map(&:to_s)}"
   end
   
   desc "Try the given text with both the index and the query (type:field optional)."

data/prototype_project/app/application.rb

@@ -11,7 +11,7 @@ class PickySearch < Application # The App Constant needs to be identical in appl
   # Note: Much more is possible, but let's start out easy.
   #
   # Ask me if you have questions!
-  #  
+  #
   
   indexes do
     illegal_characters(/[^äöüa-zA-Z0-9\s\/\-\"\&\.]/)
@@ -19,10 +19,7 @@ class PickySearch < Application # The App Constant needs to be identical in appl
     split_text_on(/[\s\/\-\"\&\.]/)
       
     type :books,
-         Sources::DB.new(
-           'SELECT id, title, author, isbn13 as isbn FROM books',
-           DB.configure(:file => 'app/db.yml')
-         ),
+         Sources::DB.new('SELECT id, title, author, isbn13 as isbn FROM books', :file => 'app/db.yml'),
          field(:title,  :qualifiers => [:t, :title, :titre], :similarity => Similarity::DoubleLevenshtone.new(3)), # Up to three similar title word indexed.
          field(:author, :qualifiers => [:s, :author, :auteur]),
          field(:isbn,   :qualifiers => [:i, :isbn],          :partial => Partial::None.new) # Partially searching on an ISBN makes not much sense.

data/prototype_project/app/logging.rb

@@ -1,7 +1,7 @@
 # Standard logging.
 #
 require 'logger'
-PickyLog = Loggers::Search.new ::Logger.new(File.expand_path(File.join(SEARCH_ROOT, 'log/search.log')))
+PickyLog = Loggers::Search.new ::Logger.new(File.expand_path(File.join(PICKY_ROOT, 'log/search.log')))
 
 # Example with using the syslog logger.
 # Falling back to the standard log if it isn't available.
@@ -16,5 +16,5 @@ PickyLog = Loggers::Search.new ::Logger.new(File.expand_path(File.join(SEARCH_RO
 # rescue StandardError
 #   puts "Could not connect to the syslog, using the normal log."
 #   require 'logger'
-#   PickyLog        = Loggers::Search.new ::Logger.new(File.join(SEARCH_ROOT, 'log/search.log'))
+#   PickyLog        = Loggers::Search.new ::Logger.new(File.join(PICKY_ROOT, 'log/search.log'))
 # end

data/prototype_project/script/console

@@ -14,12 +14,12 @@ end
 libs =  " -r irb/completion"
 libs << %( -r "picky" )
 
-ENV['SEARCH_ENV'] = case ARGV.first
+ENV['PICKY_ENV'] = case ARGV.first
   when "p"; "production"
   when "d"; "development"
   when "t"; "test"
   else
-    ARGV.first || ENV['SEARCH_ENV'] || 'development'
+    ARGV.first || ENV['PICKY_ENV'] || 'development'
 end
 
 puts "Use \x1b[1;30mLoader.load_application\x1b[m to load app."

data/spec/lib/indexers/field_spec.rb

@@ -3,18 +3,7 @@
 require 'spec_helper'
 
 describe Indexers::Field do
-
-  before(:each) do
-    @type  = stub :type, :name => :some_type, :snapshot_table_name => :some_prepared_table_name
-    @field = stub :field, :indexed_name => :some_indexed_field_name, :name => :some_field_name, :search_index_file_name => :some_index_table
-    @strategy = Indexers::Field.new @type, @field
-    @strategy.stub! :indexing_message
-  end
-
-  describe "chunksize" do
-    it "should be a specific size" do
-      @strategy.chunksize.should == 25_000
-    end
-  end
-
+  
+  
+  
 end

data/spec/lib/sources/csv_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+describe Sources::CSV do
+  
+  context "without file" do
+    it "should fail correctly" do
+      lambda { @source = Sources::CSV.new(:a, :b, :c) }.should raise_error(Sources::NoCSVFileGiven)
+    end
+  end
+  context "with file" do
+    before(:each) do
+      @source = Sources::CSV.new :a, :b, :c, :file => :some_file
+      ::CSV.should_receive(:foreach).any_number_of_times.and_yield ['7', 'a data', 'b data', 'c data']
+    end
+    describe "harvest" do
+      it "should yield the right data" do
+        field = stub :b, :name => :b
+        @source.harvest :anything, field do |id, token|
+          [id, token].should == [7, 'b data']
+        end
+      end
+    end
+    describe "get_data" do
+      it "should yield each line" do
+        @source.get_data do |data|
+          data.should == ['7', 'a data', 'b data', 'c data']
+        end
+      end
+    end
+  end
+  
+end

data/spec/lib/sources/db_spec.rb

@@ -4,12 +4,22 @@ describe Sources::DB do
   
   before(:each) do
     @type       = stub :type, :name => 'some_type_name'
-    @connection = stub :connection
     
+    @connection = stub :connection
     @adapter          = stub :adapter, :connection => @connection
+    
     @select_statement = stub :statement
     
-    @source = Sources::DB.new @select_statement, @adapter
+    @source = Sources::DB.new @select_statement, :option => :some_options
+    
+    @source.stub! :database => @adapter
+    @source.stub! :connect_backend
+  end
+  
+  describe "chunksize" do
+    it "should be a specific size" do
+      @source.chunksize.should == 25_000
+    end
   end
   
   describe "count" do
@@ -26,47 +36,49 @@ describe Sources::DB do
       @source.count @type
     end
   end
-
-  describe "harvest" do
-    before(:each) do
-      @adapter.stub! :connect
-      @source.stub! :harvest_statement_with_offset
-    end
-    context 'expectations' do
-      before(:each) do
-        @connection.stub! :execute
-      end
-      after(:each) do
-        @source.harvest :some_type, :some_field, :some_offset, :some_chunksize
-      end
-      context "with WHERE" do
-        before(:each) do
-          @source.stub! :select_statement => 'bla WHERE blu'
-        end
-        it "should connect" do
-          @adapter.should_receive(:connect).once.with
-        end
-        it "should call the harvest statement with an offset" do
-          @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset, :some_chunksize
-        end
-      end
-      context "without WHERE" do
-        it "should connect" do
-          @adapter.should_receive(:connect).once.with
-        end
-        it "should call the harvest statement with an offset" do
-          @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset, :some_chunksize
-        end
-      end
-    end
-    context 'returns' do
-      it "should return whatever the execute statement returns" do
-        @connection.stub! :execute => :some_result
-        
-        @source.harvest(:some_type, :some_field, :some_offset, :some_chunksize).should == :some_result
-      end
-    end
-  end
+  
+  # TODO Redo.
+  #
+  # describe "harvest" do
+  #   before(:each) do
+  #     @source.stub! :harvest_statement_with_offset
+  #   end
+  #   context 'expectations' do
+  #     before(:each) do
+  #       @connection.stub! :execute => []
+  #       @connection.stub! :select_value
+  #     end
+  #     after(:each) do
+  #       @source.harvest :type_name, :some_field
+  #     end
+  #     context "with WHERE" do
+  #       before(:each) do
+  #         @source.stub! :select_statement => 'bla WHERE blu'
+  #       end
+  #       it "should connect" do
+  #         @source.should_receive(:connect_backend).once.with
+  #       end
+  #       it "should call the harvest statement with an offset" do
+  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
+  #       end
+  #     end
+  #     context "without WHERE" do
+  #       it "should connect" do
+  #         @adapter.should_receive(:connect).once.with
+  #       end
+  #       it "should call the harvest statement with an offset" do
+  #         @source.should_receive(:harvest_statement_with_offset).once.with :some_type, :some_field, :some_offset
+  #       end
+  #     end
+  #   end
+  #   context 'returns' do
+  #     it "should return whatever the execute statement returns" do
+  #       @connection.stub! :execute => :some_result
+  #       
+  #       @source.harvest(:some_type, :some_field).should == :some_result
+  #     end
+  #   end
+  # end
 
   describe "harvest_statement_with_offset" do
     before(:each) do
@@ -76,15 +88,15 @@ describe Sources::DB do
     end
     it "should get a harvest statement and the chunksize to put the statement together" do
       @source.should_receive(:harvest_statement).once.and_return 'some_example_statement'
-      @source.harvest_statement_with_offset(@type, @field, :some_offset, :some_chunksize)
+      @source.harvest_statement_with_offset(@type, @field, :some_offset)
     end
     it "should add an AND if it already contains a WHERE statement" do
       @source.should_receive(:harvest_statement).and_return 'WHERE'
-      @source.harvest_statement_with_offset(@type, @field, :some_offset, :some_chunksize).should == "WHERE AND st.id > some_offset LIMIT some_chunksize"
+      @source.harvest_statement_with_offset(@type, @field, :some_offset).should == "WHERE AND st.id > some_offset LIMIT 25000"
     end
     it "should add a WHERE if it doesn't already contain one" do
       @source.should_receive(:harvest_statement).and_return 'some_statement'
-      @source.harvest_statement_with_offset(@type, @field, :some_offset, :some_chunksize).should == "some_statement WHERE st.id > some_offset LIMIT some_chunksize"
+      @source.harvest_statement_with_offset(@type, @field, :some_offset).should == "some_statement WHERE st.id > some_offset LIMIT 25000"
     end
   end
   

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Florian Hanke
@@ -14,148 +14,13 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-03 00:00:00 +02:00
+date: 2010-10-04 00:00:00 +02:00
 default_executable: picky
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: bundler
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 9
-        - 26
-        version: 0.9.26
-  type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: activesupport
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 3
-        - 8
-        version: 2.3.8
-  type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: activerecord
-  prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 3
-        - 8
-        version: 2.3.8
-  type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: rack
-  prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 2
-        - 1
-        version: 1.2.1
-  type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: rack-mount
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 6
-        - 9
-        version: 0.6.9
-  type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: rsolr
-  prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 12
-        - 1
-        version: 0.12.1
-  type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: sunspot
-  prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 1
-        - 0
-        version: 1.1.0
-  type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: text
-  prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 2
-        - 0
-        version: 0.2.0
-  type: :runtime
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: rack_fast_escape
-  prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2009
-        - 6
-        - 24
-        version: 2009.06.24
-  type: :runtime
-  version_requirements: *id009
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -164,7 +29,7 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id010
+  version_requirements: *id001
 description: Fast Combinatorial Ruby Search Engine
 email: florian.hanke+picky@gmail.com
 executables: 
@@ -199,7 +64,6 @@ files:
 - lib/picky/configuration/queries.rb
 - lib/picky/configuration/type.rb
 - lib/picky/cores.rb
-- lib/picky/db/configuration.rb
 - lib/picky/ext/ruby19/extconf.rb
 - lib/picky/extensions/array.rb
 - lib/picky/extensions/hash.rb
@@ -323,6 +187,7 @@ files:
 - spec/lib/results/base_spec.rb
 - spec/lib/routing_spec.rb
 - spec/lib/solr/schema_generator_spec.rb
+- spec/lib/sources/csv_spec.rb
 - spec/lib/sources/db_spec.rb
 - spec/lib/tokenizers/base_spec.rb
 - spec/lib/tokenizers/index_spec.rb
@@ -407,6 +272,7 @@ test_files:
 - spec/lib/results/base_spec.rb
 - spec/lib/routing_spec.rb
 - spec/lib/solr/schema_generator_spec.rb
+- spec/lib/sources/csv_spec.rb
 - spec/lib/sources/db_spec.rb
 - spec/lib/tokenizers/base_spec.rb
 - spec/lib/tokenizers/index_spec.rb

data/lib/picky/db/configuration.rb

@@ -1,23 +0,0 @@
-class DB < ActiveRecord::Base
-  
-  self.abstract_class = true
-  
-  #
-  #
-  def self.configure options = {}
-    @connection_options = if filename = options[:file]
-      File.open(File.join(SEARCH_ROOT, filename)) { |f| YAML::load(f) }
-    else
-      options
-    end
-    self
-  end
-  
-  #
-  #
-  def self.connect
-    return if SEARCH_ENVIRONMENT.to_s == 'test'
-    establish_connection @connection_options
-  end
-  
-end