jakewendt-active_record_sunspotter 4.0.0 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1cbf38085cf568de0799a1f8ce7ebb52545b26ff
-  data.tar.gz: ce140ffcd58a91c4eb55e5a4b8a2861e723bb8f7
+  metadata.gz: 79f05bc56e701726f80ddfaae0e883a2681b9153
+  data.tar.gz: ab3caa50a2fec615bc482630e736a38e3df24b77
 SHA512:
-  metadata.gz: 84df83b33a7dcd53314a4561054c45d46007b879dbcd7e2867f1fd2cf09cdc3ea3502decf9fdedf8d9d83bf2f808a0457b1be5d40b7d1ef2d8be0336fe090cff
-  data.tar.gz: 47eba5dffe54913b9f5b99b5974f23a87622913f67f9a61e11f01f44ede4a8754ce2f4feb809fc3d625ef7f7936e6f4dd9e9ab101028ce6bb5b90134cdfee111
+  metadata.gz: f10cc1f7f4330ac4be8d6ba46fe5c2099c89ec807edaf1c095ad3c83875807dc4e0521733ac9b09af317869d8c7a94c9b3e1f44b95718aee72c6b13b2edbddbe
+  data.tar.gz: ab600a96bd3d2d3d073825b1c01c9b845f69ecb4483c796ac8561249afad172439adc247e27c7111ba094651ccf675e3d16800bc75ffee6eca3e2f25931dfc2f

data/README.rdoc

@@ -7,6 +7,64 @@
 
 
 
+== Modifications
+
+I'm stuck using version 2.0.0.  Seems to be issues with java.  Anywho.
+This file loads too early for me to override, so...
+
+Need to edit ...
+
+/opt/local/lib/ruby2.0/gems/2.0.0/gems/sunspot_rails-2.0.0/lib/sunspot_rails.rb
+
+... to require 'json' rather than 'to_json' for rails 4.1.0 and above like so.
+
+
+if ::Rails.version >= '4.1'
+	require 'active_support/core_ext/object/json' 
+elsif ::Rails.version >= '3'
+	require 'active_support/core_ext/object/to_json' 
+end
+
+
+
+
+
+
+And another mod... (I may be able to patch this one.)
+
+sunspot_rails-2.0.0/lib/sunspot/rails/searchable.rb
+
+Must comment out the :include option.  
+
+251           find_in_batch_options = {
+252 #            :include => options[:include],
+253             :batch_size => options[:batch_size],
+254             :start => options[:start]
+255           }
+
+
+Don't know why now as it has been there for a while, but even looking back to 4.0.3
+it hasn't been a valid key?
+
+ArgumentError: Unknown key: :include. Valid keys are: :start, :batch_size
+/my/ruby/gems/2.0/gems/activesupport-4.1.0/lib/active_support/core_ext/hash/keys.rb:71:in `block in assert_valid_keys'
+/my/ruby/gems/2.0/gems/activesupport-4.1.0/lib/active_support/core_ext/hash/keys.rb:69:in `each_key'
+/my/ruby/gems/2.0/gems/activesupport-4.1.0/lib/active_support/core_ext/hash/keys.rb:69:in `assert_valid_keys'
+/my/ruby/gems/2.0/gems/activerecord-4.1.0/lib/active_record/relation/batches.rb:100:in `find_in_batches'
+/my/ruby/gems/2.0/gems/activerecord-4.1.0/lib/active_record/querying.rb:9:in `find_in_batches'
+/my/ruby/gems/2.0/gems/sunspot_rails-2.0.0/lib/sunspot/rails/searchable.rb:259:in `solr_index'
+
+
+
+
+== Testing
+
+
+
+ sudo /opt/local/share/mariadb/support-files/mysql.server start
+
+ c ; bundle exec rake sunspot:solr:stop ; bundle exec rake sunspot:solr:stop RAILS_ENV=test ; bundle exec rake sunspot:solr:start ; bundle exec rake sunspot:solr:start RAILS_ENV=test ; bundle exec rake test
+
 
 
 == Gemified with Jeweler

data/lib/active_record_sunspotter/rails/railtie.rb

@@ -3,6 +3,10 @@ module ActiveRecordSunspotter
 		initializer 'add_helpers_to_actionview' do |app|
 			ActiveSupport.on_load :action_view do
 				ActionView::Base.send(:include, ActiveRecordSunspotter::SunspotHelper)
+				#	:sunspot would require sunspot_helper.rb be in one of the magical load paths.
+				#ActionController::Base.helper :sunspot
+				#	So much simpler to just pass the module name.
+				ActionController::Base.helper ActiveRecordSunspotter::SunspotHelper
 			end
 		end
 #		rake_tasks do

data/lib/active_record_sunspotter/sunspot_helper.rb

@@ -184,4 +184,52 @@ module ActiveRecordSunspotter::SunspotHelper
 		html_content.html_safe
 	end
 
+
+	#	↑ and ↓
+	def sort_link(*args)
+		options = {
+			:image => true
+		}.merge(args.extract_options!)
+		column = args[0]
+		text = args[1]
+		#	make local copy so mods to muck up real params which
+		#	may still be references elsewhere.
+		local_params = params.dup
+
+#
+#	May want to NOT flip dir for other columns.  Only the current order.
+#	Will wait until someone else makes the suggestion.
+#
+		order = column.to_s.downcase.gsub(/\s+/,'_')
+		dir = ( local_params[:dir] && local_params[:dir] == 'asc' ) ? 'desc' : 'asc'
+
+		local_params[:page] = nil
+		link_text = text||column
+		classes = ['sortable',order]
+		arrow = ''
+		if local_params[:order] && local_params[:order] == order
+			classes.push('sorted')
+			arrow = if dir == 'desc'
+#				if File.exists?( sort_down_image ) && options[:image]
+#					image_tag( File.basename(sort_down_image), :class => 'down arrow')
+#				else
+					"<span class='down arrow'>&darr;</span>"
+#				end
+			else
+#				if File.exists?( sort_up_image ) && options[:image]
+#					image_tag( File.basename(sort_up_image), :class => 'up arrow')
+#				else
+					"<span class='up arrow'>&uarr;</span>"
+#				end
+			end
+		end
+		s = "<div class='#{classes.join(' ')}'>"
+		s << link_to(link_text.html_safe,local_params.merge(:order => order,:dir => dir))
+		s << arrow unless arrow.blank?
+		s << "</div>"
+		s.html_safe
+	end
+
+
+
 end

data/lib/active_record_sunspotter/sunspot_rails_2.1.0_sunspot_rails_searchable.rb

@@ -0,0 +1,497 @@
+module Sunspot #:nodoc:
+  module Rails #:nodoc:
+    # 
+    # This module adds Sunspot functionality to ActiveRecord models. As well as
+    # providing class and instance methods, it optionally adds lifecycle hooks
+    # to automatically add and remove models from the Solr index as they are
+    # created and destroyed.
+    #
+    module Searchable
+      class <<self
+        def included(base) #:nodoc:
+          base.module_eval do
+            extend(ActsAsMethods)
+          end
+        end
+      end
+
+      module ActsAsMethods
+        # 
+        # Makes a class searchable if it is not already, or adds search
+        # configuration if it is. Note that the options passed in are only used
+        # the first time this method is called for a particular class; so,
+        # search should be defined before activating any mixins that extend
+        # search configuration.
+        #
+        # The block passed into this method is evaluated by the
+        # <code>Sunspot.setup</code> method. See the Sunspot documentation for
+        # complete information on the functionality provided by that method.
+        #
+        # ==== Options (+options+)
+        # 
+        # :auto_index<Boolean>::
+        #   Automatically index models in Solr when they are saved.
+        #   Default: true
+        # :auto_remove<Boolean>::
+        #   Automatically remove models from the Solr index when they are
+        #   destroyed. <b>Setting this option to +false+ is not recommended
+        #   </b>(see the README).
+        # :if<Mixed>::
+        #   Only index models in Solr if the method, proc or string evaluates
+        #   to true (e.g. <code>:if => :should_index?</code> or <code>:if =>
+        #   proc { |model| model.foo > 2 }</code>).  Models that do not match
+        #   the constraint will be removed from the index upon save.  Multiple
+        #   constraints can be specified by passing an array (e.g. <code>:if =>
+        #   [:method1, :method2]</code>).
+        # :ignore_attribute_changes_of<Array>::
+        #   Define attributes, that should not trigger a reindex of that
+        #   object. Usual suspects are updated_at or counters.
+        # :only_reindex_attribute_changes_of<Array>::
+        #   Define attributes, that are the only attributes that should
+        #   trigger a reindex of that object. Useful if there are a small
+        #   number of searchable attributes and a large number of attributes
+        #   to ignore.
+        # :include<Mixed>::
+        #   Define default ActiveRecord includes, set this to allow ActiveRecord
+        #   to load required associations when indexing. See ActiveRecord's 
+        #   documentation on eager-loading for examples on how to set this
+        #   Default: [] 
+        # :unless<Mixed>::
+        #   Only index models in Solr if the method, proc or string evaluates
+        #   to false (e.g. <code>:unless => :should_not_index?</code> or <code>
+        #   :unless => proc { |model| model.foo <= 2 }</code>).  Models that do
+        #   not match the constraint will be removed from the index upon save.
+        #   Multiple constraints can be specified by passing an array (e.g.
+        #   <code>:unless => [:method1, :method2]</code>).
+        #
+        # ==== Example
+        #
+        #   class Post < ActiveRecord::Base
+        #     searchable do
+        #       text :title, :body
+        #       string :sort_title do
+        #         title.downcase.sub(/^(an?|the)/, '')
+        #       end
+        #       integer :blog_id
+        #       time :updated_at
+        #     end
+        #   end
+        #
+        def searchable(options = {}, &block)
+          Sunspot.setup(self, &block)
+
+          if searchable?
+            sunspot_options[:include].concat(Util::Array(options[:include]))
+          else
+            extend ClassMethods
+            include InstanceMethods
+
+            class_attribute :sunspot_options
+
+            unless options[:auto_index] == false
+              before_save :mark_for_auto_indexing_or_removal
+              after_save :perform_index_tasks
+            end
+
+            unless options[:auto_remove] == false
+              after_destroy do |searchable|
+                searchable.remove_from_index
+              end
+            end
+            options[:include] = Util::Array(options[:include])
+            
+            self.sunspot_options = options
+          end
+        end
+
+        # 
+        # This method is defined on all ActiveRecord::Base subclasses. It
+        # is false for classes on which #searchable has not been called, and
+        # true for classes on which #searchable has been called.
+        #
+        # ==== Returns
+        #
+        # +false+
+        #
+        def searchable?
+          false
+        end
+      end
+
+      module ClassMethods
+        def self.extended(base) #:nodoc:
+          class <<base
+            alias_method :search, :solr_search unless method_defined? :search
+            alias_method :search_ids, :solr_search_ids unless method_defined? :search_ids
+            alias_method :remove_all_from_index, :solr_remove_all_from_index unless method_defined? :remove_all_from_index
+            alias_method :remove_all_from_index!, :solr_remove_all_from_index! unless method_defined? :remove_all_from_index!
+            alias_method :reindex, :solr_reindex unless method_defined? :reindex
+            alias_method :index, :solr_index unless method_defined? :index
+            alias_method :index_orphans, :solr_index_orphans unless method_defined? :index_orphans
+            alias_method :clean_index_orphans, :solr_clean_index_orphans unless method_defined? :clean_index_orphans
+          end
+        end
+        # 
+        # Search for instances of this class in Solr. The block is delegated to
+        # the Sunspot.search method - see the Sunspot documentation for the full
+        # API.
+        #
+        # ==== Example
+        #
+        #   Post.search(:include => [:blog]) do
+        #     keywords 'best pizza'
+        #     with :blog_id, 1
+        #     order :updated_at, :desc
+        #     facet :category_ids
+        #   end
+        #
+        # ==== Options
+        #
+        # :include:: Specify associations to eager load
+        # :select:: Specify columns to select from database when loading results
+        #
+        # ==== Returns
+        #
+        # Sunspot::Search:: Object containing results, totals, facets, etc.
+        #
+        def solr_search(options = {}, &block)
+          solr_execute_search(options) do
+            Sunspot.new_search(self, &block)
+          end
+        end
+
+        # 
+        # Get IDs of matching results without loading the result objects from
+        # the database. This method may be useful if search is used as an
+        # intermediate step in a larger find operation. The block is the same
+        # as the block provided to the #search method.
+        #
+        # ==== Returns
+        #
+        # Array:: Array of IDs, in the order returned by the search
+        #
+        def solr_search_ids(&block)
+          solr_execute_search_ids do
+            solr_search(&block)
+          end
+        end
+
+        # 
+        # Remove instances of this class from the Solr index.
+        #
+        def solr_remove_all_from_index
+          Sunspot.remove_all(self)
+        end
+
+        # 
+        # Remove all instances of this class from the Solr index and immediately
+        # commit.
+        #
+        #
+        def solr_remove_all_from_index!
+          Sunspot.remove_all!(self)
+        end
+
+        # 
+        # Completely rebuild the index for this class. First removes all
+        # instances from the index, then loads records and indexes them.
+        #
+        # See #index for information on options, etc.
+        #
+        def solr_reindex(options = {})
+          solr_remove_all_from_index
+          solr_index(options)
+        end
+
+        #
+        # Add/update all existing records in the Solr index. The
+        # +batch_size+ argument specifies how many records to load out of the
+        # database at a time. The default batch size is 50; if nil is passed,
+        # records will not be indexed in batches. By default, a commit is issued
+        # after each batch; passing +false+ for +batch_commit+ will disable
+        # this, and only issue a commit at the end of the process. If associated
+        # objects need to indexed also, you can specify +include+ in format 
+        # accepted by ActiveRecord to improve your sql select performance
+        #
+        # ==== Options (passed as a hash)
+        #
+        # batch_size<Integer>:: Batch size with which to load records. Passing
+        #                       'nil' will skip batches.  Default is 50.
+        # batch_commit<Boolean>:: Flag signalling if a commit should be done after
+        #                         after each batch is indexed, default is 'true'
+        # include<Mixed>:: include option to be passed to the ActiveRecord find,
+        #                  used for including associated objects that need to be
+        #                  indexed with the parent object, accepts all formats
+        #                  ActiveRecord::Base.find does
+        # first_id:: The lowest possible ID for this class. Defaults to 0, which
+        #            is fine for integer IDs; string primary keys will need to
+        #            specify something reasonable here.
+        #
+        # ==== Examples
+        #   
+        #   # index in batches of 50, commit after each
+        #   Post.index 
+        #
+        #   # index all rows at once, then commit
+        #   Post.index(:batch_size => nil) 
+        #
+        #   # index in batches of 50, commit when all batches complete
+        #   Post.index(:batch_commit => false) 
+        #
+        #   # include the associated +author+ object when loading to index
+        #   Post.index(:include => :author) 
+        #
+        def solr_index(opts={})
+          options = {
+            :batch_size => Sunspot.config.indexing.default_batch_size,
+            :batch_commit => true,
+            :include => self.sunspot_options[:include],
+            :start => opts.delete(:first_id)
+          }.merge(opts)
+
+          if options[:batch_size].to_i > 0
+            batch_counter = 0
+            self.includes(options[:include]).find_in_batches(options.slice(:batch_size, :start)) do |records|
+              
+              solr_benchmark(options[:batch_size], batch_counter += 1) do
+                Sunspot.index(records.select { |model| model.indexable? })
+                Sunspot.commit if options[:batch_commit]
+              end
+
+              options[:progress_bar].increment!(records.length) if options[:progress_bar]
+            end
+          else
+            Sunspot.index! self.includes(options[:include]).select(&:indexable?)
+          end
+
+          # perform a final commit if not committing in batches
+          Sunspot.commit unless options[:batch_commit]
+        end
+
+        # 
+        # Return the IDs of records of this class that are indexed in Solr but
+        # do not exist in the database. Under normal circumstances, this should
+        # never happen, but this method is provided in case something goes
+        # wrong. Usually you will want to rectify the situation by calling
+        # #clean_index_orphans or #reindex
+        # 
+        # ==== Options (passed as a hash)
+        #
+        # batch_size<Integer>:: Batch size with which to load records. Passing
+        #                       Default is 1000 (from ActiveRecord).
+        # 
+        # ==== Returns
+        #
+        # Array:: Collection of IDs that exist in Solr but not in the database
+        def solr_index_orphans(opts={})
+          batch_size = opts[:batch_size] || Sunspot.config.indexing.default_batch_size          
+
+          solr_page = 0
+          solr_ids = []
+          while (solr_page = solr_page.next)
+            ids = solr_search_ids { paginate(:page => solr_page, :per_page => 1000) }.to_a
+            break if ids.empty?
+            solr_ids.concat ids
+          end
+
+          return solr_ids - self.connection.select_values("SELECT id FROM #{quoted_table_name}").collect(&:to_i)
+        end
+
+        # 
+        # Find IDs of records of this class that are indexed in Solr but do not
+        # exist in the database, and remove them from Solr. Under normal
+        # circumstances, this should not be necessary; this method is provided
+        # in case something goes wrong.
+        #
+        # ==== Options (passed as a hash)
+        #
+        # batch_size<Integer>:: Batch size with which to load records
+        #                       Default is 50
+        # 
+        def solr_clean_index_orphans(opts={})
+          solr_index_orphans(opts).each do |id|
+            new do |fake_instance|
+              fake_instance.id = id
+            end.solr_remove_from_index
+          end
+        end
+
+        # 
+        # Classes that have been defined as searchable return +true+ for this
+        # method.
+        #
+        # ==== Returns
+        #
+        # +true+
+        #
+        def searchable?
+          true
+        end
+        
+        def solr_execute_search(options = {})
+          options.assert_valid_keys(:include, :select)
+          search = yield
+          unless options.empty?
+            search.build do |query|
+              if options[:include]
+                query.data_accessor_for(self).include = options[:include]
+              end
+              if options[:select]
+                query.data_accessor_for(self).select = options[:select]
+              end
+            end
+          end
+          search.execute
+        end
+
+        def solr_execute_search_ids(options = {})
+          search = yield
+          search.raw_results.map { |raw_result| raw_result.primary_key.to_i }
+        end
+        
+        protected
+        
+        # 
+        # Does some logging for benchmarking indexing performance
+        #
+        def solr_benchmark(batch_size, counter,  &block)
+          start = Time.now
+          logger.info("[#{Time.now}] Start Indexing")
+          yield
+          elapsed = Time.now-start
+          logger.info("[#{Time.now}] Completed Indexing. Rows indexed #{counter * batch_size}. Rows/sec: #{batch_size/elapsed.to_f} (Elapsed: #{elapsed} sec.)")
+        end
+
+      end
+
+      module InstanceMethods
+        def self.included(base) #:nodoc:
+          base.module_eval do
+            alias_method :index, :solr_index unless method_defined? :index
+            alias_method :index!, :solr_index! unless method_defined? :index!
+            alias_method :remove_from_index, :solr_remove_from_index unless method_defined? :remove_from_index
+            alias_method :remove_from_index!, :solr_remove_from_index! unless method_defined? :remove_from_index!
+            alias_method :more_like_this, :solr_more_like_this unless method_defined? :more_like_this
+            alias_method :more_like_this_ids, :solr_more_like_this_ids unless method_defined? :more_like_this_ids
+          end
+        end
+        # 
+        # Index the model in Solr. If the model is already indexed, it will be
+        # updated. Using the defaults, you will usually not need to call this
+        # method, as models are indexed automatically when they are created or
+        # updated. If you have disabled automatic indexing (see
+        # ClassMethods#searchable), this method allows you to manage indexing
+        # manually.
+        #
+        def solr_index
+          Sunspot.index(self)
+        end
+
+        # 
+        # Index the model in Solr and immediately commit. See #index
+        #
+        def solr_index!
+          Sunspot.index!(self)
+        end
+        
+        # 
+        # Remove the model from the Solr index. Using the defaults, this should
+        # not be necessary, as models will automatically be removed from the
+        # index when they are destroyed. If you disable automatic removal
+        # (which is not recommended!), you can use this method to manage removal
+        # manually.
+        #
+        def solr_remove_from_index
+          Sunspot.remove(self)
+        end
+
+        # 
+        # Remove the model from the Solr index and commit immediately. See
+        # #remove_from_index
+        #
+        def solr_remove_from_index!
+          Sunspot.remove!(self)
+        end
+
+        def solr_more_like_this(*args, &block)
+          options = args.extract_options!
+          self.class.solr_execute_search(options) do
+            Sunspot.new_more_like_this(self, *args, &block)
+          end
+        end
+
+        def solr_more_like_this_ids(&block)
+          self.class.solr_execute_search_ids do
+            solr_more_like_this(&block)
+          end
+        end
+
+        def indexable?
+          # options[:if] is not specified or they successfully pass
+          if_passes = self.class.sunspot_options[:if].nil? ||
+                      constraint_passes?(self.class.sunspot_options[:if])
+
+          # options[:unless] is not specified or they successfully pass
+          unless_passes = self.class.sunspot_options[:unless].nil? ||
+                          !constraint_passes?(self.class.sunspot_options[:unless])
+
+          if_passes and unless_passes
+        end
+
+        private
+
+        def constraint_passes?(constraint)
+          case constraint
+          when Symbol
+            self.__send__(constraint)
+          when String
+            self.__send__(constraint.to_sym)
+          when Enumerable
+            # All constraints must pass
+            constraint.all? { |inner_constraint| constraint_passes?(inner_constraint) }
+          else
+            if constraint.respond_to?(:call) # could be a Proc or anything else that responds to call
+              constraint.call(self)
+            else
+              raise ArgumentError, "Unknown constraint type: #{constraint} (#{constraint.class})"
+            end
+          end
+        end
+
+        def mark_for_auto_indexing_or_removal
+          if indexable?
+            # :if/:unless constraints pass or were not present
+
+            @marked_for_auto_indexing =
+              if !new_record? && ignore_attributes = self.class.sunspot_options[:ignore_attribute_changes_of]
+                !(changed.map { |attr| attr.to_sym } - ignore_attributes).blank?
+              elsif !new_record? && only_attributes = self.class.sunspot_options[:only_reindex_attribute_changes_of]
+                !(changed.map { |attr| attr.to_sym } & only_attributes).blank?
+              else
+                true
+              end
+
+            @marked_for_auto_removal = false
+          else
+            # :if/:unless constraints did not pass; do not auto index and
+            # actually go one step further by removing it from the index
+            @marked_for_auto_indexing = false
+            @marked_for_auto_removal = true
+          end
+
+          true
+        end
+
+        def perform_index_tasks
+          if @marked_for_auto_indexing
+            solr_index
+            remove_instance_variable(:@marked_for_auto_indexing)
+          elsif @marked_for_auto_removal
+            solr_remove_from_index
+            remove_instance_variable(:@marked_for_auto_removal)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/active_record_sunspotter.rb

@@ -15,6 +15,19 @@ require 'active_record_sunspotter/sunspot_helper'
 #	copied in this 1 file
 require 'active_record_sunspotter/sunspot_rails_2.1.0_sunspot_rails_adapters'
 
+#	for some reason, the :include key will inspire ...
+#	ArgumentError: Unknown key: :include. Valid keys are: :start, :batch_size
+#	Tasks: TOP => sunspot:solr:reindex => sunspot:reindex
+#	Seems to have been this way for some time, but now it errors.
+#	Using 2.1.0's version makes everyone happy?
+require 'active_record_sunspotter/sunspot_rails_2.1.0_sunspot_rails_searchable'
+#	Somehow this requires the presence of a new file solr/solr.xml
+
+
+#	Really would just prefer to install 2.1.0, but it just won't work for me.
+#	Still wasting my time investigating though.
+#	Seems to be a java problem.  Surprised?
+
 
 if defined?(Rails)
 	require 'active_record_sunspotter/rails/engine'

data/lib/tasks/application.rake

@@ -0,0 +1,214 @@
+#namespace :app do
+#	task :import => :environment do
+#		f = File.open('TheKingJamesBible.txt','r')
+#		{} until( ( line = f.gets ) =~ /^The First Book of Moses:  Called Genesis\s*$/ )
+#		puts line
+#		f.close	#	File.open('TheKingJamesBible.txt','r')
+#	end	#	task :import => :environment do
+#end	#	namespace :app do
+namespace :app do
+	task :import => :environment do
+		File.open('data/TheKingJamesBible.txt','r') do |f|
+			until ( line = f.gets ).match(/^\*\*\*/) 
+			#	skip
+			end
+			while ( line = f.gets ).match(/^\s*$/)
+			#	skip
+			end
+			puts line 
+			#	=> The Old Testament of the King James Version of the Bible
+
+			chapter_num = verse_num = book = chapter = verse = nil
+			new_line_counter = 0
+			book_counter = 0
+			buffer = ''
+
+
+#	basic expectations ...
+#
+#	book ends with >4 new lines but can include new lines (<=2 only)
+#	first verse is after >2 new lines
+
+			while cval = f.getc
+				next if cval == "\r"	#13
+
+				if cval == "\n"	#10
+					new_line_counter += 1
+					buffer << ' '
+					next
+				end
+
+
+				if new_line_counter > 4 && book && chapter && verse && !buffer.blank?
+					#	last verse of book
+					puts "-- CH:#{chapter_num}:V:#{verse_num}:#{buffer}"
+					verse.body = buffer.squish
+					verse.save
+					buffer = ''
+					chapter_num = verse_num = book = chapter = verse = nil
+				end
+
+#	2 new lines isn't just for new books
+#	some chapters are separated by 2 new lines
+#	so need more somehow.  Psalms is causing the problem with the extra new line between chapters
+#	manually removed them
+				if new_line_counter > 2 && !buffer.blank?
+					puts "\n\nLooks like a new book is here:#{buffer.squish}:\n"
+					book_counter += 1
+					#book = buffer.squish
+					book = Book.create(:title => buffer.squish)
+					buffer = ''
+				end
+
+				buffer << cval	#.chr
+
+				if buffer.match(/^([\D]*)(\d+):(\d+)\s+/) #or ( buffer =~ /^([\D]*)End of the Project Gutenberg EBook/ )
+					text = $1	#	previous verse
+					if chapter_num && verse_num && text
+						puts "CH:#{chapter_num}:V:#{verse_num}:#{text}"
+						verse.body = text.squish
+						verse.save
+					end
+					#chapter = $2.squish #if $2
+					chapter = book.chapters.find_or_create_by(:position => $2.to_i)
+					chapter_num = $2.squish #if $2
+##	I guess that the verse number is just going to be the position?
+##	relax self!  just a demo!
+					#verse = $3.squish #if $3
+					verse = chapter.verses.new(:position => $3.to_i)	#create(:body => $3)
+					verse_num = $3.squish #if $3
+					buffer= ''
+				end
+
+				break if buffer =~ /End of the Project Gutenberg EBook/
+
+				#	reset counter (perhaps way too often)
+				new_line_counter = 0 if cval != "\n"	#	10
+			end
+			puts book_counter
+		end	#	File
+	end	#	task
+
+end	#	namespace
+
+__END__
+
+
+40:38 For the cloud of the LORD was upon the tabernacle by day, and
+fire was on it by night, in the sight of all the house of Israel,
+throughout all their journeys.
+
+
+
+
+The Third Book of Moses:  Called Leviticus
+
+
+1:1 And the LORD called unto Moses, and spake unto him out of the
+tabernacle of the congregation, saying, 1:2 Speak unto the children of
+Israel, and say unto them, If any man of you bring an offering unto
+the LORD, ye shall bring your offering of the cattle, even of the
+herd, and of the flock.
+
+
+
+CH:40:V:37:But if the cloud were not taken up, then they journeyed not till the day that it was taken up.  
+create new book with title:For the cloud of the LORD was upon the tabernacle by day, and fire was on it by night, in the sight of all the house of Israel, throughout all their journeys.  :
+create new book with title:   The Third Book of Moses:  Called Leviticus  :
+CH:1:V:1:And the LORD called unto Moses, and spake unto him out of the tabernacle of the congregation, saying, 
+CH:1:V:2:Speak unto the children of Israel, and 
+
+
+The First Book of Moses:  Called Genesis
+
+
+1:1 In the beginning God created the heaven and the earth.
+
+1:2 And the earth was without form, and void; and darkness was upon
+the face of the deep. And the Spirit of God moved upon the face of the
+waters.
+
+FYI, some verses actually have multiple "paragraphs" .
+
+Also, some verses are merged on the same line?
+
+3:1 Now the serpent was more subtil than any beast of the field which
+the LORD God had made. And he said unto the woman, Yea, hath God said,
+Ye shall not eat of every tree of the garden?  3:2 And the woman said
+unto the serpent, We may eat of the fruit of the trees of the garden:
+3:3 But of the fruit of the tree which is in the midst of the garden,
+God hath said, Ye shall not eat of it, neither shall ye touch it, lest
+ye die.
+
+
+
+
+50:26 So Joseph died, being an hundred and ten years old: and they
+embalmed him, and he was put in a coffin in Egypt.
+
+
+
+
+The Second Book of Moses:  Called Exodus
+
+
+1:1 Now these are the names of the children of Israel, which came
+into Egypt; every man and his household came with Jacob.
+
+1:2 Reuben, Simeon, Levi, and Judah, 1:3 Issachar, Zebulun, and
+Benjamin, 1:4 Dan, and Naphtali, Gad, and Asher.
+
+1:5 And all the souls that came out of the loins of Jacob were seventy
+souls: for Joseph was in Egypt already.
+
+1:6 And Joseph died, and all his brethren, and all that generation.
+
+1:7 And the children of Israel were fruitful, and increased
+abundantly, and multiplied, and waxed exceeding mighty; and the land
+was filled with them.
+
+1:8 Now there arose up a new king over Egypt, which knew not Joseph.
+
+1:9 And he said unto his people, Behold, the people of the children of
+Israel are more and mightier than we: 1:10 Come on, let us deal wisely
+with them; lest they multiply, and it come to pass, that, when there
+falleth out any war, they join also unto our enemies, and fight
+against us, and so get them up out of the land.
+
+1:11 Therefore they did set over them taskmasters to afflict them with
+their burdens. And they built for Pharaoh treasure cities, Pithom and
+Raamses.
+
+1:12 But the more they afflicted them, the more they multiplied and
+grew.
+
+And they were grieved because of the children of Israel.
+
+
+
+
+
+
+4:5 Behold, I will send you Elijah the prophet before the coming of
+the great and dreadful day of the LORD: 4:6 And he shall turn the
+heart of the fathers to the children, and the heart of the children to
+their fathers, lest I come and smite the earth with a curse.
+
+
+***
+
+
+
+
+The New Testament of the King James Bible
+
+
+
+
+The Gospel According to Saint Matthew
+
+
+1:1 The book of the generation of Jesus Christ, the son of David, the
+son of Abraham.
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jakewendt-active_record_sunspotter
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.1.0
 platform: ruby
 authors:
 - George 'Jake' Wendt
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-18 00:00:00.000000000 Z
+date: 2014-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sunspot_rails
@@ -42,14 +42,154 @@ dependencies:
   name: progress_bar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: will_paginate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sass-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.0.2
+- !ruby/object:Gem::Dependency
+  name: uglifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+- !ruby/object:Gem::Dependency
+  name: turbolinks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jbuilder
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: american_date
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: acts_as_list
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: active_record_sunspotter
@@ -66,9 +206,11 @@ files:
 - lib/active_record_sunspotter/sunspot_column.rb
 - lib/active_record_sunspotter/sunspot_helper.rb
 - lib/active_record_sunspotter/sunspot_rails_2.1.0_sunspot_rails_adapters.rb
+- lib/active_record_sunspotter/sunspot_rails_2.1.0_sunspot_rails_searchable.rb
 - lib/active_record_sunspotter/sunspot_rails_server.rb
 - lib/active_record_sunspotter/sunspotability.rb
 - lib/jakewendt-active_record_sunspotter.rb
+- lib/tasks/application.rake
 - vendor/assets/javascripts/sunspot.js
 - vendor/assets/stylesheets/sunspot.css.scss
 - vendor/views/layouts/_footer.html.erb
@@ -94,12 +236,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []