solrizer 0.1.0.pre2

data/.gitignore

@@ -0,0 +1,19 @@
+.DS_Store
+nohup.out
+
+*.sqlite3
+*.log
+*~
+*.swp
+
+pkg/
+coverage/*
+ 
+tmp/**/*
+tmp/performance
+ 
+rerun.txt
+ 
+.loadpath
+.project
+.buildpath

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Matt Zumwalt
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= foo
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Matt Zumwalt. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,46 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "solrizer"
+    gem.summary = %Q{A utility for building solr indexes, usually from Fedora repository content.}
+    gem.description = %Q{Use solrizer to populate solr indexes from Fedora repository content or from other sources.  You can run solrizer from within your apps, using the provided rake tasks, or as a JMS listener}
+    gem.email = "matt.zumwalt@yourmediashelf.com"
+    gem.homepage = "http://github.com/projecthydra/solrizer"
+    gem.authors = ["Matt Zumwalt"]
+    gem.add_dependency "active-fedora", "> 1.1.3"
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "solrizer #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0.pre2

data/config/fedora.yml

@@ -0,0 +1,16 @@
+development:
+  fedora:
+    url: http://fedoraAdmin:fedoraAdmin@127.0.0.1:8983/fedora
+  solr:
+    url: http://127.0.0.1:8983/solr/development
+test:
+  fedora:
+    url: http://fedoraAdmin:fedoraAdmin@127.0.0.1:8983/fedora
+  solr:
+    url: http://127.0.0.1:8983/solr/test
+production:
+  fedora:
+    url: http://fedoraAdmin:fedoraAdmin@127.0.0.1:8080/fedora
+  solr:
+    url: http://127.0.0.1:8080/solr
+    

data/config/hydra_types.yml

@@ -0,0 +1,4 @@
+info:fedora/afmodel:SaltDocument : salt_document
+info:fedora/afmodel:JP2Document : jp2_document
+info:fedora/afmodel:ModsDocument : mods_document
+info:fedora/afmodel:DCDocument : dc_document

data/config/solr.yml

@@ -0,0 +1,7 @@
+development:
+  url: http://localhost:8983/solr/development
+test: &TEST
+  url: http://localhost:8983/solr/test
+production:
+  url: http://localhost:8080/solr/production 
+

data/lib/solrizer.rb

@@ -0,0 +1,101 @@
+require 'rubygems'
+require 'solrizer/indexer.rb'
+# require 'fastercsv'
+require "ruby-debug"
+
+
+
+module Solrizer
+class Solrizer
+
+  attr_accessor :indexer, :index_full_text
+
+  #
+  # This method initializes the indexer
+  # If passed an argument of :index_full_text=>true, it will perform full-text indexing instead of indexing fields only.
+  #
+  def initialize( opts={} )
+    @@index_list = false unless defined?(@@index_list)
+    if opts[:index_full_text] == true || opts[:index_full_text] == "true"
+      @index_full_text = true 
+    else
+      @index_full_text = false 
+    end
+    @indexer = Indexer.new( :index_full_text=>@index_full_text )
+  end
+
+  #
+  # This method solrizes the given Fedora object's full-text and facets into the search index
+  #
+  def solrize( obj )
+    # retrieve the Fedora object based on the given unique id
+      
+      begin
+      
+      start = Time.now
+      print "Retrieving object #{obj} ..."
+      obj = obj.kind_of?(ActiveFedora::Base) ? obj : Repository.get_object( obj )
+        
+          obj_done = Time.now
+          obj_done_elapse = obj_done - start
+          puts  " completed. Duration: #{obj_done_elapse}"
+          
+         print "\t Indexing object #{obj.pid} ... "
+         # add the keywords and facets to the search index
+         index_start = Time.now
+         indexer.index( obj )
+         
+         index_done = Time.now
+         index_elapsed = index_done - index_start
+         
+          puts "completed. Duration:  #{index_elapsed} ."
+        
+      
+      rescue Exception => e
+           p "unable to index #{obj}.  Failed with #{e.inspect}"
+        
+      
+      end #begin
+  
+  end
+  
+  #
+  # This method retrieves a comprehensive list of all the unique identifiers in Fedora and 
+  # solrizes each object's full-text and facets into the search index
+  #
+  def solrize_objects
+    # retrieve a list of all the pids in the fedora repository
+    num_docs = 1000000   # modify this number to guarantee that all the objects are retrieved from the repository
+    puts "WARNING: You have turned off indexing of Full Text content.  Be sure to re-run indexer with @@index_full_text set to true in main.rb" if index_full_text == false
+
+    if @@index_list == false
+     
+       pids = Repository.get_pids( num_docs )
+	     puts "Shelving #{pids.length} Fedora objects"
+       pids.each do |pid|
+         unless pid[0].empty? || pid[0].nil?
+          solrize( pid )
+          end
+        end #pids.each
+     
+    else
+    
+       if File.exists?(@@index_list)
+          arr_of_pids = FasterCSV.read(@@index_list, :headers=>false)
+          
+          puts "Indexing from list at #{@@index_list}"
+          puts "Shelving #{arr_of_pids.length} Fedora objects"
+          
+         arr_of_pids.each do |row|
+            pid = row[0]
+            solrize( pid )
+	 end #FASTERCSV
+        else
+          puts "#{@@index_list} does not exists!"
+        end #if File.exists
+     
+    end #if Index_LISTS
+  end #solrize_objects
+
+end #class
+end #module

data/lib/solrizer/configuration.rb

@@ -0,0 +1,8 @@
+
+# FEDORA_URL = 'http://fedoraAdmin:fedoraAdmin@salt-dev.stanford.edu/fedora'
+# FEDORA_SOLR_URL = 'http://salt-dev.stanford.edu:8080/solr'
+# SHELVER_SOLR_URL = 'http://sulwebappdev1.stanford.edu:8100/salt_solr'
+FEDORA_URL = 'http://fedoraAdmin:fedoraAdmin@localhost:8080/fedora'
+FEDORA_SOLR_URL = 'http://localhost:8080/solr'
+SHELVER_SOLR_URL = 'http://localhost:8080/bl_solr'
+

data/lib/solrizer/extractor.rb

@@ -0,0 +1,89 @@
+require 'solr'
+require 'rexml/document'
+require "nokogiri"
+require 'yaml'
+
+module Solrizer
+class Extractor
+  
+  
+  def extract_tags(text)
+    doc = REXML::Document.new( text )
+    extract_tag(doc, 'archivist_tags').merge(extract_tag(doc, 'donor_tags'))
+  end
+  
+  def extract_tag(doc, type)
+    tags = doc.elements["/fields/#{type}"]
+    return {} unless tags
+    {type => tags.text.split(/,/).map {|t| t.strip}}
+  end
+
+  
+  #
+  # Extracts content-model and hydra-type from RELS-EXT datastream
+  #
+  def extract_rels_ext( text, solr_doc=Solr::Document.new )
+    # TODO: only read in this file once
+    
+    if defined?(RAILS_ROOT)
+      config_path = File.join(RAILS_ROOT, "config")
+    else
+      config_path = File.join(File.dirname(__FILE__), "..", "..", "config")
+    end    
+    map = YAML.load(File.open(File.join(config_path, "hydra_types.yml")))
+    
+    doc = Nokogiri::XML(text)
+    doc.xpath( '//foo:hasModel', 'foo' => 'info:fedora/fedora-system:def/model#' ).each do |element|
+      cmodel = element.attributes['resource'].to_s
+      solr_doc << Solr::Field.new( :cmodel_t => cmodel )
+      
+      if map.has_key?(cmodel)
+        solr_doc << Solr::Field.new( :hydra_type_t => map[cmodel] )
+      end
+    end
+
+    return solr_doc
+  end
+
+  #
+  # This method extracts solr fields from simple xml
+  #
+  def xml_to_solr( text, solr_doc=Solr::Document.new )
+    doc = REXML::Document.new( text )
+    doc.root.elements.each do |element|
+      solr_doc << Solr::Field.new( :"#{element.name}_t" => "#{element.text}" )
+    end
+
+    return solr_doc
+  end
+  
+  #
+  # This method strips html tags out and returns content to be indexed in solr
+  #
+  def html_content_to_solr( ds, solr_doc=Solr::Document.new )
+    
+    text = CGI.unescapeHTML(ds.content)
+    doc = Nokogiri::HTML(text)
+    
+    # html to story_display
+    stories = doc.xpath('//story')
+        
+    stories.each do |story|
+      solr_doc << Solr::Field.new(:story_display => story.children.to_xml)
+    end
+    
+    #strip out text and put in story_t
+    text_nodes = doc.xpath("//text()")
+    text = String.new
+    
+     text_nodes.each do |text_node|
+       text << text_node.content
+     end
+    
+     solr_doc << Solr::Field.new(:story_t => text)
+     
+     return solr_doc
+  end
+  
+end
+end

data/lib/solrizer/indexer.rb

@@ -0,0 +1,261 @@
+require 'solr'
+require 'solrizer/extractor'
+require 'solrizer/repository'
+
+
+module Solrizer
+class Indexer  
+  #
+  # Class variables
+  #
+  @@unique_id = 0
+
+  def self.unique_id
+    @@unique_id
+  end
+
+  #
+  # Member variables
+  #
+  attr_accessor :connection, :extractor, :index_full_text
+
+  #
+  # This method performs initialization tasks
+  #
+  def initialize( opts={} )
+    @@index_list = false unless defined?(@@index_list)
+    @extractor = Extractor.new
+    
+    if opts[:index_full_text] == true || opts[:index_full_text] == "true"
+      @index_full_text = true 
+    else
+      @index_full_text = false 
+    end
+    
+    connect
+  end
+
+  #
+  # This method connects to the Solr instance
+  #
+  def connect
+    
+    if ActiveFedora.fedora_config.empty?
+      ActiveFedora.init
+    end
+    
+    if defined?(Blacklight)
+      solr_config = Blacklight.solr_config
+    else
+      
+      if defined?(RAILS_ROOT)
+        config_path = File.join(RAILS_ROOT, "config")
+        yaml = YAML.load(File.open(File.join(config_path, "solr.yml")))
+        solr_config = yaml[RAILS_ENV]
+        puts solr_config.inspect
+      else
+        config_path = File.join(File.dirname(__FILE__), "..", "..", "config")
+        yaml = YAML.load(File.open(File.join(config_path, "solr.yml")))
+        
+        
+        if ENV["environment"].nil?
+          environment = "development"
+        else
+          environment = ENV["environment"]
+        end
+        
+        solr_config = yaml[environment]
+        puts solr_config.inspect
+      end
+      
+    end
+        
+    if index_full_text == true
+      url = solr_config['fulltext']['url']
+    elsif solr_config.has_key?("default")
+      url = solr_config['default']['url']
+    else
+      url = solr_config['url']
+    end
+    @connection = Solr::Connection.new(url, :autocommit => :on )
+  end
+
+  #
+  # This method extracts the facet categories from the given Fedora object's external tag datastream
+  #
+  def extract_xml_to_solr( obj, ds_name, solr_doc=Solr::Document.new )
+    xml_ds = Repository.get_datastream( obj, ds_name )
+    extractor.xml_to_solr( xml_ds.content, solr_doc )
+  end
+  
+  #
+  #
+  #
+  def extract_rels_ext( obj, ds_name, solr_doc=Solr::Document.new )
+    rels_ext_ds = Repository.get_datastream( obj, ds_name )
+    extractor.extract_rels_ext( rels_ext_ds.content, solr_doc )
+  end
+  
+  #
+  # This method generates the month and day facets from the date_t in solr_doc
+  #
+  
+  def generate_dates(solr_doc)
+    
+    # This will check for valid dates, but it seems most of the dates are currently invalid....
+    #date_check =  /^(19|20)\d\d([- \/.])(0[1-9]|1[012])\2(0[1-9]|[12][0-9]|3[01])/
+
+   #if there is not date_t, add on with easy-to-find value
+   if solr_doc[:date_t].nil?
+        solr_doc << Solr::Field.new( :date_t => "9999-99-99")
+   end #if
+
+    # unless date_check !~  solr_doc[:date_t]     
+    date_obj = Date._parse(solr_doc[:date_t])
+    
+    if date_obj[:mon].nil? 
+       solr_doc << Solr::Field.new(:month_facet => 99)
+    elsif 0 < date_obj[:mon] && date_obj[:mon] < 13
+      solr_doc << Solr::Field.new( :month_facet => date_obj[:mon].to_s.rjust(2, '0'))
+    else
+      solr_doc << Solr::Field.new( :month_facet => 99)
+    end
+      
+    if  date_obj[:mday].nil?
+      solr_doc << Solr::Field.new( :day_facet => 99)
+    elsif 0 < date_obj[:mday] && date_obj[:mday] < 32   
+      solr_doc << Solr::Field.new( :day_facet => date_obj[:mday].to_s.rjust(2, '0'))
+    else
+       solr_doc << Solr::Field.new( :day_facet => 99)
+    end
+    
+    return solr_doc
+#      end
+        
+  end
+  
+  
+  #
+  # This method creates a Solr-formatted XML document
+  #
+  def create_document( obj )        
+    
+    solr_doc = Solr::Document.new
+    
+    model_klazz_array = ActiveFedora::ContentModel.known_models_for( obj )
+    model_klazz_array.delete(ActiveFedora::Base)
+    
+    # If the object was passed in as an ActiveFedora::Base, call to_solr in order to get the base field entries from ActiveFedora::Base
+    # Otherwise, the object was passed in as a model instance other than ActiveFedora::Base,so call its to_solr method & allow it to insert the fields from ActiveFedora::Base
+    if obj.class == ActiveFedora::Base
+      solr_doc = obj.to_solr(solr_doc)
+      puts "  added base fields from #{obj.class.to_s}"
+    else
+      solr_doc = obj.to_solr(solr_doc)
+      model_klazz_array.delete(obj.class)
+      puts "    added base fields from #{obj.class.to_s} and model fields from #{obj.class.to_s}"
+    end
+   
+    # Load the object as an instance of each of its other models and get the corresponding solr fields
+    # Include :model_only=>true in the options in order to avoid adding the metadata from ActiveFedora::Base every time.
+    model_klazz_array.each do |klazz|
+      instance = klazz.load_instance(obj.pid)
+      solr_doc = instance.to_solr(solr_doc, :model_only=>true)
+      puts "  added solr fields from #{klazz.to_s}"
+    end
+    
+    solr_doc << Solr::Field.new( :id_t => "#{obj.pid}" )
+    solr_doc << Solr::Field.new( :id => "#{obj.pid}" ) unless solr_doc[:id]
+    
+    # increment the unique id to ensure that all documents in the search index are unique
+    @@unique_id += 1
+
+    return solr_doc
+  end
+
+  #
+  # This method adds a document to the Solr search index
+  #
+  def index( obj )
+   # print "Indexing '#{obj.pid}'..."
+    begin
+      
+      solr_doc = create_document( obj )
+      connection.add( solr_doc )
+ 
+     # puts connection.url
+     #puts solr_doc
+     #  puts "done"
+   
+    # rescue Exception => e
+    #    p "unable to index #{obj.pid}.  Failed with #{e.inspect}"
+    end
+   
+  end
+
+  #
+  # This method queries the Solr search index and returns a response
+  #
+  def query( query_str )
+    response = conn.query( query_str )
+  end
+
+  #
+  # This method prints out the results of the given query string by iterating through all the hits
+  #
+  def printResults( query_str )
+    query( query_str ) do |hit|
+      puts hit.inspect
+    end
+  end
+
+  #
+  # This method deletes a document from the Solr search index by id
+  #
+  def deleteDocument( id )
+    connection.delete( id )
+  end
+  
+  # Populates a solr doc with values from a hash.  
+  # Accepts two forms of hashes:
+  # => {'technology'=>["t1", "t2"], 'company'=>"c1", "person"=>["p1", "p2"]}
+  # or
+  # => {:facets => {'technology'=>["t1", "t2"], 'company'=>"c1", "person"=>["p1", "p2"]} }
+  #
+  # Note that values for individual fields can be a single string or an array of strings.
+  def self.solrize( input_hash, solr_doc=Solr::Document.new )    
+    facets = input_hash.has_key?(:facets) ? input_hash[:facets] : input_hash
+    facets.each_pair do |facet_name, value|
+      case value.class.to_s
+      when "String"
+        solr_doc << Solr::Field.new( :"#{facet_name}_facet" => "#{value}" )
+      when "Array"
+        value.each { |v| solr_doc << Solr::Field.new( :"#{facet_name}_facet" => "#{v}" ) } 
+      end
+    end
+    
+    if input_hash.has_key?(:symbols) 
+      input_hash[:symbols].each do |symbol_name, value|
+        case value.class.to_s
+        when "String"
+          solr_doc << Solr::Field.new( :"#{symbol_name}_s" => "#{value}" )
+	      when "Array"
+          value.each { |v| solr_doc << Solr::Field.new( :"#{symbol_name}_s" => "#{v}" ) } 
+        end
+      end
+    end
+    return solr_doc
+  end
+  
+
+  private :connect, :create_document
+
+  def class_exists?(class_name)
+    klass = Module.const_get(class_name)
+    return klass.is_a?(Class)
+  rescue NameError
+    return false
+  end
+  
+end
+end