solrizer 0.1.0.pre2

data/lib/solrizer/main.rb

@@ -0,0 +1,17 @@
+#!/bin/env ruby
+
+@index_full_text = false
+
+require 'rubygems'
+load 'configuration.rb'  
+load 'repository.rb'  
+load 'solrizer.rb'  
+
+# initialize connection to Fedora repository
+repository = Repository.new
+repository.initialize_repository
+
+# solrize all objects in the Fedora repository
+solrizer = Solrizer.new
+solrizer.solrize_objects
+

data/lib/solrizer/replicator.rb

@@ -0,0 +1,143 @@
+require 'fastercsv'
+REPLICATOR_LIST = false unless defined?(REPLICATOR_LIST)
+
+
+module Solrizer
+  class Replicator
+    
+    include Stanford::SaltControllerHelper
+    attr_accessor :dest_repo, :configs
+    
+    def initialize
+      config_path = "#{RAILS_ROOT}/config/replicator.yml"
+      raw_configs = YAML::load(File.open(config_path))
+      @configs = raw_configs[RAILS_ENV]
+      @dest_repo = Fedora::Repository.new(configs["destination"]["fedora"]["url"])
+      
+      ActiveFedora.fedora_config[:url] = configs["source"]["fedora"]["url"]
+      logger.info("REPLICATOR: re-initializing Fedora with fedora_config: #{ActiveFedora.fedora_config.inspect}")
+ 
+      Fedora::Repository.register(ActiveFedora.fedora_config[:url])
+      logger.info("REPLICATOR: re-initialized Fedora as: #{Fedora::Repository.instance.inspect}")
+      
+      # Register Solr
+      ActiveFedora.solr_config[:url] = configs["source"]["solr"]["url"]
+      
+      logger.info("REPLICATOR: re-initializing ActiveFedora::SolrService with solr_config: #{ActiveFedora.solr_config.inspect}")
+ 
+      ActiveFedora::SolrService.register(ActiveFedora.solr_config[:url])
+      
+    end
+    
+    def replicate_objects
+     # retrieve a list of all the pids in the fedora repository
+      num_docs = 1000000   # modify this number to guarantee that all the objects are retrieved from the repository
+
+      if REPLICATOR_LIST == false
+
+         pids = Repository.get_pids( num_docs )
+         puts "Replicating #{pids.length} Fedora objects"
+          pids.each do |pid|
+            unless pid[0].empty? || pid[0].nil? || !pid[0].include?("druid:")
+              puts "Processing #{pid}"
+              replicate_object( pid )
+            end #unless
+          end #pids.each
+
+      else
+
+         if File.exists?(REPLICATOR_LIST)
+            arr_of_pids = FasterCSV.read(REPLICATOR_LIST, :headers=>false)
+
+            puts "Replicating from list at #{REPLICATOR_LIST}"
+            puts "Replicating #{arr_of_pids.length} Fedora objects"
+
+           arr_of_pids.each do |row|
+              pid = row[0]
+              replicate_object( pid )   
+           end #FASTERCSV
+           
+          else
+            puts "#{REPLICATOR_LIST} does not exists!"
+          end #if File.exists
+
+      end #if Index_LISTS
+    end #replicate_objects
+
+ 
+    def replicate_object(obj)
+	#source_doc = Document.load_instance(pid)
+       obj = obj.kind_of?(ActiveFedora::Base) ? obj : Repository.get_object( obj )
+	     p "Indexing object #{obj.pid} with label #{obj.label}"
+      begin
+        unless obj.nil?
+		      create_stub(obj)
+        	p "Successfully replicated #{obj.pid}"
+   	    end  
+      rescue Exception => e
+        p "unable to create stub.  Failed with #{e.inspect}"
+      end
+    end
+    
+    # Creates a stub object in @dest_repo with the datastreams that we need in the stubs
+    def create_stub(source_object)
+      
+      begin
+        
+       jp2 = downloadables(source_object, :canonical=>true, :mime_type=>"image/jp2")   
+       jp2.new_object = true
+       jp2.control_group = 'M'
+       jp2.blob = jp2.content
+      
+       	stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
+       	dest_repo.save(stub_object)   
+	      dest_repo.save(jp2)
+      
+      ["properties", "extProperties", "descMetadata", "location"].each do |ds_name|
+        ds = source_object.datastreams[ds_name]
+        ds.new_object = true
+        ds.blob = ds.content
+        dest_repo.save(ds)
+      end
+     
+     rescue
+         #for object without jp2s
+         #this is a temp fix to the downloadables() issue
+         
+         
+         pid = source_object.pid 
+	        p "> #{pid}"
+        
+          jp2_file = File.new('spec/fixtures/image.jp2')
+          ds = ActiveFedora::Datastream.new(:dsID => "image.jp2", :dsLabel => 'image.jp2', :controlGroup => 'M', :blob => jp2_file)
+	        source_object.add_datastream(ds)
+          source_object.save 
+	        #  source_object = Document.load_instance(pid)
+ 	        source_object = ActiveFedora::Base.load_instance(pid)
+       	  stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
+          dest_repo.save(stub_object)
+         
+          jp2 = downloadables(source_object, :canonical=>true, :mime_type=>"image/jp2")   
+          jp2.new_object = true
+          jp2.control_group = 'M'
+          jp2.blob = jp2.content
+
+          	stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
+          	dest_repo.save(stub_object)   
+   	      dest_repo.save(jp2)
+
+         ["properties", "extProperties", "descMetadata", "location"].each do |ds_name|
+           ds = source_object.datastreams[ds_name]
+           ds.new_object = true
+           ds.blob = ds.content
+           dest_repo.save(ds)
+         end  
+    
+      end    
+    end
+    def logger
+      @logger ||= defined?(RAILS_DEFAULT_LOGGER) ? RAILS_DEFAULT_LOGGER : Logger.new(STDOUT)
+    end
+    
+  end
+end

data/lib/solrizer/repository.rb

@@ -0,0 +1,54 @@
+
+require 'active-fedora'
+
+
+module Solrizer
+class Repository
+
+  #
+  # This method initializes the fedora repository and solr instance
+  #
+  def initialize_repository
+    Fedora::Repository.register( FEDORA_URL )
+    ActiveFedora::SolrService.register( FEDORA_SOLR_URL )
+  end
+
+  #
+  # This method retrieves a comprehensive list of unique ids in the fedora repository
+  #
+  def self.get_pids( num_docs )
+    solr_results = ActiveFedora::SolrService.instance.conn.query( "active_fedora_model_field:Document", { :rows => num_docs } )
+    id_array = []
+    solr_results.hits.each do |hit|
+      id_array << hit[SOLR_DOCUMENT_ID]
+    end
+    return id_array
+  end
+  
+  #
+  # This method retrieves the object associated with the given unique id
+  #
+  def self.get_object( pid )
+    object = ActiveFedora::Base.load_instance( pid )
+  end
+  
+  #
+  # This method retrieves a comprehensive list of datastreams for the given object
+  #
+  def self.get_datastreams( obj )
+    ds_keys = obj.datastreams.keys
+  end
+  
+  #
+  # This method retrieves the datastream for the given object with the given datastream name
+  #
+  def self.get_datastream( obj, ds_name )
+    begin
+      obj.datastreams[ ds_name ]
+    rescue
+      return nil
+    end
+  end
+
+end
+end

data/lib/tasks/solrizer.rake

@@ -0,0 +1,33 @@
+namespace :solrizer do
+  
+  desc 'Index a fedora object of the given pid.'
+  task :solrize => :environment do 
+    index_full_text = ENV['FULL_TEXT'] == 'true'
+    if ENV['PID']
+      puts "indexing #{ENV['PID'].inspect}"
+      solrizer = Solrizer::Solrizer.new :index_full_text=> index_full_text
+      solrizer.solrize(ENV['PID'])
+      puts "Finished shelving #{ENV['PID']}"
+    else
+      puts "You must provide a pid using the format 'solrizer::solrize_object PID=sample:pid'."
+    end
+  end
+  
+  desc 'Index all objects in the repository.'
+  task :solrize_objects => :environment do
+    index_full_text = ENV['FULL_TEXT'] == 'true'
+    if ENV['INDEX_LIST']
+      @@index_list = ENV['INDEX_LIST']
+    end
+    
+    puts "Re-indexing Fedora Repository."
+    puts "Fedora URL: #{ActiveFedora.fedora_config[:url]}"
+    puts "Fedora Solr URL: #{ActiveFedora.solr_config[:url]}"
+    puts "Blacklight Solr Config: #{Blacklight.solr_config.inspect}"
+    puts "Doing full text index." if index_full_text
+    solrizer = Solrizer::Solrizer.new :index_full_text=> index_full_text
+    solrizer.solrize_objects
+    puts "Solrizer task complete."
+  end  
+  
+end

data/solrizer.gemspec

@@ -0,0 +1,80 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{solrizer}
+  s.version = "0.1.0.pre2"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Matt Zumwalt"]
+  s.date = %q{2010-05-15}
+  s.description = %q{Use solrizer to populate solr indexes from Fedora repository content or from other sources.  You can run solrizer from within your apps, using the provided rake tasks, or as a JMS listener}
+  s.email = %q{matt.zumwalt@yourmediashelf.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "config/fedora.yml",
+     "config/hydra_types.yml",
+     "config/solr.yml",
+     "lib/solrizer.rb",
+     "lib/solrizer/configuration.rb",
+     "lib/solrizer/extractor.rb",
+     "lib/solrizer/indexer.rb",
+     "lib/solrizer/main.rb",
+     "lib/solrizer/replicator.rb",
+     "lib/solrizer/repository.rb",
+     "lib/tasks/solrizer.rake",
+     "solrizer.gemspec",
+     "spec/fixtures/druid-bv448hq0314-descMetadata.xml",
+     "spec/fixtures/druid-bv448hq0314-extProperties.xml",
+     "spec/fixtures/druid-cm234kq4672-extProperties.xml",
+     "spec/fixtures/druid-cm234kq4672-stories.xml",
+     "spec/fixtures/druid-hc513kw4806-descMetadata.xml",
+     "spec/fixtures/rels_ext_cmodel.xml",
+     "spec/integration/indexer_spec.rb",
+     "spec/rcov.opts",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/units/extractor_spec.rb",
+     "spec/units/indexer_spec.rb",
+     "spec/units/shelver_spec.rb"
+  ]
+  s.homepage = %q{http://github.com/projecthydra/solrizer}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{A utility for building solr indexes, usually from Fedora repository content.}
+  s.test_files = [
+    "spec/integration/indexer_spec.rb",
+     "spec/spec_helper.rb",
+     "spec/units/extractor_spec.rb",
+     "spec/units/indexer_spec.rb",
+     "spec/units/shelver_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<active-fedora>, ["> 1.1.3"])
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+    else
+      s.add_dependency(%q<active-fedora>, ["> 1.1.3"])
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    end
+  else
+    s.add_dependency(%q<active-fedora>, ["> 1.1.3"])
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+  end
+end
+

data/spec/fixtures/druid-bv448hq0314-descMetadata.xml

@@ -0,0 +1,11 @@
+<dc xmlns:dcterms="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <dcterms:type xsi:type="DCMITYPE">text</dcterms:type>
+  <dcterms:medium>Paper Document</dcterms:medium>
+  <dcterms:rights>Presumed under copyright. Do not publish.</dcterms:rights>
+  <dcterms:date>1985-12-30</dcterms:date>
+  <dcterms:format>application/tiff</dcterms:format>
+  <dcterms:format>application/jp2000</dcterms:format>
+  <dcterms:format>application/pdf</dcterms:format>
+  <dcterms:title>This is a Sample Title</dcterms:title>
+  <dcterms:publisher>Sample Unversity</dcterms:publisher>
+</dc>

data/spec/fixtures/druid-bv448hq0314-extProperties.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <attributes>
+    <attribute type="item">5958</attribute>
+    <attribute type="objectid">FEI0010-00013142</attribute>
+    <attribute type="title">Letter from Ellie Engelmore to Professor K. C. Reddy</attribute>
+    <attribute type="copyright">Presumed under copyright. Do not publish.</attribute>
+    <attribute type="description"/>
+    <attribute type="date">1985-12-30</attribute>
+    <attribute type="datestr">30/12/1985</attribute>
+    <attribute type="docurl">https://www.stanford.edu/group/salt_project/SLA/Feigenbaum/eaf7000/pdf/00013142.pdf</attribute>
+    <attribute type="doctn">http://www.stanford.edu/group/salt_project/cgi-bin/SLA/Feigenbaum/eaf7000/png/small_00013142.png</attribute>
+    <attribute type="url"/>
+    <attribute type="industryterm"/>
+    <attribute type="technology">artificial intelligence</attribute>
+    <attribute type="company"/>
+    <attribute type="person">ELLIE ENGELMORE</attribute>
+    <attribute type="year">1985</attribute>
+    <attribute type="organization">Mathematics and Computer/Information Sciences University of Hyderabad Central University P. O. Hyder</attribute>
+    <attribute type="sourcelocation">Folder 15</attribute>
+  </attributes>
+  <facets>
+    <facet type="year" id="49">1980s</facet>
+    <facet type="year" id="49">1985</facet>
+    <facet type="year" id="42">1980s</facet>
+    <facet type="sourcelocation" id="592">Feigenbaum</facet>
+    <facet type="sourcelocation" id="592">eaf7000</facet>
+    <facet type="sourcelocation" id="592">Box 51A</facet>
+    <facet type="sourcelocation" id="594">Feigenbaum</facet>
+    <facet type="sourcelocation" id="594">eaf7000</facet>
+    <facet type="sourcelocation" id="594">Box 51A</facet>
+    <facet type="sourcelocation" id="594">Folder 15</facet>
+    <facet type="sourcelocation" id="691">Feigenbaum</facet>
+    <facet type="sourcelocation" id="692">Feigenbaum</facet>
+    <facet type="sourcelocation" id="692">eaf7000</facet>
+    <facet type="doctype" id="32">Correspondence</facet>
+    <facet type="city" id="82">Ann Arbor</facet>
+    <facet type="city" id="910">Hyderabad</facet>
+    <facet type="city" id="1519">Palo Alto</facet>
+    <facet type="country" id="68">India</facet>
+    <facet type="emailaddress" id="288">EENGELMORE@SUMEX-AIM.ARPA</facet>
+    <facet type="organization" id="5065">Heuristic Programming Project</facet>
+    <facet type="organization" id="7012">Mathematics and Computer/Information Sciences University of Hyderabad Central University P. O. Hyder</facet>
+    <facet type="organization" id="8878">Professor K. C. Reddy School of Mathematics and Computer/Information Sciences</facet>
+    <facet type="person" id="5810">ELLIE ENGELMORE</facet>
+    <facet type="person" id="17934">Reddy</facet>
+    <facet type="person" id="5787">EDWARD FEIGENBAUM</facet>
+    <facet type="provinceorstate" id="96">Michigan</facet>
+    <facet type="provinceorstate" id="27">California</facet>
+    <facet type="technology" id="1713">artificial intelligence</facet>
+  </facets>
+</document>

data/spec/fixtures/druid-cm234kq4672-extProperties.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document>
+ <attributes><attribute type="item">4290</attribute><attribute type="objectid">FEI0010-00011325</attribute><attribute type="title">Letter from Ellie Engelmore to Wemara Lichty</attribute><attribute type="copyright">Presumed under copyright. Do not publish.</attribute><attribute type="description"/><attribute type="date">1984-6-4</attribute><attribute type="datestr">4/6/1984</attribute><attribute type="docurl">https://www.stanford.edu/group/salt_project/SLA/Feigenbaum/eaf7000/pdf/00011325.pdf</attribute><attribute type="doctn">http://www.stanford.edu/group/salt_project/cgi-bin/SLA/Feigenbaum/eaf7000/png/small_00011325.png</attribute><attribute type="url"/><attribute type="industryterm"/><attribute type="technology">artificial intelligence</attribute><attribute type="company"/><attribute type="person">A. FEIGENBAUM</attribute><attribute type="year">1984</attribute><attribute type="organization">McAlcster Hall University</attribute><attribute type="sourcelocation">Folder 5</attribute></attributes>
+ <facets><facet type="year" id="48">1980s</facet><facet type="year" id="48">1984</facet><facet type="year" id="42">1980s</facet><facet type="sourcelocation" id="578">Feigenbaum</facet><facet type="sourcelocation" id="578">eaf7000</facet><facet type="sourcelocation" id="578">Box 51</facet><facet type="sourcelocation" id="587">Feigenbaum</facet><facet type="sourcelocation" id="587">eaf7000</facet><facet type="sourcelocation" id="587">Box 51</facet><facet type="sourcelocation" id="587">Folder 5</facet><facet type="sourcelocation" id="692">Feigenbaum</facet><facet type="sourcelocation" id="692">eaf7000</facet><facet type="sourcelocation" id="691">Feigenbaum</facet><facet type="doctype" id="32">Correspondence</facet><facet type="city" id="1948">Stanford</facet><facet type="country" id="33">Columbia</facet><facet type="facility" id="2551">U. Missouri library</facet><facet type="organization" id="5065">Heuristic Programming Project</facet><facet type="organization" id="7026">McAlcster Hall University</facet><facet type="organization" id="9645">STANFORD UNIVERSITY</facet><facet type="organization" id="11964">University of Missouri</facet><facet type="organization" id="12407">Wemara Lichty Psychology Department</facet><facet type="person" id="15650">Morton Hunt</facet><facet type="person" id="37">A. FEIGENBAUM</facet><facet type="person" id="5810">ELLIE ENGELMORE</facet><facet type="provinceorstate" id="27">California</facet><facet type="provinceorstate" id="100">Missouri</facet><facet type="publishedmedium" id="44">Artificial Intelligence</facet><facet type="technology" id="1713">artificial intelligence</facet></facets>
+</document>

data/spec/fixtures/druid-cm234kq4672-stories.xml

@@ -0,0 +1,17 @@
+
+<html>
+  <body>
+    <pre>
+This is
+preformatted text.
+It preserves      both spaces
+and line breaks.
+</pre>
+    <p>The pre tag is good for displaying computer code:</p>
+    <pre>
+for i = 1 to 10
+     print i
+next i
+</pre>
+  </body>
+</html>

data/spec/fixtures/druid-hc513kw4806-descMetadata.xml

@@ -0,0 +1,11 @@
+<dc xmlns:dcterms="http://purl.org/dc/terms/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <dcterms:type xsi:type="DCMITYPE">text</dcterms:type>
+  <dcterms:medium>Paper Document</dcterms:medium>
+  <dcterms:rights>Copyright © 2006 All rights reserved. Distribution for commercial purposes is prohibited.</dcterms:rights>
+  <dcterms:date/>
+  <dcterms:format>application/tiff</dcterms:format>
+  <dcterms:format>application/jp2000</dcterms:format>
+  <dcterms:format>application/pdf</dcterms:format>
+  <dcterms:title>The Rise and Fall of the YouTube Empire</dcterms:title>
+  <dcterms:publisher>Sample Unversity</dcterms:publisher>
+</dc>

data/spec/fixtures/rels_ext_cmodel.xml

@@ -0,0 +1,8 @@
+<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+	<rdf:Description rdf:about="info:fedora/demo:multipurpose-objects-model_and_sdef">
+		<hasModel xmlns="info:fedora/fedora-system:def/model#" rdf:resource="info:fedora/fedora-system:ContentModel-3.0"/>
+		<hasModel xmlns="info:fedora/fedora-system:def/model#" rdf:resource="info:fedora/afmodel:SaltDocument"/>
+		<hasModel xmlns="info:fedora/fedora-system:def/model#" rdf:resource="info:fedora/afmodel:JP2Document"/>
+		<hasModel xmlns="info:fedora/fedora-system:def/model#" rdf:resource="info:fedora/afmodel:DCDocument"/>
+	</rdf:Description>
+</rdf:RDF>

data/spec/integration/indexer_spec.rb

@@ -0,0 +1,18 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'solrizer'
+
+describe Solrizer::Indexer do
+  
+  before(:each) do
+    @indexer = Solrizer::Indexer.new
+  end
+  
+  describe "index" do
+    it "should update solr with the metadata from the given object" do
+      pending "Got to decide if/how to handle fixtures in this gem. Probably should just mock out Fedora & Solr entirely."
+      obj = Solrizer::Repository.get_object( "druid:sb733gr4073" )
+      @indexer.index( obj )
+    end
+  end
+  
+end