solrizer 0.2.0 → 0.3.0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: solrizer
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Matt Zumwalt
@@ -15,57 +15,151 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-15 00:00:00 -05:00
+date: 2010-10-26 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: active-fedora
+  name: solr-ruby
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 25
+        hash: 3
         segments: 
-        - 1
-        - 1
-        - 5
-        version: 1.1.5
+        - 0
+        version: "0"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: om
+  name: nokogiri
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 23
+        hash: 3
         segments: 
-        - 1
-        - 0
         - 0
-        version: 1.0.0
+        version: "0"
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: om
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 13
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: mediashelf-loggable
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: ruby-debug-base
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - <
+      - !ruby/object:Gem::Version 
+        hash: 15
         segments: 
-        - 1
         - 2
-        - 9
-        version: 1.2.9
+        - 0
+        - 0
+        version: 2.0.0
   type: :development
-  version_requirements: *id003
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: mocha
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id010
 description: Use solrizer to populate solr indexes from Fedora repository content or from other sources.  You can run solrizer from within your apps, using the provided rake tasks, or as a JMS listener
 email: matt.zumwalt@yourmediashelf.com
 executables: []
@@ -77,6 +171,8 @@ extra_rdoc_files:
 - README.textile
 files: 
 - .gitignore
+- Gemfile
+- Gemfile.lock
 - History.txt
 - LICENSE
 - README.textile
@@ -86,34 +182,32 @@ files:
 - config/hydra_types.yml
 - config/solr.yml
 - config/solr_mappings.yml
+- config/solr_mappings_af_0.1.yml
 - lib/solrizer.rb
-- lib/solrizer/configuration.rb
 - lib/solrizer/extractor.rb
+- lib/solrizer/field_mapper.rb
 - lib/solrizer/field_name_mapper.rb
-- lib/solrizer/indexer.rb
-- lib/solrizer/main.rb
-- lib/solrizer/replicator.rb
-- lib/solrizer/repository.rb
+- lib/solrizer/html.rb
+- lib/solrizer/html/extractor.rb
 - lib/solrizer/xml.rb
+- lib/solrizer/xml/extractor.rb
 - lib/solrizer/xml/terminology_based_solrizer.rb
 - lib/tasks/solrizer.rake
 - solrizer.gemspec
+- spec/.rspec
 - spec/fixtures/druid-bv448hq0314-descMetadata.xml
 - spec/fixtures/druid-bv448hq0314-extProperties.xml
 - spec/fixtures/druid-cm234kq4672-extProperties.xml
 - spec/fixtures/druid-cm234kq4672-stories.xml
 - spec/fixtures/druid-hc513kw4806-descMetadata.xml
 - spec/fixtures/mods_articles/hydrangea_article1.xml
-- spec/fixtures/rels_ext_cmodel.xml
-- spec/fixtures/solr_mappings_af_0.1.yml
-- spec/integration/indexer_spec.rb
+- spec/fixtures/test_solr_mappings.yml
 - spec/rcov.opts
-- spec/spec.opts
 - spec/spec_helper.rb
 - spec/units/extractor_spec.rb
+- spec/units/field_mapper_spec.rb
 - spec/units/field_name_mapper_spec.rb
-- spec/units/indexer_spec.rb
-- spec/units/shelver_spec.rb
+- spec/units/xml_extractor_spec.rb
 - spec/units/xml_terminology_based_solrizer_spec.rb
 has_rdoc: true
 homepage: http://github.com/projecthydra/solrizer
@@ -150,10 +244,9 @@ signing_key:
 specification_version: 3
 summary: A utility for building solr indexes, usually from Fedora repository content.
 test_files: 
-- spec/integration/indexer_spec.rb
 - spec/spec_helper.rb
 - spec/units/extractor_spec.rb
+- spec/units/field_mapper_spec.rb
 - spec/units/field_name_mapper_spec.rb
-- spec/units/indexer_spec.rb
-- spec/units/shelver_spec.rb
+- spec/units/xml_extractor_spec.rb
 - spec/units/xml_terminology_based_solrizer_spec.rb

data/lib/solrizer/configuration.rb

@@ -1,8 +0,0 @@
-
-# FEDORA_URL = 'http://fedoraAdmin:fedoraAdmin@salt-dev.stanford.edu/fedora'
-# FEDORA_SOLR_URL = 'http://salt-dev.stanford.edu:8080/solr'
-# SHELVER_SOLR_URL = 'http://sulwebappdev1.stanford.edu:8100/salt_solr'
-FEDORA_URL = 'http://fedoraAdmin:fedoraAdmin@localhost:8080/fedora'
-FEDORA_SOLR_URL = 'http://localhost:8080/solr'
-SHELVER_SOLR_URL = 'http://localhost:8080/bl_solr'
-

data/lib/solrizer/indexer.rb

@@ -1,261 +0,0 @@
-require 'solr'
-require 'solrizer/extractor'
-require 'solrizer/repository'
-
-
-module Solrizer
-class Indexer  
-  #
-  # Class variables
-  #
-  @@unique_id = 0
-
-  def self.unique_id
-    @@unique_id
-  end
-
-  #
-  # Member variables
-  #
-  attr_accessor :connection, :extractor, :index_full_text
-
-  #
-  # This method performs initialization tasks
-  #
-  def initialize( opts={} )
-    @@index_list = false unless defined?(@@index_list)
-    @extractor = Extractor.new
-    
-    if opts[:index_full_text] == true || opts[:index_full_text] == "true"
-      @index_full_text = true 
-    else
-      @index_full_text = false 
-    end
-    
-    connect
-  end
-
-  #
-  # This method connects to the Solr instance
-  #
-  def connect
-    
-    if ActiveFedora.fedora_config.empty?
-      ActiveFedora.init
-    end
-    
-    if defined?(Blacklight)
-      solr_config = Blacklight.solr_config
-    else
-      
-      if defined?(RAILS_ROOT)
-        config_path = File.join(RAILS_ROOT, "config")
-        yaml = YAML.load(File.open(File.join(config_path, "solr.yml")))
-        solr_config = yaml[RAILS_ENV]
-        puts solr_config.inspect
-      else
-        config_path = File.join(File.dirname(__FILE__), "..", "..", "config")
-        yaml = YAML.load(File.open(File.join(config_path, "solr.yml")))
-        
-        
-        if ENV["environment"].nil?
-          environment = "development"
-        else
-          environment = ENV["environment"]
-        end
-        
-        solr_config = yaml[environment]
-        puts solr_config.inspect
-      end
-      
-    end
-        
-    if index_full_text == true
-      url = solr_config['fulltext']['url']
-    elsif solr_config.has_key?("default")
-      url = solr_config['default']['url']
-    else
-      url = solr_config['url']
-    end
-    @connection = Solr::Connection.new(url, :autocommit => :on )
-  end
-
-  #
-  # This method extracts the facet categories from the given Fedora object's external tag datastream
-  #
-  def extract_xml_to_solr( obj, ds_name, solr_doc=Solr::Document.new )
-    xml_ds = Repository.get_datastream( obj, ds_name )
-    extractor.xml_to_solr( xml_ds.content, solr_doc )
-  end
-  
-  #
-  #
-  #
-  def extract_rels_ext( obj, ds_name, solr_doc=Solr::Document.new )
-    rels_ext_ds = Repository.get_datastream( obj, ds_name )
-    extractor.extract_rels_ext( rels_ext_ds.content, solr_doc )
-  end
-  
-  #
-  # This method generates the month and day facets from the date_t in solr_doc
-  #
-  
-  def generate_dates(solr_doc)
-    
-    # This will check for valid dates, but it seems most of the dates are currently invalid....
-    #date_check =  /^(19|20)\d\d([- \/.])(0[1-9]|1[012])\2(0[1-9]|[12][0-9]|3[01])/
-
-   #if there is not date_t, add on with easy-to-find value
-   if solr_doc[:date_t].nil?
-        solr_doc << Solr::Field.new( :date_t => "9999-99-99")
-   end #if
-
-    # unless date_check !~  solr_doc[:date_t]     
-    date_obj = Date._parse(solr_doc[:date_t])
-    
-    if date_obj[:mon].nil? 
-       solr_doc << Solr::Field.new(:month_facet => 99)
-    elsif 0 < date_obj[:mon] && date_obj[:mon] < 13
-      solr_doc << Solr::Field.new( :month_facet => date_obj[:mon].to_s.rjust(2, '0'))
-    else
-      solr_doc << Solr::Field.new( :month_facet => 99)
-    end
-      
-    if  date_obj[:mday].nil?
-      solr_doc << Solr::Field.new( :day_facet => 99)
-    elsif 0 < date_obj[:mday] && date_obj[:mday] < 32   
-      solr_doc << Solr::Field.new( :day_facet => date_obj[:mday].to_s.rjust(2, '0'))
-    else
-       solr_doc << Solr::Field.new( :day_facet => 99)
-    end
-    
-    return solr_doc
-#      end
-        
-  end
-  
-  
-  #
-  # This method creates a Solr-formatted XML document
-  #
-  def create_document( obj )        
-    
-    solr_doc = Solr::Document.new
-    
-    model_klazz_array = ActiveFedora::ContentModel.known_models_for( obj )
-    model_klazz_array.delete(ActiveFedora::Base)
-    
-    # If the object was passed in as an ActiveFedora::Base, call to_solr in order to get the base field entries from ActiveFedora::Base
-    # Otherwise, the object was passed in as a model instance other than ActiveFedora::Base,so call its to_solr method & allow it to insert the fields from ActiveFedora::Base
-    if obj.class == ActiveFedora::Base
-      solr_doc = obj.to_solr(solr_doc)
-      puts "  added base fields from #{obj.class.to_s}"
-    else
-      solr_doc = obj.to_solr(solr_doc)
-      model_klazz_array.delete(obj.class)
-      puts "    added base fields from #{obj.class.to_s} and model fields from #{obj.class.to_s}"
-    end
-   
-    # Load the object as an instance of each of its other models and get the corresponding solr fields
-    # Include :model_only=>true in the options in order to avoid adding the metadata from ActiveFedora::Base every time.
-    model_klazz_array.each do |klazz|
-      instance = klazz.load_instance(obj.pid)
-      solr_doc = instance.to_solr(solr_doc, :model_only=>true)
-      puts "  added solr fields from #{klazz.to_s}"
-    end
-    
-    solr_doc << Solr::Field.new( :id_t => "#{obj.pid}" )
-    solr_doc << Solr::Field.new( :id => "#{obj.pid}" ) unless solr_doc[:id]
-    
-    # increment the unique id to ensure that all documents in the search index are unique
-    @@unique_id += 1
-
-    return solr_doc
-  end
-
-  #
-  # This method adds a document to the Solr search index
-  #
-  def index( obj )
-   # print "Indexing '#{obj.pid}'..."
-    begin
-      
-      solr_doc = create_document( obj )
-      connection.add( solr_doc )
- 
-     # puts connection.url
-     #puts solr_doc
-     #  puts "done"
-   
-    # rescue Exception => e
-    #    p "unable to index #{obj.pid}.  Failed with #{e.inspect}"
-    end
-   
-  end
-
-  #
-  # This method queries the Solr search index and returns a response
-  #
-  def query( query_str )
-    response = conn.query( query_str )
-  end
-
-  #
-  # This method prints out the results of the given query string by iterating through all the hits
-  #
-  def printResults( query_str )
-    query( query_str ) do |hit|
-      puts hit.inspect
-    end
-  end
-
-  #
-  # This method deletes a document from the Solr search index by id
-  #
-  def deleteDocument( id )
-    connection.delete( id )
-  end
-  
-  # Populates a solr doc with values from a hash.  
-  # Accepts two forms of hashes:
-  # => {'technology'=>["t1", "t2"], 'company'=>"c1", "person"=>["p1", "p2"]}
-  # or
-  # => {:facets => {'technology'=>["t1", "t2"], 'company'=>"c1", "person"=>["p1", "p2"]} }
-  #
-  # Note that values for individual fields can be a single string or an array of strings.
-  def self.solrize( input_hash, solr_doc=Solr::Document.new )    
-    facets = input_hash.has_key?(:facets) ? input_hash[:facets] : input_hash
-    facets.each_pair do |facet_name, value|
-      case value.class.to_s
-      when "String"
-        solr_doc << Solr::Field.new( :"#{facet_name}_facet" => "#{value}" )
-      when "Array"
-        value.each { |v| solr_doc << Solr::Field.new( :"#{facet_name}_facet" => "#{v}" ) } 
-      end
-    end
-    
-    if input_hash.has_key?(:symbols) 
-      input_hash[:symbols].each do |symbol_name, value|
-        case value.class.to_s
-        when "String"
-          solr_doc << Solr::Field.new( :"#{symbol_name}_s" => "#{value}" )
-	      when "Array"
-          value.each { |v| solr_doc << Solr::Field.new( :"#{symbol_name}_s" => "#{v}" ) } 
-        end
-      end
-    end
-    return solr_doc
-  end
-  
-
-  private :connect, :create_document
-
-  def class_exists?(class_name)
-    klass = Module.const_get(class_name)
-    return klass.is_a?(Class)
-  rescue NameError
-    return false
-  end
-  
-end
-end

data/lib/solrizer/main.rb

@@ -1,17 +0,0 @@
-#!/bin/env ruby
-
-@index_full_text = false
-
-require 'rubygems'
-load 'configuration.rb'  
-load 'repository.rb'  
-load 'solrizer.rb'  
-
-# initialize connection to Fedora repository
-repository = Repository.new
-repository.initialize_repository
-
-# solrize all objects in the Fedora repository
-solrizer = Solrizer.new
-solrizer.solrize_objects
-

data/lib/solrizer/replicator.rb

@@ -1,143 +0,0 @@
-require 'fastercsv'
-REPLICATOR_LIST = false unless defined?(REPLICATOR_LIST)
-
-
-module Solrizer
-  class Replicator
-    
-    include Stanford::SaltControllerHelper
-    attr_accessor :dest_repo, :configs
-    
-    def initialize
-      config_path = "#{RAILS_ROOT}/config/replicator.yml"
-      raw_configs = YAML::load(File.open(config_path))
-      @configs = raw_configs[RAILS_ENV]
-      @dest_repo = Fedora::Repository.new(configs["destination"]["fedora"]["url"])
-      
-      ActiveFedora.fedora_config[:url] = configs["source"]["fedora"]["url"]
-      logger.info("REPLICATOR: re-initializing Fedora with fedora_config: #{ActiveFedora.fedora_config.inspect}")
- 
-      Fedora::Repository.register(ActiveFedora.fedora_config[:url])
-      logger.info("REPLICATOR: re-initialized Fedora as: #{Fedora::Repository.instance.inspect}")
-      
-      # Register Solr
-      ActiveFedora.solr_config[:url] = configs["source"]["solr"]["url"]
-      
-      logger.info("REPLICATOR: re-initializing ActiveFedora::SolrService with solr_config: #{ActiveFedora.solr_config.inspect}")
- 
-      ActiveFedora::SolrService.register(ActiveFedora.solr_config[:url])
-      
-    end
-    
-    def replicate_objects
-     # retrieve a list of all the pids in the fedora repository
-      num_docs = 1000000   # modify this number to guarantee that all the objects are retrieved from the repository
-
-      if REPLICATOR_LIST == false
-
-         pids = Repository.get_pids( num_docs )
-         puts "Replicating #{pids.length} Fedora objects"
-          pids.each do |pid|
-            unless pid[0].empty? || pid[0].nil? || !pid[0].include?("druid:")
-              puts "Processing #{pid}"
-              replicate_object( pid )
-            end #unless
-          end #pids.each
-
-      else
-
-         if File.exists?(REPLICATOR_LIST)
-            arr_of_pids = FasterCSV.read(REPLICATOR_LIST, :headers=>false)
-
-            puts "Replicating from list at #{REPLICATOR_LIST}"
-            puts "Replicating #{arr_of_pids.length} Fedora objects"
-
-           arr_of_pids.each do |row|
-              pid = row[0]
-              replicate_object( pid )   
-           end #FASTERCSV
-           
-          else
-            puts "#{REPLICATOR_LIST} does not exists!"
-          end #if File.exists
-
-      end #if Index_LISTS
-    end #replicate_objects
-
- 
-    def replicate_object(obj)
-	#source_doc = Document.load_instance(pid)
-       obj = obj.kind_of?(ActiveFedora::Base) ? obj : Repository.get_object( obj )
-	     p "Indexing object #{obj.pid} with label #{obj.label}"
-      begin
-        unless obj.nil?
-		      create_stub(obj)
-        	p "Successfully replicated #{obj.pid}"
-   	    end  
-      rescue Exception => e
-        p "unable to create stub.  Failed with #{e.inspect}"
-      end
-    end
-    
-    # Creates a stub object in @dest_repo with the datastreams that we need in the stubs
-    def create_stub(source_object)
-      
-      begin
-        
-       jp2 = downloadables(source_object, :canonical=>true, :mime_type=>"image/jp2")   
-       jp2.new_object = true
-       jp2.control_group = 'M'
-       jp2.blob = jp2.content
-      
-       	stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
-       	dest_repo.save(stub_object)   
-	      dest_repo.save(jp2)
-      
-      ["properties", "extProperties", "descMetadata", "location"].each do |ds_name|
-        ds = source_object.datastreams[ds_name]
-        ds.new_object = true
-        ds.blob = ds.content
-        dest_repo.save(ds)
-      end
-     
-     rescue
-         #for object without jp2s
-         #this is a temp fix to the downloadables() issue
-         
-         
-         pid = source_object.pid 
-	        p "> #{pid}"
-        
-          jp2_file = File.new('spec/fixtures/image.jp2')
-          ds = ActiveFedora::Datastream.new(:dsID => "image.jp2", :dsLabel => 'image.jp2', :controlGroup => 'M', :blob => jp2_file)
-	        source_object.add_datastream(ds)
-          source_object.save 
-	        #  source_object = Document.load_instance(pid)
- 	        source_object = ActiveFedora::Base.load_instance(pid)
-       	  stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
-          dest_repo.save(stub_object)
-         
-          jp2 = downloadables(source_object, :canonical=>true, :mime_type=>"image/jp2")   
-          jp2.new_object = true
-          jp2.control_group = 'M'
-          jp2.blob = jp2.content
-
-          	stub_object = Fedora::FedoraObject.new(:pid=>source_object.pid)
-          	dest_repo.save(stub_object)   
-   	      dest_repo.save(jp2)
-
-         ["properties", "extProperties", "descMetadata", "location"].each do |ds_name|
-           ds = source_object.datastreams[ds_name]
-           ds.new_object = true
-           ds.blob = ds.content
-           dest_repo.save(ds)
-         end  
-    
-      end    
-    end
-    def logger
-      @logger ||= defined?(RAILS_DEFAULT_LOGGER) ? RAILS_DEFAULT_LOGGER : Logger.new(STDOUT)
-    end
-    
-  end
-end