cul-fedora 0.8.6 → 1.0.0

data/VERSION

@@ -1 +1 @@
-0.8.6
+1.0.0

data/cul-fedora.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cul-fedora}
-  s.version = "0.8.6"
+  s.version = "1.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["James Stuart"]
-  s.date = %q{2011-07-14}
+  s.authors = [%q{James Stuart}]
+  s.date = %q{2011-09-19}
   s.description = %q{Columbia-specific Fedora libraries}
   s.email = %q{tastyhat@jamesstuart.org}
   s.extra_rdoc_files = [
@@ -62,19 +62,11 @@ Gem::Specification.new do |s|
     "test_fedora_item.rb"
   ]
   s.homepage = %q{http://github.com/tastyhat/cul-fedora}
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.6}
   s.summary = %q{Columbia University Fedora Hooks}
-  s.test_files = [
-    "test/helper.rb",
-    "test/test_cul-fedora.rb",
-    "test/test_fedora_item.rb",
-    "test/test_fedora_server.rb",
-    "test/test_fedora_solr.rb"
-  ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then

data/lib/cul-fedora.rb

@@ -1,6 +1,10 @@
 require "httpclient"
 require "nokogiri"
-require "activesupport"
+begin
+  require "active_support"
+rescue
+  require "activesupport"
+end
 require "rsolr"
 require "rsolr-ext"
 require "open3"

data/lib/cul-fedora/item.rb

@@ -1,4 +1,9 @@
 require "open3"
+begin
+  require "active_support/core_ext/array/extract_options"
+rescue
+  require "activesupport"
+end
 
 module Cul
   module Fedora
@@ -40,6 +45,7 @@ module Cul
           request
           return true
         rescue Exception => e # we should really do some better checking of error type etc here
+          logger.error e.message
           return false
         end
       end
@@ -82,7 +88,8 @@ module Cul
             i = i + MAX_LIST_MEMBERS_PER_REQUEST
           end
           return items
-        rescue
+        rescue Exception => e
+          logger.error e.message
           []
         end
       end
@@ -90,8 +97,9 @@ module Cul
       def getSize()
         begin
           request(:method => "/objects", :sdef => "methods/ldpd:sdef.Aggregator", :request => "getSize").to_i
-        rescue
-          -1
+        rescue Exception => e
+          logger.error e.message
+          return -1
         end
       end
 
@@ -103,6 +111,7 @@ module Cul
             @server.item(metadata.attributes["uri"].value)
           end
         rescue Exception => e
+          logger.error e.message
           []
         end
       end
@@ -113,7 +122,8 @@ module Cul
           result.xpath("/rdf:RDF/rdf:Description/*[local-name()='memberOf']").collect do |member|
             @server.item(member.attributes["resource"].value)
           end
-        rescue
+        rescue Exception => e
+          logger.error e.message
           []
         end
       end
@@ -136,6 +146,7 @@ module Cul
         author_roles = ["author","creator","editor","speaker","moderator","interviewee","interviewer","contributor"]
         other_name_roles = ["thesis advisor"]
         corporate_author_roles = ["author"]
+        corporate_department_roles = ["originator"]
 
         organizations = []
         departments = []
@@ -182,7 +193,7 @@ module Cul
                 note_org = true
                 all_author_names << fullname
                 if(!name_node["ID"].nil?)
-                  add_field.call("author_id_uni", name_node["ID"])
+                  add_field.call("author_uni", name_node["ID"])
                 end
                 add_field.call("author_search", fullname.downcase)
                 add_field.call("author_facet", fullname)
@@ -209,7 +220,7 @@ module Cul
             end
             
             mods.css("name[@type='corporate']").each do |corp_name_node|
-              if(!corp_name_node["ID"].nil? && corp_name_node["ID"].include?("originator"))
+              if((!corp_name_node["ID"].nil? && corp_name_node["ID"].include?("originator")) || corp_name_node.css("role>roleTerm").collect(&:content).any? { |role| corporate_department_roles.include?(role) })
                 name_part = corp_name_node.at_css("namePart").text
                 if(name_part.include?(". "))
                   name_part_split = name_part.split(". ")
@@ -230,13 +241,12 @@ module Cul
               end
             end
 
-            add_field.call("authors_display",all_author_names.join("; "))
-            add_field.call("pub_date", mods.at_css("*[@keyDate='yes']"))
+            add_field.call("author_display",all_author_names.join("; "))
+            add_field.call("pub_date_facet", mods.at_css("*[@keyDate='yes']"))
 
             mods.css("genre").each do |genre_node|
               add_field.call("genre_facet", genre_node)
               add_field.call("genre_search", genre_node)
-
             end
 
 
@@ -247,14 +257,14 @@ module Cul
               if(subject_node.attributes.count == 0)
                 subject_node.css("topic").each do |topic_node|
                   add_field.call("keyword_search", topic_node.content.downcase)
-                  add_field.call("subject", topic_node)
+                  add_field.call("subject_facet", topic_node)
                   add_field.call("subject_search", topic_node)
                 end
               end
             end
 
 
-            add_field.call("tableOfContents", mods.at_css("tableOfContents"))
+            add_field.call("table_of_contents", mods.at_css("tableOfContents"))
 
             mods.css("note").each { |note| add_field.call("notes", note) }
 
@@ -277,7 +287,7 @@ module Cul
             
             if(related_series = mods.at_css("relatedItem[@type='series']"))
               if(related_series.has_attribute?("ID"))
-                add_field.call("series", related_series.at_css("titleInfo>title"))
+                add_field.call("series_facet", related_series.at_css("titleInfo>title"))
               end
             end
 
@@ -290,23 +300,24 @@ module Cul
 
             mods.css("typeOfResource").each { |tr| add_field.call("type_of_resource_facet", tr)}
             mods.css("subject>geographic").each do |geo|
-              add_field.call("geographic_area", geo)
+              add_field.call("geographic_area_display", geo)
               add_field.call("geographic_area_search", geo)
             end
 
-            add_field.call("export_as_mla_citation_txt","")
+            # This is just a placeholder, reminding us that we need to implement citations in some way
+            # add_field.call("export_as_mla_citation_txt","")
             
             if(organizations.count > 0)
               organizations = organizations.uniq
               organizations.each do |organization|
-                add_field.call("affiliation_organization", organization)
+                add_field.call("organization_facet", organization)
               end
             end
             
             if(departments.count > 0)
               departments = departments.uniq
               departments.each do |department|
-                add_field.call("affiliation_department", department.to_s.sub(", Department of", "").strip)
+                add_field.call("department_facet", department.to_s.sub(", Department of", "").strip)
               end
             end
             

data/lib/cul-fedora/server.rb

@@ -1,3 +1,9 @@
+begin
+  require "active_support/core_ext/array/extract_options"
+rescue
+  require "activesupport"
+end
+
 module Cul
   module Fedora
     class Server
@@ -6,10 +12,10 @@ module Cul
 
       def initialize(*args)
         options = args.extract_options!
-        @riurl = options[:riurl] || raise(ArgumentError, "Must provide riurl argument")
-        @riquery = options[:riquery] || raise(ArgumentError, "Must provide riquery argument")
-        @hc = options[:http_client]
-        @logger = options[:logger] 
+        @riurl = options[:riurl] || options["riurl"] || raise(ArgumentError, "Must provide riurl argument")
+        @riquery = options[:riquery] || options["riquery"] || raise(ArgumentError, "Must provide riquery argument")
+        @hc = options[:http_client] || options["http_client"]
+        @logger = options[:logger] || options["logger"]
       end
 
       def logger

data/lib/cul-fedora/solr.rb

@@ -1,12 +1,18 @@
+begin
+  require "active_support/core_ext/array/extract_options"
+rescue
+  require "activesupport"
+end
+
 module Cul
   module Fedora
     class Solr
       
       attr_reader :url
       
-      def initialize(config = {})
-        @url = config[:url] || raise(ArgumentError, "must provide url")
-        @logger = config[:logger]
+      def initialize(options = {})
+        @url = options[:url] || options["url"] || raise(ArgumentError, "must provide url")
+        @logger = options[:logger] || options["logger"]
       end
 
       def logger
@@ -22,31 +28,53 @@ module Cul
       end
 
       def delete_index
+        logger.info "Deleting Solr index..."
         rsolr.delete_by_query("*:*")
         rsolr.commit
       end
       
-      def delete_removed(fedora_server)
+      def delete_removed(fedora_server, fedora_item_pids = nil)
+        
+        removed = identify_removed(fedora_server)
+        logger.info "Deleting items removed from Fedora..."
+        removed.each do |id|
+          logger.info "Deleting " + id + "..."
+          rsolr.delete_by_query("id:" + id.to_s.gsub(/:/,'\\:'))
+        end
+        
+        rsolr.commit
         
+      end
+      
+      def identify_removed(fedora_server, fedora_item_pids = nil)
         start = 0
         rows = 500
+        removed = []
         results = rsolr.select({:q => "", :fl => "id", :start => start, :rows => rows})
-        logger.info "Deleting items removed from Fedora..."
+        logger.info "Identifying items removed from Fedora..."
         while(!results["response"]["docs"].empty?)
           
+          logger.info("Checking Solr index from " + start.to_s + " to " + (start + rows).to_s + "...")
           results["response"]["docs"].each do |doc|
-            if(!fedora_server.item(doc["id"]).exists?)
-              logger.info "Deleting " + doc["id"] + "..."
-              rsolr.delete_by_query("id:" + doc["id"].to_s.gsub(/:/,'\\:'))
+            
+            if(fedora_item_pids.nil?)
+              if(!fedora_server.item(doc["id"]).exists?)
+                logger.info "Noting removed item " + doc["id"] + "..."
+                removed << doc["id"].to_s
+              end
+            else
+              if(!fedora_item_pids.include?(doc["id"].to_s))
+                logger.info "Noting removed item " + doc["id"] + "..."
+                removed << doc["id"].to_s
+              end
             end
+            
           end
           
           start = start + rows
           results = rsolr.select({:q => "", :fl => "id", :start => start, :rows => rows})
         end
-        
-        rsolr.commit
-        
+        return removed
       end
       
       def ingest(options = {})
@@ -63,14 +91,9 @@ module Cul
 
         delete = options.delete(:delete_removed) || false
         overwrite = options.delete(:overwrite) || false
-        process = options.delete(:process) || nil
-        skip = options.delete(:skip) || nil
-
-        processed_successfully = 0
+        skip = options.delete(:skip) || []
 
-        if delete == true
-          delete_removed(fedora_server)
-        end
+        indexed_count = 0
         
         logger.info "Preparing the items for indexing..."
         collections.each do |collection|
@@ -79,24 +102,26 @@ module Cul
 
         items.sort!
 
-        to_add = []
         results = Hash.new { |h,k| h[k] = [] }
-        errors = {}
+        errors = []
+      
+        item_pids = []
+        items.each do |item|
+          item_pids << item.pid
+        end
+        if delete == true
+          delete_removed(fedora_server, item_pids)
+        end
       
         logger.info "Preparing to index " + items.length.to_s + " items..."
       
         items.each do |i|
           
-          if(ignore.index(i.pid).nil? == false)
-            logger.info "Ignoring " + i.pid + "..."
+          if(ignore.index(i.pid).nil? == false || skip.index(i.pid).nil? == false)
+            logger.info "Ignoring/skipping " + i.pid + "..."
+            results[:skipped] << i.pid
             next
           end
-          
-          if process && skip && skip > 0
-            skip -= 1
-            next
-          end
-
            
           if item_exists?(i)
             unless overwrite == true
@@ -104,45 +129,33 @@ module Cul
               next
             end
           end    
-          
 
           logger.info "Indexing " + i.pid + "..."
 
           result_hash = i.send("index_for_#{format}", options)
 
-          results[result_hash[:status]]  << i.pid
+          results[result_hash[:status]] << i.pid
 
           case result_hash[:status]
           when :success
-            to_add << result_hash[:results]
-            processed_successfully += 1
+            begin
+              rsolr.add(result_hash[:results])
+              indexed_count += 1
+            rescue Exception => e
+              errors << i.pid
+              logger.error e.message
+            end
           when :error
-            errors[i.pid] = result_hash[:error_message]
-          end
-
-          if process
-            process -= 1
-            break if process <= 0
+            errors << i.pid
+            logger.error result_hash[:error_message]
           end
 
-          if to_add.length >= 500
-            logger.info "Adding batch to commit queue..."
-            rsolr.add(to_add)
-            to_add.clear
-          end
-
-        end
-        
-        if to_add.length > 0
-          logger.info "Adding batch to commit queue..."
-          rsolr.add(to_add)
-          to_add.clear
         end
         
         logger.info "Committing changes to Solr..."
         rsolr.commit
 
-        return {:results => results, :errors => errors, :processed_successfully => processed_successfully}
+        return {:results => results, :errors => errors, :indexed_count => indexed_count}
 
       end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cul-fedora
 version: !ruby/object:Gem::Version 
-  hash: 51
-  prerelease: false
+  hash: 23
+  prerelease: 
   segments: 
+  - 1
+  - 0
   - 0
-  - 8
-  - 6
-  version: 0.8.6
+  version: 1.0.0
 platform: ruby
 authors: 
 - James Stuart
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-14 00:00:00 -04:00
-default_executable: 
+date: 2011-09-19 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: shoulda
@@ -175,7 +174,6 @@ files:
 - test/test_fedora_server.rb
 - test/test_fedora_solr.rb
 - test_fedora_item.rb
-has_rdoc: true
 homepage: http://github.com/tastyhat/cul-fedora
 licenses: []
 
@@ -205,13 +203,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
 summary: Columbia University Fedora Hooks
-test_files: 
-- test/helper.rb
-- test/test_cul-fedora.rb
-- test/test_fedora_item.rb
-- test/test_fedora_server.rb
-- test/test_fedora_solr.rb
+test_files: []
+