cul-fedora 0.5.1 → 0.5.2

data/VERSION

@@ -1 +1 @@
-0.5.1
+0.5.2

data/cul-fedora.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cul-fedora}
-  s.version = "0.5.1"
+  s.version = "0.5.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Stuart"]
-  s.date = %q{2010-10-11}
+  s.date = %q{2010-10-12}
   s.description = %q{Columbia-specific Fedora libraries}
   s.email = %q{tastyhat@jamesstuart.org}
   s.extra_rdoc_files = [
@@ -47,6 +47,7 @@ Gem::Specification.new do |s|
      "lib/tika/poi-3.5-beta5.jar",
      "lib/tika/poi-ooxml-3.5-beta5.jar",
      "lib/tika/poi-scratchpad-3.5-beta5.jar",
+     "lib/tika/scratch/1286827167_3249395",
      "lib/tika/tika-0.3.jar",
      "lib/tika/xercesImpl-2.8.1.jar",
      "lib/tika/xml-apis-1.0.b2.jar",

data/lib/cul-fedora/item.rb

@@ -1,11 +1,21 @@
+require "open3"
+
 module Cul
   module Fedora
     class Item
       attr_reader :server, :pid
+      include Open3 
 
       URI_TO_PID = 'info:fedora/'
 
 
+      def <=>(other)
+        pid <=> other.pid
+      end
+
+      def pid_escaped
+        pid.gsub(/:/,'\\:')
+      end
 
       def initialize(*args)
         options = args.extract_options!
@@ -38,7 +48,7 @@ module Cul
 
       def risearch_for_members()
         results = JSON::parse(@server.request(:method => "", :request => "risearch", :format => "json", :lang => "itql", :query => sprintf(@server.riquery, @pid)))["results"]
-        
+
         results.collect { |r| @server.item(r["member"]) }
 
       end
@@ -73,7 +83,14 @@ module Cul
         end
       end
 
-      def index_for_ac2
+      def index_for_ac2(options = {})
+        do_fulltext = options[:fulltext] || false
+        do_metadata = options[:metadata] || true
+
+        status = :success
+        error_message = ""
+
+
         results = Hash.new { |h,k| h[k] = [] }
         normalize_space = lambda { |s| s.to_s.strip.gsub(/\s{2,}/," ") }
         search_to_content = lambda { |x| x.kind_of?(Nokogiri::XML::Element) ? x.content : x.to_s }
@@ -83,125 +100,150 @@ module Cul
 
         roles = ["Author","author","Creator","Thesis Advisor","Collector","Owner","Speaker","Seminar Chairman","Secretary","Rapporteur","Committee Member","Degree Grantor","Moderator","Editor","Interviewee","Interviewer","Organizer of Meeting","Originator","Teacher"]
 
-        collections = self.belongsTo
-        meta = describedBy.first
 
-        meta = Nokogiri::XML(meta.datastream("CONTENT")) if meta
-        mods = meta.at_css("mods") if meta
+        begin
+          collections = self.belongsTo
+          meta = describedBy.first
+
+          meta = Nokogiri::XML(meta.datastream("CONTENT")) if meta
+          mods = meta.at_css("mods") if meta
+
+          if mods && do_metadata
+            # baseline blacklight fields: id is the unique identifier, format determines by default, what partials get called
+            add_field.call("id", @pid)
+            add_field.call("internal_h",  collections.first.to_s + "/")
+            add_field.call("pid", @pid)
+            collections.each do |collection|
+              add_field.call("member_of", collection)
+            end
+
 
-        return {} unless mods
-        # baseline blacklight fields: id is the unique identifier, format determines by default, what partials get called
-        add_field.call("id", @pid)
-        add_field.call("internal_h",  collections.first.to_s + "/")
-        add_field.call("pid", @pid)
-        collections.each do |collection|
-          add_field.call("member_of", collection)
-        end
 
+            title = normalize_space.call(mods.css("titleInfo>nonSort,title").collect(&:content).join(" "))
+            add_field.call("title_display", title)
+            add_field.call("title_search", title)
 
-        
-          title = normalize_space.call(mods.css("titleInfo>nonSort,title").collect(&:content).join(" "))
-          add_field.call("title_display", title)
-          add_field.call("title_search", title)
+            all_names = []
+            mods.css("name[@type='personal']").each do |name_node|
+              if name_node.css("role>roleTerm[@type='text']").collect(&:content).any? { |role| roles.include?(role) }
 
-          all_names = []
-          mods.css("name[@type='personal']").each do |name_node|
-            if name_node.css("role>roleTerm[@type='text']").collect(&:content).any? { |role| roles.include?(role) }
+                fullname = get_fullname.call(name_node)
 
-              fullname = get_fullname.call(name_node)
+                all_names << fullname
+                add_field.call("author_id_uni", name_node.at_css("authorID[@type='institution']"))
+                add_field.call("author_id_repository", name_node.at_css("authorID[@type='repository']"))
+                add_field.call("author_id_naf", name_node.at_css("authorID[@type='naf']"))
+                add_field.call("author_search", fullname.downcase)
+                add_field.call("author_facet", fullname)
 
-              all_names << fullname
-              add_field.call("author_id_uni", name_node.at_css("authorID[@type='institution']"))
-              add_field.call("author_id_repository", name_node.at_css("authorID[@type='repository']"))
-              add_field.call("author_id_naf", name_node.at_css("authorID[@type='naf']"))
-              add_field.call("author_search", fullname.downcase)
-              add_field.call("author_facet", fullname)
+              end
 
             end
 
-          end
+            add_field.call("authors_display",all_names.join("; "))
+            add_field.call("date", mods.at_css("*[@keyDate='yes']"))
 
-          add_field.call("authors_display",all_names.join("; "))
-          add_field.call("date", mods.at_css("*[@keyDate='yes']"))
+            mods.css("genre").each do |genre_node|
+              add_field.call("genre_facet", genre_node)
+              add_field.call("genre_search", genre_node)
 
-          mods.css("genre").each do |genre_node|
-            add_field.call("genre_facet", genre_node)
-            add_field.call("genre_search", genre_node)
+            end
 
-          end
 
+            add_field.call("abstract", mods.at_css("abstract"))
+            add_field.call("handle", mods.at_css("identifier[@type='hdl']"))
 
-          add_field.call("abstract", mods.at_css("abstract"))
-          add_field.call("handle", mods.at_css("identifier[@type='hdl']"))
+            mods.css("subject:not([@authority='local'])>topic").each do |topic_node|
+              add_field.call("keyword_search", topic_node.content.downcase)
+              add_field.call("keyword_facet", topic_node)
+            end
 
-          mods.css("subject:not([@authority='local'])>topic").each do |topic_node|
-            add_field.call("keyword_search", topic_node.content.downcase)
-            add_field.call("keyword_facet", topic_node)
-          end
+            mods.css("subject[@authority='local']>topic").each do |topic_node|
+              add_field.call("subject", topic_node)
+              add_field.call("subject_search", topic_node)
+            end
 
-          mods.css("subject[@authority='local']>topic").each do |topic_node|
-            add_field.call("subject", topic_node)
-            add_field.call("subject_search", topic_node)
-          end
 
+            add_field.call("tableOfContents", mods.at_css("tableOfContents"))
+
+            mods.css("note").each { |note| add_field.call("notes", note) }
 
-          add_field.call("tableOfContents", mods.at_css("tableOfContents"))
+            if (related_host = mods.at_css("relatedItem[@type='host']"))
+              book_journal_title = related_host.at_css("titleInfo>title") 
 
-          mods.css("note").each { |note| add_field.call("notes", note) }
+              if book_journal_title
+                book_journal_subtitle = mods.at_css("name>titleInfo>subTitle")
 
-          if (related_host = mods.at_css("relatedItem[@type='host']"))
-            book_journal_title = related_host.at_css("titleInfo>title") 
+                book_journal_title = book_journal_title.content + ": " + book_journal_subtitle.content.to_s if book_journal_subtitle
 
-            if book_journal_title
-              book_journal_subtitle = mods.at_css("name>titleInfo>subTitle")
+              end
 
-              book_journal_title = book_journal_title.content + ": " + book_journal_subtitle.content.to_s if book_journal_subtitle
+              add_field.call("book_journal_title", book_journal_title)
 
+              add_field.call("book_author", get_fullname.call(related_host.at_css("name"))) 
+
+              add_field.call("issn", related_host.at_css("identifier[@type='issn']"))
             end
 
-            add_field.call("book_journal_title", book_journal_title)
+            add_field.call("publisher", mods.at_css("relatedItem>originInfo>publisher"))
+            add_field.call("publisher_location", mods.at_css("relatedItem > originInfo>place>placeTerm[@type='text']"))
+            add_field.call("isbn", mods.at_css("relatedItem>identifier[@type='isbn']"))
+            add_field.call("doi", mods.at_css("identifier[@type='doi'][@displayLabel='Published version']"))
 
-            add_field.call("book_author", get_fullname.call(related_host.at_css("name"))) 
+            mods.css("physicalDescription>internetMediaType").each { |mt| add_field.call("media_type_facet", mt) }
 
-            add_field.call("issn", related_host.at_css("identifier[@type='issn']"))
+            mods.css("typeOfResource").each { |tr| add_field.call("type_of_resource_facet", tr)}
+            mods.css("subject>geographic").each do |geo|
+              add_field.call("geographic_area", geo)
+              add_field.call("geographic_area_search", geo)
+            end
           end
 
-          add_field.call("publisher", mods.at_css("relatedItem>originInfo>publisher"))
-          add_field.call("publisher_location", mods.at_css("relatedItem > originInfo>place>placeTerm[@type='text']"))
-          add_field.call("isbn", mods.at_css("relatedItem>identifier[@type='isbn']"))
-          add_field.call("doi", mods.at_css("identifier[@type='doi'][@displayLabel='Published version']"))
 
-          mods.css("physicalDescription>internetMediaType").each { |mt| add_field.call("media_type_facet", mt) }
+          if do_fulltext 
+            listMembers.each_with_index do |member, i|
+              tika_directory = File.expand_path(File.join(File.expand_path(File.dirname(__FILE__)), "..", "tika"))
 
-          mods.css("typeOfResource").each { |tr| add_field.call("type_of_resource_facet", tr)}
-          mods.css("subject>geographic").each do |geo|
-            add_field.call("geographic_area", geo)
-            add_field.call("geographic_area_search", geo)
-          end
+              resource_file_name = File.join(tika_directory, "scratch", Time.now.to_i.to_s + "_" + rand(10000000).to_s)
+              tika_jar = File.join(tika_directory, "tika-0.3.jar")
+
+              File.open(resource_file_name, "w") { |f| f.puts(member.datastream("CONTENT")) }
 
 
+              tika_result = []
+              tika_error = []
 
-        
-        listMembers.each_with_index do |member, i|
-          tika_directory = File.expand_path(File.join(File.expand_path(File.dirname(__FILE__)), "..", "tika"))
+              Open3.popen3("java -jar #{tika_jar} -t #{resource_file_name}") do |stdin, stdout, stderr|
+                tika_result = stdout.readlines
+                tika_error = stderr.readlines
+              end
 
-          resource_file_name = File.join(tika_directory, "scratch", Time.now.to_i.to_s + "_" + rand(10000000).to_s)
-          tika_jar = File.join(tika_directory, "tika-0.3.jar")
+              unless tika_error.empty?
+                status = :error
+                error_message += tika_error.join("\n")
+              else
 
-          File.open(resource_file_name, "w") { |f| f.puts(member.datastream("CONTENT")) }
 
-          
-          tika_result = %x[java -jar #{tika_jar} -t #{resource_file_name}]
-          
+                add_field.call("ac.fulltext_#{i}", tika_result)
+              end
 
-          add_field.call("ac.fulltext_#{i}", tika_result)
-       
-          File.delete(resource_file_name)
+              File.delete(resource_file_name)
+            end
+          end
+
+        rescue Exception => e
+          status = :error
+          error_message += e.message
         end
 
-        return results
+        status = :invalid_format  if results.empty?
+
+        return {:status => status, :error_message => error_message, :results => results}
+
       end
 
+
+
       def to_s
         @pid
       end

data/lib/cul-fedora/solr.rb

@@ -7,30 +7,83 @@ module Cul
 
       end
 
+      def item_exists?(item)
+        !rsolr.find(:filters => {:id => item.pid_escaped})["response"]["docs"].empty?
+      end
+
       def rsolr
         @rsolr ||= RSolr.connect(:url => @url)
       end
 
       def ingest(options = {})
         format = options.delete(:format) || raise(ArgumentError, "needs format")
+
         items = options.delete(:items) || []
         items = [items] unless items.kind_of?(Array)
-        
         collections = options.delete(:collections) || []
         collections = [collections] unless collections.kind_of?(Array)
+
+        overwrite = options.delete(:overwrite) || false
+        process = options.delete(:process) || nil
+        skip = options.delete(:skip) || nil
+
+
+
         collections.each do |collection|
           items |= collection.listMembers
         end
 
+        items.sort!
+
+        to_add = []
+        results = Hash.new { |h,k| h[k] = [] }
+        errors = {}
+      
+        items.each do |i|
+          if process && skip && skip > 0
+            skip -= 1
+            next
+          end
+
+           
+          if item_exists?(i)
+            
+            unless overwrite
+              results[:skipped] << i.pid
+              next
+            end
+          end    
+          
 
 
-        rsolr.add(items.collect { |i| i.send("index_for_#{format}")}.reject { |doc| doc == {}})
-       
+
+          result_hash = i.send("index_for_#{format}", options)
+
+          results[result_hash[:status]]  << i.pid
+
+          case result_hash[:status]
+          when :success
+            to_add << result_hash[:results]
+          when :error
+            errors[i.pid] = result_hash[:error_message]
+          end
+
+          if process
+            process -= 1
+            break if process <= 0
+          end
+
+        end
+          
+        rsolr.add(to_add)
         rsolr.commit
+
+        return {:results => results, :errors => errors}
+
       end
 
     end
-    
+
 
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cul-fedora
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 15
   prerelease: false
   segments: 
   - 0
   - 5
-  - 1
-  version: 0.5.1
+  - 2
+  version: 0.5.2
 platform: ruby
 authors: 
 - James Stuart
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-11 00:00:00 -04:00
+date: 2010-10-12 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -158,6 +158,7 @@ files:
 - lib/tika/poi-3.5-beta5.jar
 - lib/tika/poi-ooxml-3.5-beta5.jar
 - lib/tika/poi-scratchpad-3.5-beta5.jar
+- lib/tika/scratch/1286827167_3249395
 - lib/tika/tika-0.3.jar
 - lib/tika/xercesImpl-2.8.1.jar
 - lib/tika/xml-apis-1.0.b2.jar