solr_ead 0.0.1

data/lib/solr_ead/indexer.rb

@@ -0,0 +1,128 @@
+module SolrEad
+
+# The main entry point for your ead going into solr.
+#
+# SolrEad uses RSolr to connect to your solr server and then gives you a couple of
+# simple methods for creating, updating and deleting your ead documents.
+#
+# You'll need to have your solr configuration defined in config/solr.yml.  If you're
+# working within the Rails environment, it will obey your environment settings.  However,
+# if you are using the gem by itself outside of rails, you can use the RAILS_ENV environment
+# variable, otherwise, it will default to the development url.
+#
+# ==Default indexing
+# This will index your ead into one solr document for the main portion of ead and then
+# multiple documents for the component documents.  The fields for the main document
+# are defined in SolrEad::Document and fields for the component are defined in SolrEad::Component.
+#   > file = File.new("path/to/your/ead.xml")
+#   > indexer = SolrEad::Indexer.new
+#   > indexer.create(file)
+#   > indexer.delete("EAD-ID")
+#
+# ==Simple indexing
+# By using the :simple option, SolrEad will create only one solr document from one ead.
+# The default implementation of SolrEad is to create multiple documents, so fields
+# defined in SolrEad::Document reflect this.  For example, no component fields are defined in
+# SolrEad::Document, so none would be indexed.  If you elect to use the :simple option, you'll
+# want to override SolrEad::Document with your own and define any additional component fields
+# you want to appear in your index.
+#   > file = File.new("path/to/your/ead.xml")
+#   > indexer = SolrEad::Indexer.new(:simple => true)
+#   > indexer.create(file)
+#   > indexer.delete("EAD-ID")
+
+class Indexer
+
+  include RSolr
+  include SolrEad::Behaviors
+
+  attr_accessor :solr, :options
+
+  # Creates a new instance of SolrEad::Indexer and connects to your solr server
+  # using the url supplied in your config/solr.yml file.
+  def initialize(opts={})
+    if defined?(Rails.root)
+      url = YAML.load_file(File.join(Rails.root,"config","solr.yml"))[Rails.env]['url']
+    elsif ENV['RAILS_ENV']
+      url = YAML.load_file(File.join(Rails.root,"config","solr.yml"))[ENV['RAILS_ENV']]['url']
+    else
+      url = YAML.load_file("config/solr.yml")['development']['url']
+    end
+    self.solr = RSolr.connect :url => url
+    self.options = opts
+  end
+
+  # Indexes your ead and additional component documents with the supplied file, then
+  # commits the results to your solr server.
+  def create(file)
+    doc = om_document(File.new(file))
+    solr_doc = doc.to_solr
+    solr.add solr_doc
+    add_components(file) unless options[:simple]
+    solr.commit
+  end
+
+  # Updates your ead from a given file by first deleting the existing ead document and
+  # any component documents, then creating a new index from the supplied file.
+  # This method will also commit the results to your solr server when complete.
+  def update(file)
+    doc = om_document(File.new(file))
+    solr_doc = doc.to_solr
+    solr.delete_by_query( 'eadid_s:"' + solr_doc["id"] + '"' )
+    solr.add solr_doc
+    add_components(file) unless options[:simple]
+    solr.commit
+  end
+
+  # Deletes the ead document and any component documents from your solr index and
+  # commits the results.
+  def delete(id)
+    solr.delete_by_query( 'eadid_s:"' + id + '"')
+    solr.commit
+  end
+
+  protected
+
+  # Returns an OM document from a given file.
+  #
+  # Determines if you have specified a custom definition for your ead document.
+  # If you've defined a class CustomDocument, and have passed it as an option
+  # to your indexer, then SolrEad will use that class instead of SolrEad::Document.
+  def om_document(file)
+    options[:document] ? options[:document].from_xml(File.new(file)) : SolrEad::Document.from_xml(File.new(file))
+  end
+
+  # Returns an OM document from a given Nokogiri node
+  #
+  # Determines if you have specified a custom definition for your ead component.
+  # If you've defined a class CustomComponent, and have passed it as an option
+  # to your indexer, then SolrEad will use that class instead of SolrEad::Component.
+  def om_component_from_node(node)
+    options[:component] ? options[:component].from_xml(prep(node)) : SolrEad::Component.from_xml(prep(node))
+  end
+
+  # Creates solr documents for each individual component node in the ead.  Field names
+  # and values are determined according to the OM terminology outlined in
+  # SolrEad::Component as well as additional fields taken from the rest of the ead
+  # document as described in SolrEad::Behaviors#additional_component_fields.
+  #
+  # Fields from both the terminology and #additional_component_fields are all assembled
+  # into one solr document via the SolrEad::Component#to_solr method.  Any customizations to
+  # the contents or appearance of the fields can take place within that method.
+  #
+  # Furthermore, one final field is added to the solr document after the #to_solr method.
+  # A sorting field *sort_i* is added to the document using the index values from the array
+  # of <c> nodes.  This allows us to preserve the order of <c> nodes as they appear
+  # in the original ead document.
+  def add_components(file)
+    counter = 1
+    components(file).each do |node|
+      solr_doc = om_component_from_node(node).to_solr(additional_component_fields(node))
+      solr_doc.merge!({"sort_i" => counter.to_s})
+      solr.add solr_doc
+      counter = counter + 1
+    end
+  end
+
+end
+end

data/lib/solr_ead/om_behaviors.rb

@@ -0,0 +1,38 @@
+module SolrEad::OmBehaviors
+
+# This modifies the behavior of the OM gem, specifically, the way it creates
+# documents using existing xml.
+#
+# Instead of using the xml as-as, this module will override OM::XML::Container.to_xml
+# and remove all the namespaces from the xml first, then return the Nokogiri object.
+# This makes working with the terminologies in SolrEad::Document much easier.
+#
+# Any customized ead document definitions should include this module. ex:
+#   class MyDocument < SolrEad::Document
+#
+#     include OM::XML::Document
+#     include Solrizer::XML::TerminologyBasedSolrizer
+#     include SolrEad::Container
+#
+#   end
+
+  module ClassMethods
+
+    def from_xml(xml=nil, tmpl=self.new) # :nodoc:
+      if xml.nil?
+        # noop: handled in #ng_xml accessor..  tmpl.ng_xml = self.xml_template
+      elsif xml.kind_of? Nokogiri::XML::Node
+        tmpl.ng_xml = xml.remove_namespaces!
+      else
+        tmpl.ng_xml = Nokogiri::XML::Document.parse(xml).remove_namespaces!
+      end
+      return tmpl
+    end
+
+  end
+
+  def self.included(klass)
+    klass.extend(ClassMethods)
+  end
+
+end

data/lib/solr_ead/railtie.rb

@@ -0,0 +1,7 @@
+module SolrEad
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.join(File.dirname(__FILE__),"../tasks/solr_ead.rake")
+    end
+  end
+end

data/lib/solr_ead/version.rb

@@ -0,0 +1,3 @@
+module SolrEad
+  VERSION = "0.0.1"
+end

data/lib/solr_ead.rb

@@ -0,0 +1,20 @@
+require "nokogiri"
+require "solrizer"
+require "om"
+require "rsolr"
+
+module SolrEad
+  def self.version
+    SolrEad::VERSION
+  end
+end
+
+require "ead_mapper"
+require "terminology_based_solrizer"
+require "solr_ead/behaviors"
+require "solr_ead/om_behaviors"
+require "solr_ead/indexer"
+require "solr_ead/document"
+require "solr_ead/component"
+
+require 'solr_ead/railtie' if defined?(Rails)

data/lib/tasks/solr_ead.rake

@@ -0,0 +1,30 @@
+require "solr_ead"
+
+namespace :solr_ead do
+
+  desc "Index and ead into solr using FILE=<path/to/ead.xml>"
+  task :index do
+    raise "Please specify your ead, ex. FILE=<path/to/ead.xml" unless ENV['FILE']
+    indexer = SolrEad::Indexer.new
+    indexer.update(ENV['FILE'])
+  end
+
+  desc "Delete and ead from your solr index using ID='<eadid>'"
+  task :delete do
+    raise "Please specify your ead id, ex. ID=<eadid>" unless ENV['ID']
+    indexer = SolrEad::Indexer.new
+    indexer.delete(ENV['ID'])
+  end
+
+  desc "Index a directory of ead files given by DIR=path/to/directory"
+  task :index_dir do
+    raise "Please specify your direction, ex. DIR=path/to/directory" unless ENV['DIR']
+    indexer = SolrEad::Indexer.new
+    Dir.glob(File.join(ENV['DIR'],"*")).each do |file|
+      print "Indexing #{File.basename(file)}..."
+      indexer.update(file) if File.extname(file).match("xml$")
+      print "done.\n"
+    end
+  end
+
+end

data/lib/terminology_based_solrizer.rb

@@ -0,0 +1,9 @@
+# Overrides Solrizer::XML::TerminologyBasedSolrizer in the solrizer gem and
+# uses our own custom field mapper defined in EadMapper
+module Solrizer::XML::TerminologyBasedSolrizer
+
+  def self.default_field_mapper
+    @@default_field_mapper ||= EadMapper.new
+  end
+
+end

data/solr_ead.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/solr_ead/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Adam Wead"]
+  gem.email         = ["amsterdamos@gmail.com"]
+  gem.description   = %q{A gem indexing ead into solr using OM. Define your own OM terminology to create the solr fields you want from your ead, then use solr-based applications like Blacklight to search and display the results.}
+  gem.summary       = %q{A gem for indexing ead into solr using OM}
+  gem.homepage      = "http://github.com/awead/ead_solr"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "solr_ead"
+  gem.require_paths = ["lib"]
+  gem.version       = SolrEad::VERSION
+
+  # Dependencies
+  gem.add_dependency('om')
+  gem.add_dependency('solrizer')
+  gem.add_dependency('rsolr')
+  gem.add_dependency('sanitize')
+  # For Development
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'debugger'
+  gem.add_development_dependency 'rdoc'
+end

data/spec/behaviors_spec.rb

@@ -0,0 +1,105 @@
+require "spec_helper"
+
+describe SolrEad::Behaviors do
+
+  before :all do
+    @not_numbered = fixture "ARC-0005.xml"
+    @numbered     = fixture "pp002010.xml"
+    class TestClass
+      include SolrEad::Behaviors
+    end
+    @test = TestClass.new
+  end
+
+  describe "#components" do
+
+    before :all do
+      @non_numbered_nodeset = @test.components(@not_numbered)
+      @numbered_nodeset     = @test.components(@numbered)
+    end
+
+    it "should return a nodeset" do
+      @non_numbered_nodeset.should be_a_kind_of(Nokogiri::XML::NodeSet)
+      @non_numbered_nodeset.first.should be_a_kind_of(Nokogiri::XML::Element)
+    end
+
+    it "should be able to handle both numbered and non-numbered <c> nodes" do
+      @non_numbered_nodeset.count.should == 135
+      @numbered_nodeset.count.should == 83
+    end
+  end
+
+  describe "#prep" do
+    it "should return a single component document" do
+      part = @test.prep(@numbered_nodeset)
+      part.should be_a_kind_of(Nokogiri::XML::Document)
+    end
+  end
+
+  describe "#component_children?" do
+
+    before :all do
+      @true = '
+        <c id="ref167" level="file">
+          <did>
+            <unittitle>Zines</unittitle>
+          </did>
+          <c id="ref169" level="file">
+            <did>
+              <unittitle>Contagion</unittitle>
+              <container id="cid384011" type="Box" label="Graphic materials">SF2</container>
+              <container parent="cid384011" type="Folder">8</container>
+              <unitdate>1980-1981</unitdate>
+            </did>
+          </c>
+          <c id="ref171" level="file">
+            <did>
+              <unittitle>Single issues</unittitle>
+              <container id="cid384012" type="Box" label="Graphic materials">SF2</container>
+              <container parent="cid384012" type="Folder">9</container>
+              <unitdate>1977-1985</unitdate>
+            </did>
+          </c>
+        </c>
+      '
+      @false = '
+        <c id="ref167" level="file">
+          <did>
+            <unittitle>Zines</unittitle>
+          </did>
+        </c>
+      '
+    end
+
+
+    it "should return true for components that have c nodes below them" do
+      node = Nokogiri::XML(@true)
+      @test.component_children?(node.elements.first).should be_true
+    end
+
+    it "should return false for components that do not have c nodes below them" do
+      node = Nokogiri::XML(@false)
+      @test.component_children?(node.elements.first).should be_false
+    end
+
+    it "should return true for series-level components" do
+      pending "This is an old test"
+      ["series","subseries"].each do |level|
+        xml = '<c id="ref42" level="' + level +'"></c>'
+        node = Nokogiri::XML(xml)
+        @test.component_children?(node.elements.first).should be_true
+      end
+    end
+
+    it "should return false for item-level components" do
+      pending "This is an old test"
+      ["file","item"].each do |level|
+        xml = '<c id="ref42" level="' + level +'"></c>'
+        node = Nokogiri::XML(xml)
+        @test.component_children?(node.elements.first).should be_false
+      end
+    end
+
+  end
+
+end

data/spec/component_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+describe SolrEad::Component do
+
+  before(:all) do
+    file = "component_template.xml"
+    @doc = SolrEad::Component.from_xml(fixture file)
+  end
+
+  describe "the solr document" do
+
+    it "should accept additional fields from a hash" do
+      additional_fields = {
+        "id"                      => "TEST-0001:ref010",
+        "eadid_s"                 => "TEST-0001",
+        "parent_id_s"             => "ref001",
+        "parent_id_list_t"        => ["ref001", "ref002", "ref003"],
+        "parent_unittitle_list_t" => ["Series I", "Subseries A", "Subseries 1"],
+        "component_children_b"    => FALSE
+      }
+      solr_doc = @doc.to_solr(additional_fields)
+      solr_doc["id"].should == "TEST-0001:ref010"
+      solr_doc["level_facet"].should include "item"
+      solr_doc["heading_display"].should == "Series I >> Subseries A >> Subseries 1 >> Internal Revenue Service Form Information Return [RESTRICTED]"
+      solr_doc["accessrestrict_t"].first.should match /^This item .* is available.$/
+      solr_doc["accessrestrict_heading_display"].should include "Access Restrictions"
+
+    end
+
+  end
+
+end

data/spec/document_spec.rb

@@ -0,0 +1,80 @@
+require "spec_helper"
+
+describe SolrEad::Document do
+
+  before(:all) do
+    @ex1 = SolrEad::Document.from_xml(fixture "ARC-0005.xml")
+    @ex2 = SolrEad::Document.from_xml(fixture "pp002010.xml")
+    @solr_ex1 = @ex1.to_solr
+    @solr_ex2 = @ex2.to_solr
+  end
+
+  describe "#terminology" do
+
+    it "should have an id field" do
+      @ex1.eadid.first.should == "ARC-0005"
+    end
+
+    it "should have a simple title" do
+      @ex1.title.first.should match "Eddie Cochran Historical Organization Collection"
+    end
+
+    it "should have some subject headings" do
+      @ex1.persname.should include "Cochran, Eddie, 1938-1960"
+      @ex1.genreform.should include "Newspapers"
+      @ex1.subject.should include "Rockabilly music"
+      @ex2.corpname.should include "Tuskegee Normal and Industrial Institute--1880-1940."
+      @ex2.genreform.should include "Group portraits--1880-1940."
+      @ex2.geogname.should include "Washington, D.C."
+      @ex2.name.should include "Bell, J.S., Portland, OR"
+      @ex2.persname.should include "Johnston, Frances Benjamin, 1864-1952, photographer."
+      @ex2.subject.should include "Buildings--1880-1940."
+    end
+
+    it "should have scope and contents" do
+      @ex2.scopecontent.first.should match /^Photographs/
+    end
+
+    it "should have one separatedmaterial material note from the archdesc section" do
+      @ex1.separatedmaterial.first.should match /^Commercially-released publications.*materials are available.$/
+    end
+
+    it "should have its xml" do
+      @ex1.to_xml.should match "<c\s"
+      @ex2.to_xml.should match "<c01\s"\
+    end
+
+  end
+
+  describe ".to_solr" do
+
+    it "should have the appropriate id fields" do
+      @solr_ex1["eadid_s"].should == "ARC-0005"
+      @solr_ex1["id"].should == "ARC-0005"
+      @solr_ex2["eadid_s"].should == "http://hdl.loc.gov/loc.pnp/eadpnp.pp002010"
+      @solr_ex2["id"].should == "http://hdl.loc.gov/loc.pnp/eadpnp.pp002010"
+
+    end
+
+    it "should have faceted terms created from subject headings" do
+      @solr_ex1["persname_facet"].should include "Cochran, Eddie, 1938-1960"
+      @solr_ex1["genreform_facet"].should include "Newspapers"
+      @solr_ex1["subject_facet"].should include "Rockabilly music"
+      @solr_ex2["corpname_facet"].should include "Tuskegee Normal and Industrial Institute--1880-1940."
+      @solr_ex2["genreform_facet"].should include "Group portraits--1880-1940."
+      @solr_ex2["geogname_facet"].should include "Washington, D.C."
+      @solr_ex2["name_facet"].should include "Bell, J.S., Portland, OR"
+      @solr_ex2["persname_facet"].should include "Johnston, Frances Benjamin, 1864-1952, photographer."
+      @solr_ex2["subject_facet"].should include "Buildings--1880-1940."
+    end
+
+    it "should index head tags as display and p tags as text" do
+      @solr_ex1["separatedmaterial_heading_display"].should include "Separated Materials"
+      @solr_ex1["separatedmaterial_t"].first.should match /^Commercially-released publications.*materials are available.$/
+      @solr_ex2["scopecontent_heading_display"].should include "Scope and Content Note"
+      @solr_ex2["scopecontent_t"].first.should match /^Photographs/
+    end
+
+  end
+
+end