dor-services 2.2.4

data/lib/gsearch/schema.xml

@@ -0,0 +1,229 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- This is the Solr schema file. This file should be named "schema.xml" and
+ should be in the conf directory under the solr home
+ (i.e. ./solr/conf/schema.xml by default) 
+ or located where the classloader for the Solr webapp can find it.
+
+ For more information, on how to customize this file, please see
+ http://wiki.apache.org/solr/SchemaXml
+-->
+
+<schema name="active_fedora" version="0.1">
+  <!-- attribute "name" is the name of this schema and is only used for display purposes.
+       Applications should change this to reflect the nature of the search collection.
+       version="1.1" is Solr's version number for the schema syntax and semantics.  It should
+       not normally be changed by applications.
+       1.0: multiValued attribute did not exist, all fields are multiValued by nature
+       1.1: multiValued attribute introduced, false by default -->
+
+  <types>
+    <!-- field type definitions. The "name" attribute is
+       just a label to be used by field definitions.  The "class"
+       attribute and any other attributes determine the real
+       behavior of the fieldtype.
+         Class names starting with "solr" refer to java classes in the
+       org.apache.solr.analysis package.
+    -->
+
+    <!-- The StrField type is not analyzed, but indexed/stored verbatim.  
+       - StrField and TextField support an optional compressThreshold which
+       limits compression (if enabled in the derived fields) to values which
+       exceed a certain size (in characters).
+    -->
+    <fieldtype name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
+
+    <!-- boolean type: "true" or "false" -->
+    <fieldtype name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
+
+    <!-- The optional sortMissingLast and sortMissingFirst attributes are
+         currently supported on types that are sorted internally as strings.
+       - If sortMissingLast="true", then a sort on this field will cause documents
+         without the field to come after documents with the field,
+         regardless of the requested sort order (asc or desc).
+       - If sortMissingFirst="true", then a sort on this field will cause documents
+         without the field to come before documents with the field,
+         regardless of the requested sort order.
+       - If sortMissingLast="false" and sortMissingFirst="false" (the default),
+         then default lucene sorting will be used which places docs without the
+         field first in an ascending sort and last in a descending sort.
+    -->    
+
+
+    <!-- numeric field types that store and index the text
+         value verbatim (and hence don't support range queries, since the
+         lexicographic ordering isn't equal to the numeric ordering) -->
+    <fieldtype name="integer" class="solr.IntField" omitNorms="true"/>
+    <fieldtype name="long" class="solr.LongField" omitNorms="true"/>
+    <fieldtype name="float" class="solr.FloatField" omitNorms="true"/>
+    <fieldtype name="double" class="solr.DoubleField" omitNorms="true"/>
+
+
+    <!-- Numeric field types that manipulate the value into
+         a string value that isn't human-readable in its internal form,
+         but with a lexicographic ordering the same as the numeric ordering,
+         so that range queries work correctly. -->
+    <fieldtype name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
+    <fieldtype name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
+    <fieldtype name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
+    <fieldtype name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
+
+
+    <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
+         is a more restricted form of the canonical representation of dateTime
+         http://www.w3.org/TR/xmlschema-2/#dateTime    
+         The trailing "Z" designates UTC time and is mandatory.
+         Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
+         All other components are mandatory.
+
+         Expressions can also be used to denote calculations that should be
+         performed relative to "NOW" to determine the value, ie...
+
+               NOW/HOUR
+                  ... Round to the start of the current hour
+               NOW-1DAY
+                  ... Exactly 1 day prior to now
+               NOW/DAY+6MONTHS+3DAYS
+                  ... 6 months and 3 days in the future from the start of
+                      the current day
+                      
+         Consult the DateField javadocs for more information.
+      -->
+    <fieldtype name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
+
+    <!-- solr.TextField allows the specification of custom text analyzers
+         specified as a tokenizer and a list of token filters. Different
+         analyzers may be specified for indexing and querying.
+
+         The optional positionIncrementGap puts space between multiple fields of
+         this type on the same document, with the purpose of preventing false phrase
+         matching across fields.
+
+         For more info on customizing your analyzer chain, please see
+         http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
+     -->
+
+    <!-- One can also specify an existing Analyzer class that has a
+         default constructor via the class attribute on the analyzer element
+    <fieldtype name="text_greek" class="solr.TextField">
+      <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
+    </fieldType>
+    -->
+
+    <!-- A text field that only splits on whitespace for exact matching of words -->
+    <fieldtype name="text_ws" class="solr.TextField" positionIncrementGap="100">
+      <analyzer>
+        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+      </analyzer>
+    </fieldtype>
+
+    <fieldtype name="text" class="solr.TextField" positionIncrementGap="100">
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.StandardFilterFactory"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+      </analyzer>
+    </fieldtype>
+
+    <fieldtype name="text_zh" class="solr.TextField">
+      <analyzer class="org.apache.lucene.analysis.cn.ChineseAnalyzer"/>
+    </fieldtype>
+
+
+    <!-- Less flexible matching, but less false matches.  Probably not ideal for product names,
+         but may be good for SKUs.  Can insert dashes in the wrong place and still match. -->
+    <fieldtype name="textTight" class="solr.TextField" positionIncrementGap="100" >
+      <analyzer>
+        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
+        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>
+        <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
+      </analyzer>
+    </fieldtype>
+
+ </types>
+
+
+ <fields>
+   <!-- Valid attributes for fields:
+     name: mandatory - the name for the field
+     type: mandatory - the name of a previously defined type from the <types> section
+     indexed: true if this field should be indexed (searchable or sortable)
+     stored: true if this field should be retrievable
+     compressed: [false] if this field should be stored using gzip compression
+       (this will only apply if the field type is compressable; among
+       the standard field types, only TextField and StrField are)
+     multiValued: true if this field may contain multiple values per document
+     omitNorms: (expert) set to true to omit the norms associated with
+       this field (this disables length normalization and index-time
+       boosting for the field, and saves some memory).  Only full-text
+       fields or fields that need an index-time boost need norms.
+   -->
+
+   <field name="PID" type="string" indexed="true" stored="true"/>
+
+   <!-- catchall field, containing all other searchable text fields (implemented
+        via copyField further on in this schema  -->
+   <field name="text" type="text" indexed="true" stored="false" multiValued="true"/>
+   <!-- catchall field, containing all other searchable text fields (implemented
+        via copyField further on in this schema  -->
+   <field name="date" type="date" indexed="true" stored="false" multiValued="true"/>
+
+   <!-- Dynamic field definitions.  If a field name is not found, dynamicFields
+        will be used if the name matches any of the patterns.
+        RESTRICTION: the glob-like pattern in the name attribute must have
+        a "*" only at the start or the end.
+        EXAMPLE:  name="*_i" will match any field ending in _i (like myid_i, z_i)
+        Longer patterns will be matched first.  if equal size patterns
+        both match, the first appearing in the schema will be used.  -->
+	<dynamicField name="*_date"  type="date"  indexed="true"  stored="true" multiValued="true"/>
+	<dynamicField name="*_name"  type="date"  indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_field"  type="text"  indexed="true"  stored="true" multiValued="true"/>
+
+   <dynamicField name="*_facet"  type="string"  indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_zh_text"   type="text_zh"    indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_text"   type="text"    indexed="true"  stored="true" multiValued="true"/>
+   <dynamicField name="*_display"   type="text"    indexed="false"  stored="true" multiValued="true"/>   
+ </fields>
+
+ <!-- field to use to determine and enforce document uniqueness. -->
+ <uniqueKey>PID</uniqueKey>
+
+ <!-- field for the QueryParser to use when an explicit fieldname is absent -->
+ <defaultSearchField>text</defaultSearchField>
+
+ <!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
+ <solrQueryParser defaultOperator="AND"/>
+
+  <!-- copyField commands copy one field to another at the time a document
+        is added to the index.  It's used either to index the same field differently,
+        or to add multiple fields to the same field for easier/faster searching.  -->
+ <copyField source="*_date" dest="date"/>
+ <copyField source="*_text" dest="text"/>
+ <copyField source="*_facet" dest="text"/>
+
+ <!-- Similarity is the scoring routine for each document vs. a query.
+      A custom similarity may be specified here, but the default is fine
+      for most applications.  -->
+ <!-- <similarity class="org.apache.lucene.search.DefaultSimilarity"/> -->
+
+</schema>

data/lib/tasks/rdoc.rake

@@ -0,0 +1,32 @@
+desc "Generate RDoc"
+task :doc => ['doc:generate']
+
+namespace :doc do
+  project_root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+  doc_destination = File.join(project_root, 'rdoc')
+
+  begin
+    require 'yard'
+    require 'yard/rake/yardoc_task'
+
+    YARD::Rake::YardocTask.new(:generate) do |yt|
+      yt.files   =  Dir.glob(File.join(project_root, 'lib', '*.rb')) + 
+                    Dir.glob(File.join(project_root, 'lib', '**', '*.rb')) +
+                   [ File.join(project_root, 'README.rdoc') ] +
+                   [ File.join(project_root, 'LICENSE') ]
+                   
+      yt.options = ['--output-dir', doc_destination, '--readme', 'README.rdoc']
+    end
+  rescue LoadError
+    desc "Generate YARD Documentation"
+    task :generate do
+      abort "Please install the YARD gem to generate rdoc."
+    end
+  end
+
+  desc "Remove generated documenation"
+  task :clean do
+    rm_r doc_destination if File.exists?(doc_destination)
+  end
+
+end

data/lib/xml_models/foxml.rb

@@ -0,0 +1,261 @@
+require 'nokogiri'
+
+# TODO: Rewrite in OM
+
+class Foxml
+  attr_reader :xml
+
+  NAMESPACES = {
+    "dc"           => "http://purl.org/dc/elements/1.1/", 
+    "fedora-model" => "info:fedora/fedora-system:def/model#", 
+    "foxml"        => "info:fedora/fedora-system:def/foxml#",
+    "oai_dc"       => "http://www.openarchives.org/OAI/2.0/oai_dc/", 
+    "rdf"          => "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 
+    "rel"          => "info:fedora/fedora-system:def/relations-external#",
+    "hydra"        => "http://projecthydra.org/ns/relations#"
+  } 
+  
+  def initialize(pid=nil, label=nil, content_model=nil, identity_metadata=nil, admin_policy_object=nil, parent = nil)
+    @xml = Nokogiri::XML(XML_TEMPLATE) { |config| config.default_xml.noblanks }
+    @defined_namespaces = { '' => nil }
+    @xml.traverse { |node|
+      if node.respond_to?(:namespace_definitions)
+        node.namespace_definitions.each { |ns|
+          @defined_namespaces[ns.prefix] = ns
+        }
+      end
+    }
+
+    self.pid = pid.to_s
+    self.label = label.to_s
+    self.content_model = content_model
+    self.identity_metadata = identity_metadata
+    self.admin_policy_object = admin_policy_object
+    self.parent = parent
+  end
+
+  def pid
+    self.xpath('/foxml:digitalObject/@PID').first.value
+  end
+  
+  def pid=(value)
+    self.xpath('/foxml:digitalObject/@PID').first.value = value
+    self.get_datastream("RELS-EXT","rdf:RDF/rdf:Description/@rdf:about").value = "info:fedora/#{value}"
+    update_dc_identifiers
+  end
+
+  def content_model
+    self.get_rels_ext_resource("fedora-model:hasModel")
+  end
+  
+  def content_model=(value)
+    self.set_rels_ext_resource("fedora-model:hasModel",value)
+  end
+  
+  def dublin_core
+    self.get_datastream("DC","oai_dc:dc")
+  end
+  
+  def dublin_core=(value)
+    self.set_datastream("DC",value,'<oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"/>')
+  end
+  
+  def identity_metadata
+    self.get_datastream("identityMetadata","identityMetadata")
+  end
+  
+  def identity_metadata=(value)
+    self.set_datastream("identityMetadata",value,"<identityMetadata/>")
+    # strip the namespace Nokogiri attaches to identityMetadata
+    self.get_datastream("identityMetadata","*[local-name()='identityMetadata']").traverse { |n| n.namespace = nil }
+    update_dc_identifiers
+  end
+  
+  def label
+    self.xpath('//foxml:property[@NAME="info:fedora/fedora-system:def/model#label"]/@VALUE').first
+  end
+  
+  def label=(value)
+    self.xpath('//foxml:property[@NAME="info:fedora/fedora-system:def/model#label"]/@VALUE').first.value = value[0..254]
+    if existing_title = self.get_datastream("DC","//dc:title")
+      existing_title.remove
+    end
+    new_child = @xml.create_element('dc:title')
+    new_child.content = value
+    self.dublin_core.add_child(new_child)
+    fix_namespaces(new_child)
+    return new_child
+  end
+
+  def admin_policy_object
+    self.get_rels_ext_resource("hydra:isGovernedBy")
+  end
+  
+  def admin_policy_object=(value)
+    self.set_rels_ext_resource("hydra:isGovernedBy",value)
+  end
+  
+  def parent
+    self.get_rels_ext_resource("rel:isPartOf")
+  end
+  
+  def parent=(value)
+    self.set_rels_ext_resource("rel:isPartOf",value)
+  end
+  
+  def get_rels_ext_resource(predicate)
+    resource = self.get_datastream("RELS-EXT","//#{predicate}/@rdf:resource")
+    if resource.nil?
+      nil
+    else
+      resource.value.split(/\//).last
+    end
+  end
+  
+  def set_rels_ext_resource(predicate,value)
+    if existing_parent = self.get_datastream("RELS-EXT","//#{predicate}")
+      existing_parent.remove
+    end
+    if value.nil?
+      return nil
+    else
+      new_child = @xml.create_element(predicate, 'rdf:resource' => "info:fedora/#{value}")
+      self.get_datastream("RELS-EXT","//rdf:Description").add_child(new_child)
+      fix_namespaces(new_child)
+      return new_child
+    end
+  end
+  
+  def get_datastream(ds_name, *paths)
+    result = self.xpath(%{//foxml:datastream[@ID="#{ds_name}"]/foxml:datastreamVersion/foxml:xmlContent}).first
+    paths.each do |path|
+      result = result.xpath(path, NAMESPACES).first
+    end
+    return result
+  end
+  
+  def set_datastream(ds_name, new_value, nil_value = nil)
+    new_value ||= nil_value
+    if new_value.is_a?(Nokogiri::XML::Document)
+      new_value = new_value.root.clone
+    elsif new_value.is_a?(Nokogiri::XML::Node)
+      new_value = new_value.clone
+    end
+    parent = self.get_datastream(ds_name)
+    parent.children.each { |c| c.remove }
+    unless new_value.nil?
+      parent.add_child(new_value)
+    end
+  end
+  
+  def to_xml(opts = {})
+    undent_datastreams = opts.delete(:undent_datastreams)
+    @xml.traverse { |node| if node.is_a?(Nokogiri::XML::Text) and node.text.chomp.strip.empty?; node.remove; end }
+    result = @xml.to_xml(opts)
+
+    # Fedora FOXML ingest treats some (but not all) whitespace as significant. In order to get
+    # our datastreams to look pretty, we need to un-indent them. This an ugly hack, and only
+    # works because we've strictly controlled Nokogiri's serialization of whitespace.
+    if undent_datastreams
+      result.gsub!(/^(\s*)<foxml:xmlContent>\n(.+?)\n\s*<\/foxml:xmlContent>/m) { |m|
+        sp = $1
+        data = $2
+        "#{sp}<foxml:xmlContent>\n" + data.gsub(/^#{sp}  /,'') + "\n#{sp}</foxml:xmlContent>"
+      }
+    end
+    return result
+  end
+  
+  def xpath(path)
+    @xml.xpath(path, NAMESPACES)
+  end
+  
+  def update_dc_identifiers
+    new_ids = self.identity_metadata.xpath('*[local-name() = "sourceId" or local-name() = "otherId"]').collect do |n| 
+      n.attribute_nodes.first.value + ":" + n.content
+    end
+    unless self.pid.nil? or self.pid.empty?
+      new_ids << self.pid 
+    end
+    
+    dc = self.dublin_core
+    dc.xpath('dc:identifier',NAMESPACES).each { |existing_id| existing_id.remove }
+    new_ids.uniq.each do |id|
+      new_child = @xml.create_element('dc:identifier')
+      new_child.content = id
+      dc.add_child(new_child)
+      fix_namespaces(new_child)
+    end
+    return new_ids
+  end
+  
+  def fix_namespaces(node)
+    if node.is_a?(Nokogiri::XML::CharacterData)
+      node.namespace = nil
+    else
+      if node.name =~ /:/
+        (prefix,name) = node.name.split(/:/)
+        node.name = name
+        node.namespace = @defined_namespaces[prefix]
+      end
+      unless node.is_a?(Nokogiri::XML::Attr)
+        node.children.each { |n| fix_namespaces(n) }
+        node.attribute_nodes.each { |n| fix_namespaces(n) }
+      end
+    end
+  end
+  
+  def method_missing(sym,*args)
+    if @xml.respond_to?(sym)
+      @xml.send(sym,*args)
+    else
+      super
+    end
+  end
+  
+  XML_TEMPLATE = %{
+<?xml version="1.0" encoding="UTF-8"?>
+<foxml:digitalObject PID="$$PID$$" VERSION="1.1"
+  xmlns:foxml="info:fedora/fedora-system:def/foxml#"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="info:fedora/fedora-system:def/foxml# http://www.fedora.info/definitions/1/0/foxml1-1.xsd">
+  <foxml:objectProperties>
+    <foxml:property NAME="info:fedora/fedora-system:def/model#state" VALUE="Active"/>
+    <foxml:property NAME="info:fedora/fedora-system:def/model#label" VALUE="$$LABEL$$"/>
+    <foxml:property NAME="info:fedora/fedora-system:def/model#ownerId" VALUE="DOR"/>
+  </foxml:objectProperties>
+  <foxml:datastream CONTROL_GROUP="X" ID="DC" STATE="A" VERSIONABLE="false">
+    <foxml:datastreamVersion
+      FORMAT_URI="http://www.openarchives.org/OAI/2.0/oai_dc/" ID="DC1.0"
+      LABEL="Dublin Core Record for this object" MIMETYPE="text/xml">
+        <foxml:xmlContent>
+          <oai_dc:dc xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"/>
+        </foxml:xmlContent>
+    </foxml:datastreamVersion>
+  </foxml:datastream>
+  <foxml:datastream CONTROL_GROUP="X" ID="identityMetadata" STATE="A" VERSIONABLE="false">
+    <foxml:datastreamVersion
+      ID="identityMetadata.0" LABEL="Identity Metadata" MIMETYPE="text/xml">
+        <foxml:xmlContent>
+          <identityMetadata/>
+        </foxml:xmlContent>
+    </foxml:datastreamVersion>
+  </foxml:datastream>
+  <foxml:datastream CONTROL_GROUP="X" ID="RELS-EXT" STATE="A">
+    <foxml:datastreamVersion
+      FORMAT_URI="info:fedora/fedora-system:FedoraRELSExt-1.0" ID="RELS-EXT.0"
+      LABEL="RDF Statements about this object" MIMETYPE="application/rdf+xml">
+      <foxml:xmlContent>
+        <rdf:RDF xmlns:fedora-model="info:fedora/fedora-system:def/model#" 
+            xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" 
+            xmlns:fedora="info:fedora/fedora-system:def/relations-external#"
+            xmlns:hydra="http://projecthydra.org/ns/relations#">
+            <rdf:Description rdf:about="info:fedora/$$PID$$">
+            </rdf:Description>
+        </rdf:RDF>
+      </foxml:xmlContent>
+    </foxml:datastreamVersion>
+  </foxml:datastream>
+</foxml:digitalObject>
+}.strip
+
+end