geoblacklight-schema 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9ff360f9e0d93b2b8973cb0b7a698a83ed7df395
+  data.tar.gz: 214fe068e10569e722f8659815216dd4829f45a3
+SHA512:
+  metadata.gz: de1dde11f1a3f1f622d1bc7a84c1f863a244e14cd7a055c012cb0f5d2b97ba447132dc01763988f6fca1fb7a35bd9df74697017806fa8fc9ad6b844693b37020
+  data.tar.gz: bd45f44fe4f4bf18c1e8372ad53ad9414204be49f4cb0844b8421575c072e9ce69e4d76babc9f0a18bc8bd9bbece2a645c2b77b1e9dc2da178696071f6ddaa4a

data/.gitignore

@@ -0,0 +1,3 @@
+Gemfile.lock
+data
+vendor/*.jar

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (c) 2014 by The Board of Trustees of the Leland Stanford
+Junior University.  All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you
+may not use this file except in compliance with the License.  You
+may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied.  See the License for the specific language governing
+permissions and limitations under the License.

data/README.md

@@ -0,0 +1,44 @@
+#### Introduction
+
+This schema focuses on discovery use cases for patrons and analysts in a
+research library setting, although it is likely useful in other settings. Text
+search, faceted search and refinement, and spatial search and relevancy are
+among the primary features that the schema enables. The schema design supports
+a variety of discovery applications and GIS data types. We especially wanted to
+provide for contextual collection-oriented discovery applications as well as
+traditional portal applications.
+
+The source code for the Solr implementation and the design document are on
+Github:
+
+  http://github.com/geoblacklight/geoblacklight-schema
+
+The conf/ folder has everything you need to implement the schema in Solr 4.7,
+and the examples/ folder has 100 example Solr documents that you can use. The
+lib/ folder has some initial, but incomplete, implementations for metadata
+format conversions (e.g., FGDC -> MODS, OGP -> GeoBlacklight, etc.).
+
+#### Example
+
+The `examples` folder has some Solr documents that uses this schema. First,
+install the schema into a Solr 4 instance, then upload the documents.
+
+```bash
+% cd examples
+% ruby upload-to-solr.rb your-collection-name http://localhost:8080/solr
+```
+
+## Requirements
+
+* Saxon-HE v9, the Home Edition from http://saxon.sourceforge.net installed as
+  `vendor/saxon9he.jar`
+
+#  Schema for GeoBlacklight
+
+Please refer to http://journal.code4lib.org/articles/9710 which describes the schema in detail.
+
+* Hardy, D., K. Durante. 2014. A Metadata Schema for Geospatial Resource Discovery Use Cases. _code4lib_ 25. 
+
+Abstract: We introduce a metadata schema that focuses on GIS discovery use cases for patrons in a research library setting. Text search, faceted refinement, and spatial search and relevancy are among GeoBlacklight’s primary use cases for federated geospatial holdings. The schema supports a variety of GIS data types and enables contextual, collection-oriented discovery applications as well as traditional portal applications. One key limitation of GIS resource discovery is the general lack of normative metadata practices, which has led to a proliferation of metadata schemas and duplicate records. The ISO 19115/19139 and FGDC standards specify metadata formats, but are intricate, lengthy, and not focused on discovery. Moreover, they require sophisticated authoring environments and cataloging expertise. Geographic metadata standards target preservation and quality measure use cases, but they do not provide for simple inter-institutional sharing of metadata for discovery use cases. To this end, our schema reuses elements from Dublin Core and GeoRSS to leverage their normative semantics, community best practices, open-source software implementations, and extensive examples already deployed in discovery contexts such as web search and mapping. Finally, we discuss a Solr implementation of the schema using a “geo” extension to MODS.
+
+

data/bin/fgdc2mods.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+cmd = "bin/xsltproc-saxon lib/xslt/fgdc2mods.xsl #{ARGV[0]}"
+# puts cmd
+system cmd

data/bin/mods2geoblacklight.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+cmd = "bin/xsltproc-saxon lib/xslt/mods2geoblacklight.xsl #{ARGV[0]}"
+# puts cmd
+system cmd

data/bin/xsltproc-saxon

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# Usage: xsltproc-saxon stylesheet.xsl input.xml [output.xml [flags or params]]
+#
+# See http://saxonica.com/documentation/html/using-xsl/commandline.html
+#
+s=$1; shift
+i=$1; shift
+o=$1; shift
+if [ -n "$o" ]; then
+  o="-o:$o"
+fi
+
+java -jar ../vendor/saxon9he.jar "-s:$i" "-xsl:$s" $o $*

data/conf/protwords.txt

@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+

data/conf/schema.xml

@@ -0,0 +1,158 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema name="geoblacklight-schema" version="1.5">
+  <uniqueKey>uuid</uniqueKey>
+  <fields>
+    <field name="_version_" type="long"   stored="true" indexed="true"/>
+    <field name="timestamp" type="date"   stored="true" indexed="true" default="NOW"/>
+    <field name="uuid"      type="string" stored="true" indexed="true" required="true"/>
+
+    <!-- core generated fields -->
+    <field name="text" type="text_en" stored="false" indexed="true" multiValued="true"
+                       termVectors="true" termPositions="true" termOffsets="true" />
+
+    <!-- dynamic field with simple types by suffix -->
+    <dynamicField name="*_b"    type="boolean" stored="true"  indexed="true"/>
+    <dynamicField name="*_d"    type="double"  stored="true"  indexed="true"/>
+    <dynamicField name="*_dt"   type="date"    stored="true"  indexed="true"/>
+    <dynamicField name="*_f"    type="float"   stored="true"  indexed="true"/>
+    <dynamicField name="*_i"    type="int"     stored="true"  indexed="true"/>
+    <dynamicField name="*_l"    type="long"    stored="true"  indexed="true"/>
+    <dynamicField name="*_s"    type="string"  stored="true"  indexed="true"/>
+    <dynamicField name="*_ss"   type="string"  stored="true"  indexed="false"/>
+    <dynamicField name="*_si"   type="string"  stored="false" indexed="true"/>
+    <dynamicField name="*_sim"  type="string"  stored="false" indexed="true" multiValued="true" />
+    <dynamicField name="*_sm"   type="string"  stored="true"  indexed="true" multiValued="true" />
+    <dynamicField name="*_url"  type="string"  stored="true"  indexed="false"/>
+    <dynamicField name="*_blob" type="binary"  stored="true"  indexed="false"/>
+
+    <!-- dynamic Text fields by suffix without storage -->
+    <dynamicField name="*_t"    type="text_en" stored="false"  indexed="true"
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tm"   type="text_en" stored="false"  indexed="true" multiValued="true"
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_ti"   type="text_en" stored="false" indexed="true"
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tmi"  type="text_en" stored="false" indexed="true" multiValued="true"
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_sort" type="text_sort" stored="false" indexed="true" multiValued="false"/>
+
+    <!-- Spatial field types:
+
+         Solr3:
+           <field name="my_pt">83.1,-117.312</field>
+             as (y,x)
+
+         Solr4:
+
+           <field name="my_bbox">-117.312 83.1 -115.39 84.31</field>
+             as (W S E N)
+
+           <field name="my_geom">ENVELOPE(-117.312, -115.39, 84.31, 83.1)</field>
+             as (W E N S)
+
+           <field name="my_jts">POLYGON((1 8, 1 9, 2 9, 2 8, 1 8))</field>
+             as WKT for point, linestring, polygon
+
+      -->
+    <dynamicField name="*_pt"   type="location"     stored="true" indexed="true"/>
+    <dynamicField name="*_bbox" type="location_rpt" stored="true" indexed="true"/>
+    <dynamicField name="*_geom" type="location_rpt" stored="true" indexed="true"/>
+    <!-- <dynamicField name="*_jts"  type="location_jts" stored="true" indexed="true"/> -->
+  </fields>
+
+  <types>
+    <fieldType name="string"  class="solr.StrField"  sortMissingLast="true" />
+    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+
+    <fieldType name="int"    class="solr.TrieIntField"     precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="float"  class="solr.TrieFloatField"   precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="long"   class="solr.TrieLongField"    precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField"  precisionStep="8" positionIncrementGap="0"/>
+
+    <!-- The format for this date field is of the form 1995-12-31T23:59:59Z.
+         The trailing "Z" designates UTC time and is mandatory.
+         A Trie based date field for faster date range queries and date faceting. -->
+    <fieldType name="date" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+
+    <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
+    <fieldtype name="binary" class="solr.BinaryField"/>
+
+    <!-- A text field with defaults appropriate for English: it
+         tokenizes with StandardTokenizer, removes English stop words
+         (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
+         finally applies Porter's stemming.  The query time analyzer
+         also applies synonyms from synonyms.txt. -->
+    <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords_en.txt"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+        <filter class="solr.PorterStemFilterFactory"/>
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords_en.txt"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+        <filter class="solr.PorterStemFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- for alpha sorting as a single token -->
+    <fieldType name="text_sort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+      <analyzer>
+        <tokenizer class="solr.KeywordTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory" />
+        <filter class="solr.TrimFilterFactory" />
+        <filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z0-9 ])" replacement="" replace="all"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- Spatial field types -->
+    <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
+
+    <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
+               distErrPct="0.025"
+               maxDistErr="0.000009"
+               units="degrees"
+            />
+
+    <!-- JTS-enabled spatial predicates; requires JTS installation -->
+    <!-- <fieldType name="location_jts" class="solr.SpatialRecursivePrefixTreeFieldType"
+               spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory"
+               distErrPct="0.025"
+               maxDistErr="0.000009"
+               units="degrees"
+            /> -->
+  </types>
+
+  <!-- for scoring formula -->
+  <copyField source="dct_spatial_sm"     dest="dct_spatial_tmi"     maxChars="10000"/>
+  <copyField source="dct_temporal_sm"    dest="dct_temporal_tmi"    maxChars="10000"/>
+  <copyField source="dc_creator_sm"      dest="dc_creator_tmi"      maxChars="1000"/>
+  <copyField source="dc_description_s"   dest="dc_description_ti"   maxChars="10000"/>
+  <copyField source="dc_format_s"        dest="dc_format_ti"        maxChars="100"/>
+  <copyField source="dc_identifier_s"    dest="dc_identifier_ti"    maxChars="100"/>
+  <copyField source="dc_publisher_s"     dest="dc_publisher_ti"     maxChars="1000"/>
+  <copyField source="dc_rights_s"        dest="dc_rights_ti"        maxChars="100"/>
+  <copyField source="dct_provenance_s"   dest="dct_provenance_ti"   maxChars="1000"/>
+  <copyField source="dc_subject_sm"      dest="dc_subject_tmi"      maxChars="10000"/>
+  <copyField source="dc_title_s"         dest="dc_title_ti"         maxChars="1000"/>
+  <copyField source="dct_isPartOf_sm"    dest="dct_isPartOf_tmi"    maxChars="1000"/>
+  <copyField source="layer_geom_type_s"  dest="layer_geom_type_ti"  maxChars="100"/>
+  <copyField source="layer_slug_s"       dest="layer_slug_ti"       maxChars="100"/>
+
+  <!-- core text search -->
+  <copyField source="*_ti"               dest="text" />
+  <copyField source="*_tmi"              dest="text" />
+
+  <!-- for sorting text fields -->
+  <copyField source="dct_provenance_s"   dest="dct_provenance_sort"/>
+  <copyField source="dc_publisher_s"     dest="dc_publisher_sort"/>
+  <copyField source="dc_title_s"         dest="dc_title_sort"/>
+
+</schema>

data/conf/solrconfig.xml

@@ -0,0 +1,160 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- 
+     For more details about configurations options that may appear in
+     this file, see http://wiki.apache.org/solr/SolrConfigXml. 
+-->
+<config>
+  <luceneMatchVersion>4.7</luceneMatchVersion>
+  <dataDir>${solr.data.dir:}</dataDir>
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+  <codecFactory class="solr.SchemaCodecFactory"/>
+  <schemaFactory class="ClassicIndexSchemaFactory"/>
+  <indexConfig>
+    <lockType>${solr.lock.type:native}</lockType>
+  </indexConfig>
+
+  <!-- The default high-performance update handler -->
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.ulog.dir:}</str>
+    </updateLog>
+    <autoCommit>
+      <maxTime>15000</maxTime>
+      <openSearcher>false</openSearcher>
+    </autoCommit>
+  </updateHandler>
+
+  <!-- realtime get handler, guaranteed to return the latest stored fields 
+    of any document, without the need to commit or open a new searcher. The current 
+    implementation relies on the updateLog feature being enabled. -->
+  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+    <lst name="defaults">
+      <str name="omitHeader">true</str>
+    </lst>
+  </requestHandler>  
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
+
+  <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+       Query section - these settings control query time things like caches
+       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+  <query>
+    <maxBooleanClauses>1024</maxBooleanClauses>
+    <filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <enableLazyFieldLoading>true</enableLazyFieldLoading>
+    <queryResultWindowSize>20</queryResultWindowSize>
+    <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+    <listener event="newSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+       <lst><str name="q">stanford</str></lst>
+       <lst><str name="q">polygon</str></lst>
+      </arr>
+    </listener>
+    <listener event="firstSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+        <lst>
+          <str name="q">static firstSearcher warming in solrconfig.xml</str>
+        </lst>
+      </arr>
+    </listener>
+    <useColdSearcher>false</useColdSearcher>
+    <maxWarmingSearchers>2</maxWarmingSearchers>
+  </query>
+
+  <requestDispatcher handleSelect="false">
+    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048"/>
+    <httpCaching never304="true"/>
+   </requestDispatcher>
+
+  <requestHandler name="/search" class="solr.SearchHandler"/>
+
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <lst name="defaults">
+      <int name="start">0</int>
+      <int name="rows">10</int>
+      <str name="wt">json</str>
+      <int name="indent">2</int>
+      <str name="defType">edismax</str>
+      <str name="echoParams">all</str>
+      <str name="fl">*,score</str>
+      <str name="sort">score desc, dc_title_sort asc</str>
+      <str name="q.alt">*:*</str>
+      <str name="qf">
+        text^1
+        dc_description_ti^2
+        dc_creator_tmi^3
+        dc_publisher_ti^3
+        dct_isPartOf_tmi^4
+        dc_subject_tmi^5
+        dct_spatial_tmi^5
+        dct_temporal_tmi^5
+        dc_title_ti^6
+        dc_rights_ti^7
+        dct_provenance_ti^8
+        layer_geom_type_ti^9
+        layer_slug_ti^10
+        dc_identifier_ti^10
+      </str>
+      <bool name="facet">true</bool>
+      <int name="facet.mincount">1</int>
+      <int name="facet.limit">10</int>
+      <str name="facet.field">dct_isPartOf_sm</str>
+      <str name="facet.field">dct_provenance_s</str>
+      <str name="facet.field">dct_spatial_sm</str>
+      <str name="facet.field">dc_creator_sm</str>
+      <str name="facet.field">dc_format_s</str>
+      <str name="facet.field">dc_language_s</str>
+      <str name="facet.field">dc_publisher_s</str>
+      <str name="facet.field">dc_rights_s</str>
+      <str name="facet.field">dc_subject_sm</str>
+      <str name="facet.field">layer_geom_type_s</str>
+      <str name="facet.field">solr_year_i</str>
+    </lst>
+  </requestHandler>
+
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"/>
+  <requestHandler name="/admin/" class="solr.admin.AdminHandlers"/>
+
+  <!-- ping/healthcheck -->
+  <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+    <lst name="invariants">
+      <str name="q">solrpingquery</str>
+    </lst>
+    <lst name="defaults">
+      <str name="echoParams">all</str>
+    </lst>
+    <!-- An optional feature of the PingRequestHandler is to configure the 
+         handler with a "healthcheckFile" which can be used to enable/disable 
+         the PingRequestHandler.
+         relative paths are resolved against the data dir 
+      -->
+    <str name="healthcheckFile">server-enabled.txt</str>
+  </requestHandler>
+  
+  <requestHandler name="/analysis/field" 
+                  startup="lazy"
+                  class="solr.FieldAnalysisRequestHandler" />
+                  
+  <!-- Legacy config for the admin interface -->
+  <admin>
+    <defaultQuery>*:*</defaultQuery>
+  </admin>
+</config>

data/conf/stopwords_en.txt

@@ -0,0 +1,34 @@
+# Standard english stop words taken from Lucene's StopAnalyzer
+a
+an
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with