geohydra 0.3.1 → 0.3.3

data/scripts/status.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require 'csv'
+# require 'json'
+require 'net/http'
+# require 'awesome_print'
+
+STDOUT.sync = true
+
+CSV.foreach('status.csv') do |url|
+  # ap({:url => url.first})
+  uri = URI(url.first)
+  druid = 'unknown'
+  druid = $1 if uri.to_s =~ /druid%3A([a-z0-9]+)/
+  # ap({:uri => uri})
+  uri.host = 'localhost'
+  uri.port = 8080
+  # ap({:uri => uri})
+  begin
+    start = Time.now
+    res = Net::HTTP.get_response(uri)
+    sz = res.body.size
+    open("images/#{druid}.png", 'wb') do |f|
+      f.write(res.body)
+    end
+    finish = Time.now
+    puts [finish, druid, res.code, res['content-type'], sz, res['geowebcache-cache-result'], finish - start].join(', ')
+  rescue => e
+    puts e.class, e
+  end
+  sleep(0.5)
+end

data/{bin → scripts}/validate_data.rb

@@ -10,7 +10,7 @@ def validate(path, flags)
     puts "Processing #{shp}" if flags[:debug]
     basefn = File.basename(shp, '.shp')
     unless GeoHydra::Utils.shapefile?(shp)
-      puts "Error <#{shp}>. Trying to repair..."
+      puts "SyntaxError: Not a shapefile <#{shp}>. Trying to repair..."
       Dir.glob("#{File.dirname(shp)}/#{basefn.gsub(' ', "\\ ")}.*") do |fn|
         newfn = File.join(File.dirname(fn), File.basename(fn).gsub(/[^a-zA-Z0-9_]/, '_'))
         FileUtils.mv fn, newfn

data/solr/kurma-app-dev/conf/lang/stopwords_en.txt

@@ -0,0 +1,34 @@
+# Standard english stop words taken from Lucene's StopAnalyzer
+a
+an
+and
+are
+as
+at
+be
+but
+by
+for
+if
+in
+into
+is
+it
+no
+not
+of
+on
+or
+such
+that
+the
+their
+then
+there
+these
+they
+this
+to
+was
+will
+with

data/solr/kurma-app-dev/conf/protwords.txt

@@ -0,0 +1,21 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+# Use a protected word file to protect against the stemmer reducing two
+# unrelated words to the same base word.
+
+# Some non-words that normally won't be encountered,
+# just to test that they won't be stemmed.
+dontstems
+zwhacky
+

data/solr/kurma-app-dev/conf/schema.xml

@@ -0,0 +1,156 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema name="kurma-app-dev" version="1.5">
+  <uniqueKey>uuid</uniqueKey>
+  <fields>
+    <field name="_version_" type="long"   stored="true" indexed="true"/>
+    <field name="timestamp" type="date"   stored="true" indexed="true" default="NOW"/>
+    <field name="uuid"      type="string" stored="true" indexed="true" required="true"/>
+    
+    <!-- core generated fields -->
+    <field name="text" type="text_en" stored="false" indexed="true" multiValued="true" 
+                       termVectors="true" termPositions="true" termOffsets="true" />
+
+    <!-- dynamic field with simple types by suffix -->
+    <dynamicField name="*_b"    type="boolean" stored="true"  indexed="true"/>
+    <dynamicField name="*_d"    type="double"  stored="true"  indexed="true"/>
+    <dynamicField name="*_dt"   type="date"    stored="true"  indexed="true"/>
+    <dynamicField name="*_f"    type="float"   stored="true"  indexed="true"/>
+    <dynamicField name="*_i"    type="int"     stored="true"  indexed="true"/>
+    <dynamicField name="*_l"    type="long"    stored="true"  indexed="true"/>
+    <dynamicField name="*_s"    type="string"  stored="true"  indexed="true"/>
+    <dynamicField name="*_ss"   type="string"  stored="true"  indexed="false"/>
+    <dynamicField name="*_si"   type="string"  stored="false" indexed="true"/>
+    <dynamicField name="*_sim"  type="string"  stored="false" indexed="true" multiValued="true" />
+    <dynamicField name="*_sm"   type="string"  stored="true"  indexed="true" multiValued="true" />
+    <dynamicField name="*_url"  type="string"  stored="true"  indexed="false"/>
+    <dynamicField name="*_blob" type="binary"  stored="true"  indexed="false"/>
+
+    <!-- dynamic Text fields by suffix without storage -->
+    <dynamicField name="*_t"    type="text_en" stored="false"  indexed="true" 
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tm"   type="text_en" stored="false"  indexed="true" multiValued="true" 
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_ti"   type="text_en" stored="false" indexed="true" 
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_tmi"  type="text_en" stored="false" indexed="true" multiValued="true" 
+                                termVectors="true" termPositions="true" termOffsets="true"/>
+    <dynamicField name="*_sort" type="text_sort" stored="false" indexed="true" multiValued="false"/>
+
+    <!-- Spatial field types:
+    
+         Solr3:
+           <field name="my_pt">83.1,-117.312</field> 
+             as (y,x)
+
+         Solr4:             
+
+           <field name="my_bbox">-117.312 83.1 -115.39 84.31</field> 
+             as (W S E N)
+
+           <field name="my_geom">POLYGON((1 8, 1 9, 2 9, 2 8, 1 8))</field> 
+             as WKT for point, linestring, polygon
+
+      -->
+    <dynamicField name="*_pt"   type="location"     stored="true" indexed="true"/>
+    <dynamicField name="*_bbox" type="location_rpt" stored="true" indexed="true"/>
+    <dynamicField name="*_geom" type="location_jts" stored="true" indexed="true"/>
+    <dynamicField name="*_wkt"  type="location_jts" stored="true" indexed="true"/>
+  </fields>
+
+  <types>
+    <fieldType name="string"  class="solr.StrField"  sortMissingLast="true" />
+    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+
+    <fieldType name="int"    class="solr.TrieIntField"     precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="float"  class="solr.TrieFloatField"   precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="long"   class="solr.TrieLongField"    precisionStep="8" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.TrieDoubleField"  precisionStep="8" positionIncrementGap="0"/>
+
+    <!-- The format for this date field is of the form 1995-12-31T23:59:59Z.
+         The trailing "Z" designates UTC time and is mandatory. 
+         A Trie based date field for faster date range queries and date faceting. -->
+    <fieldType name="date" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+
+    <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
+    <fieldtype name="binary" class="solr.BinaryField"/>
+
+    <!-- A text field with defaults appropriate for English: it
+         tokenizes with StandardTokenizer, removes English stop words
+         (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
+         finally applies Porter's stemming.  The query time analyzer
+         also applies synonyms from synonyms.txt. -->
+    <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
+      <analyzer type="index">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+        <filter class="solr.PorterStemFilterFactory"/>
+      </analyzer>
+      <analyzer type="query">
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
+        <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.EnglishPossessiveFilterFactory"/>
+        <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
+        <filter class="solr.PorterStemFilterFactory"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- for alpha sorting as a single token -->
+    <fieldType name="text_sort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+      <analyzer>
+        <tokenizer class="solr.KeywordTokenizerFactory"/>
+        <filter class="solr.LowerCaseFilterFactory" />
+        <filter class="solr.TrimFilterFactory" />
+        <filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z0-9 ])" replacement="" replace="all"/>
+      </analyzer>
+    </fieldType>
+
+    <!-- Spatial field types -->
+    <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
+
+    <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
+               distErrPct="0.025"
+               maxDistErr="0.000009"
+               units="degrees"
+            />
+
+    <!-- JTS-enabled spatial predicates; requires JTS installation -->
+    <fieldType name="location_jts" class="solr.SpatialRecursivePrefixTreeFieldType"
+               spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory"
+               distErrPct="0.025"
+               maxDistErr="0.000009"
+               units="degrees"
+            />
+  </types>
+
+  <!-- for scoring formula -->
+  <copyField source="dct_spatial_sm"     dest="dct_spatial_tmi" maxChars="10000"/>
+  <copyField source="dct_temporal_sm"    dest="dct_temporal_tmi" maxChars="10000"/>
+  <copyField source="dc_creator_sm"      dest="dc_creator_tmi"      maxChars="1000"/>
+  <copyField source="dc_description_s"   dest="dc_description_ti"   maxChars="10000"/>
+  <copyField source="dc_format_s"        dest="dc_format_ti"        maxChars="100"/>  
+  <copyField source="dc_identifier_s"    dest="dc_identifier_ti"    maxChars="100"/>
+  <copyField source="dc_publisher_s"     dest="dc_publisher_ti"     maxChars="1000"/>
+  <copyField source="dc_rights_s"        dest="dc_rights_ti"        maxChars="100"/>
+  <copyField source="dct_provenance_s"   dest="dct_provenance_ti"    maxChars="1000"/>
+  <copyField source="dc_subject_sm"      dest="dc_subject_tmi"      maxChars="10000"/>
+  <copyField source="dc_title_s"         dest="dc_title_ti"         maxChars="1000"/>
+  <copyField source="layer_collection_s" dest="layer_collection_ti" maxChars="1000"/>
+  <copyField source="layer_geom_type_s"  dest="layer_geom_type_ti"  maxChars="100"/>
+  <copyField source="layer_slug_s"       dest="layer_slug_ti"       maxChars="100"/>
+
+  <!-- core text search -->
+  <copyField source="*_ti"               dest="text" />
+  <copyField source="*_tmi"              dest="text" />
+
+  <!-- for sorting text fields -->
+  <copyField source="dc_title_s"         dest="dc_title_sort"/>
+  <copyField source="dc_publisher_s"     dest="dc_publisher_sort"/>
+  <copyField source="layer_collection_s" dest="layer_collection_sort"/>
+
+</schema>
+

data/solr/kurma-app-dev/conf/solrconfig.xml

@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- 
+     For more details about configurations options that may appear in
+     this file, see http://wiki.apache.org/solr/SolrConfigXml. 
+-->
+<config>
+  <luceneMatchVersion>LUCENE_CURRENT</luceneMatchVersion>
+  <dataDir>${solr.data.dir:}</dataDir>
+  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+  <codecFactory class="solr.SchemaCodecFactory"/>
+  <schemaFactory class="ClassicIndexSchemaFactory"/>
+  <indexConfig>
+    <lockType>${solr.lock.type:native}</lockType>
+  </indexConfig>
+
+  <!-- The default high-performance update handler -->
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <updateLog>
+      <str name="dir">${solr.ulog.dir:}</str>
+    </updateLog>
+    <autoCommit>
+      <maxTime>15000</maxTime>
+      <openSearcher>false</openSearcher>
+    </autoCommit>
+  </updateHandler>
+
+  <!-- realtime get handler, guaranteed to return the latest stored fields 
+    of any document, without the need to commit or open a new searcher. The current 
+    implementation relies on the updateLog feature being enabled. -->
+  <requestHandler name="/get" class="solr.RealTimeGetHandler">
+    <lst name="defaults">
+      <str name="omitHeader">true</str>
+    </lst>
+  </requestHandler>  
+  
+  <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
+
+  <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+       Query section - these settings control query time things like caches
+       ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -->
+  <query>
+    <maxBooleanClauses>1024</maxBooleanClauses>
+    <filterCache class="solr.FastLRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <queryResultCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <documentCache class="solr.LRUCache" size="512" initialSize="512" autowarmCount="0"/>
+    <enableLazyFieldLoading>true</enableLazyFieldLoading>
+    <queryResultWindowSize>20</queryResultWindowSize>
+    <queryResultMaxDocsCached>200</queryResultMaxDocsCached>
+    <listener event="newSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+       <lst><str name="q">stanford</str></lst>
+       <lst><str name="q">polygon</str></lst>
+      </arr>
+    </listener>
+    <listener event="firstSearcher" class="solr.QuerySenderListener">
+      <arr name="queries">
+        <lst>
+          <str name="q">static firstSearcher warming in solrconfig.xml</str>
+        </lst>
+      </arr>
+    </listener>
+    <useColdSearcher>false</useColdSearcher>
+    <maxWarmingSearchers>2</maxWarmingSearchers>
+  </query>
+
+  <requestDispatcher handleSelect="false">
+    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048"/>
+    <httpCaching never304="true"/>
+   </requestDispatcher>
+
+  <requestHandler name="/search" class="solr.SearchHandler"/>
+
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <lst name="defaults">
+      <int name="start">0</int>
+      <int name="rows">10</int>
+      <str name="wt">json</str>
+      <int name="indent">2</int>
+      <str name="defType">edismax</str>
+      <str name="echoParams">all</str>
+      <str name="fl">*,score</str>
+      <str name="sort">score desc, dc_title_sort asc</str>
+      <str name="q.alt">*:*</str>
+      <str name="qf">
+        text^1
+        dc_description_ti^2
+        dc_creator_ti^3
+        dc_publisher_ti^3
+        layer_collection_ti^4
+        dc_subject_tmi^5
+        dct_spatial_tmi^5
+        dct_temporal_tmi^5
+        dc_title_ti^6
+        dc_rights_ti^7
+        dct_provenance_ti^8
+        layer_geom_type_ti^9
+        layer_slug_ti^10
+        dc_identifier_ti^10
+      </str>
+      <bool name="facet">true</bool>
+      <int name="facet.mincount">1</int>
+      <int name="facet.limit">10</int>
+      <str name="facet.field">dct_spatial_sm</str>
+      <str name="facet.field">dc_creator_s</str>
+      <str name="facet.field">dc_format_s</str>
+      <str name="facet.field">dc_language_s</str>
+      <str name="facet.field">dc_publisher_s</str>
+      <str name="facet.field">dc_rights_s</str>
+      <str name="facet.field">dct_provenance_s</str>
+      <str name="facet.field">dc_subject_sm</str>
+      <str name="facet.field">layer_collection_s</str>
+      <str name="facet.field">layer_geom_type_s</str>
+      <str name="facet.field">layer_srs_s</str>
+      <str name="facet.field">layer_year_i</str>
+    </lst>
+  </requestHandler>
+
+  <requestHandler name="/update" class="solr.UpdateRequestHandler"/>
+  <requestHandler name="/admin/" class="solr.admin.AdminHandlers"/>
+
+  <!-- ping/healthcheck -->
+  <requestHandler name="/admin/ping" class="solr.PingRequestHandler">
+    <lst name="invariants">
+      <str name="q">solrpingquery</str>
+    </lst>
+    <lst name="defaults">
+      <str name="echoParams">all</str>
+    </lst>
+    <!-- An optional feature of the PingRequestHandler is to configure the 
+         handler with a "healthcheckFile" which can be used to enable/disable 
+         the PingRequestHandler.
+         relative paths are resolved against the data dir 
+      -->
+    <str name="healthcheckFile">server-enabled.txt</str>
+  </requestHandler>
+  
+  <requestHandler name="/analysis/field" 
+                  startup="lazy"
+                  class="solr.FieldAnalysisRequestHandler" />
+                  
+  <!-- Legacy config for the admin interface -->
+  <admin>
+    <defaultQuery>*:*</defaultQuery>
+  </admin>
+</config>

data/solr/kurma-app-dev/conf/synonyms.txt

@@ -0,0 +1,29 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaafoo => aaabar
+bbbfoo => bbbfoo bbbbar
+cccfoo => cccbar cccbaz
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma
+