geo_combine 0.2.0 → 0.5.1

data/lib/geo_combine/railtie.rb

@@ -0,0 +1,7 @@
+module GeoCombine
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'tasks/geo_combine.rake'
+    end
+  end
+end

data/lib/geo_combine/version.rb

@@ -1,3 +1,3 @@
 module GeoCombine
-  VERSION = '0.2.0'
+  VERSION = '0.5.1'
 end

data/lib/tasks/geo_combine.rake

@@ -1,39 +1,73 @@
-require 'find'
 require 'net/http'
 require 'json'
 require 'rsolr'
+require 'find'
+require 'geo_combine/geo_blacklight_harvester'
 
 namespace :geocombine do
+  commit_within = (ENV['SOLR_COMMIT_WITHIN'] || 5000).to_i
   ogm_path = ENV['OGM_PATH'] || 'tmp/opengeometadata'
   solr_url = ENV['SOLR_URL'] || 'http://127.0.0.1:8983/solr/blacklight-core'
-  desc 'Clone all OpenGeoMetadata repositories'
-  task :clone do
-    ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
-    ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map{ |repo| repo['git_url']}
+  whitelist = %w[
+    https://github.com/OpenGeoMetadata/big-ten.git
+  ]
+
+  desc 'Clone OpenGeoMetadata repositories'
+  task :clone, [:repo] do |_t, args|
+    if args.repo
+      ogm_repos = ["https://github.com/OpenGeoMetadata/#{args.repo}.git"]
+    else
+      ogm_api_uri = URI('https://api.github.com/orgs/opengeometadata/repos')
+      ogm_repos = JSON.parse(Net::HTTP.get(ogm_api_uri)).map do |repo|
+        repo['clone_url'] if repo['size'] > 0
+      end.compact
+      ogm_repos.select! { |repo| whitelist.include?(repo) || repo =~ /(edu|org|uk)\..*\.git$/ }
+    end
     ogm_repos.each do |repo|
-      if repo =~ /^git:\/\/github.com\/OpenGeoMetadata\/(edu|org|uk)\..*/
-        system "mkdir -p #{ogm_path} && cd #{ogm_path} && git clone --depth 1 #{repo}"
-      end
+      system "echo #{repo} && mkdir -p #{ogm_path} && cd #{ogm_path} && git clone --depth 1 #{repo}"
     end
   end
+
   desc '"git pull" OpenGeoMetadata repositories'
-  task :pull do
-    Dir.glob("#{ogm_path}/*").map{ |dir| system "cd #{dir} && git pull origin master" if dir =~ /.*(edu|org|uk)\..*./ }
+  task :pull, [:repo] do |_t, args|
+    paths = if args.repo
+              [File.join(ogm_path, args.repo)]
+            else
+              Dir.glob("#{ogm_path}/*")
+            end
+    paths.each do |path|
+      next unless File.directory?(path)
+      system "echo #{path} && cd #{path} && git pull origin"
+    end
   end
-  desc 'Index all of the GeoBlacklight documents'
+
+  desc 'Index all of the GeoBlacklight JSON documents'
   task :index do
-    solr = RSolr.connect :url => solr_url
+    puts "Indexing #{ogm_path} into #{solr_url}"
+    solr = RSolr.connect url: solr_url, adapter: :net_http_persistent
     Find.find(ogm_path) do |path|
-      next unless path =~ /.*geoblacklight.json$/
+      next unless File.basename(path) == 'geoblacklight.json'
       doc = JSON.parse(File.read(path))
-      begin
-        solr.update params: { commitWithin: 500, overwrite: true },
-                    data: [doc].to_json,
-                    headers: { 'Content-Type' => 'application/json' }
-
-      rescue RSolr::Error::Http => error
-        puts error
+      [doc].flatten.each do |record|
+        begin
+          puts "Indexing #{record['layer_slug_s']}: #{path}" if $DEBUG
+          solr.update params: { commitWithin: commit_within, overwrite: true },
+                      data: [record].to_json,
+                      headers: { 'Content-Type' => 'application/json' }
+        rescue RSolr::Error::Http => error
+          puts error
+        end
       end
     end
+    solr.commit
+  end
+
+  namespace :geoblacklight_harvester do
+    desc 'Harvest documents from a configured GeoBlacklight instance'
+    task :index, [:site] => [:environment] do |_t, args|
+      raise ArgumentError, 'A site argument is required' unless args.site
+
+      GeoCombine::GeoBlacklightHarvester.new(args.site.to_sym).index
+    end
   end
 end

data/lib/xslt/fgdc2html.xsl

@@ -1,12 +1,15 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
-  <!-- 
-     fgdc2html.xsl - Transformation from CDSDGM/FGDC into HTML 
+<xsl:output method="html" encoding="utf-8" indent="yes" />
+<!--
+     fgdc2html.xsl - Transformation from CSDGM/FGDC into HTML
      Created by Kim Durante, Stanford University Libraries
 
-     -->
+     Modified by Keith Jenkins, Cornell University Library, 2018-01-25
+     to render attributes in a more readable form.
+-->
   <xsl:template match="/">
-    <xsl:text disable-output-escaping='yes'>&lt;!DOCTYPE html></xsl:text>
+    <xsl:text disable-output-escaping='yes'>&lt;!DOCTYPE html&gt;</xsl:text>
     <html>
       <head>
         <title>
@@ -64,6 +67,7 @@
       </body>
     </html>
   </xsl:template>
+
   <!-- Identification -->
   <xsl:template match="idinfo">
     <div id="fgdc-identification-info">
@@ -335,6 +339,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Data Quality -->
   <xsl:template match="dataqual">
     <div id="fgdc-data-quality-info">
@@ -524,6 +529,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Spatial Data Organization -->
   <xsl:template match="spdoinfo">
     <div id="fgdc-spatialdataorganization-info">
@@ -637,6 +643,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Spatial Reference -->
   <xsl:template match="spref">
     <div id="fgdc-spatialreference-info">
@@ -1112,6 +1119,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Entity and Attribute -->
   <xsl:template match="eainfo">
     <div id="fgdc-entityattribute-info">
@@ -1145,7 +1153,7 @@
                   </dl>
                 </dd>
               </xsl:for-each>
-              <xsl:apply-templates select="attr" />
+              <xsl:call-template name='attributes' />
             </xsl:for-each>
             <xsl:for-each select="overview">
               <xsl:for-each select="eaover">
@@ -1166,6 +1174,94 @@
       </dl>
     </div>
   </xsl:template>
+
+  <xsl:template name='attributes'>
+    <xsl:if test='attr'>
+      <dt>Attributes</dt>
+      <dd>
+        <dl>
+          <xsl:for-each select="attr">
+            <dt><xsl:value-of select="attrlabl" /></dt>
+            <dd>
+              <xsl:value-of select="attrdef" />
+              <xsl:apply-templates select="attrdomv" />
+              <dl>
+                <xsl:for-each select="begdatea">
+                  <dt>Beginning Date of Attribute Values</dt>
+                  <dd>
+                    <xsl:value-of select="." />
+                  </dd>
+                </xsl:for-each>
+                <xsl:for-each select="enddatea">
+                  <dt>Ending Date of Attribute Values</dt>
+                  <dd>
+                    <xsl:value-of select="." />
+                  </dd>
+                </xsl:for-each>
+                <xsl:for-each select="attrvai">
+                  <xsl:for-each select="attrva">
+                    <dt>Attribute Value Accuracy</dt>
+                    <dd>
+                      <xsl:value-of select="." />
+                    </dd>
+                  </xsl:for-each>
+                  <xsl:for-each select="attrvae">
+                    <dt>Attribute Value Accuracy Explanation</dt>
+                    <dd>
+                      <xsl:value-of select="." />
+                    </dd>
+                  </xsl:for-each>
+                </xsl:for-each>
+                <xsl:for-each select="attrmfrq">
+                  <dt>Attribute Measurement Frequency</dt>
+                  <dd>
+                    <xsl:value-of select="." />
+                  </dd>
+                </xsl:for-each>
+              </dl>
+            </dd>
+          </xsl:for-each>
+        </dl>
+      </dd>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="attrdomv[codesetd]">
+    <xsl:text> (</xsl:text>
+    <xsl:value-of select="codesetd/codesetn" />
+    <xsl:apply-templates select="codesetd/codesets/text()" />
+    <xsl:text>)</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="attrdomv[edom]">
+    <br />
+    <button onclick="this.nextElementSibling.style.display = (this.nextElementSibling.style.display==='none') ? '' : 'none';">show/hide coded values</button>
+    <dl style="display:none">
+      <xsl:for-each select="edom">
+        <dt><xsl:value-of select="edomv" /></dt>
+        <dd><xsl:value-of select="edomvd" /></dd>
+      </xsl:for-each>
+    </dl>
+  </xsl:template>
+
+  <xsl:template match="attrdomv[rdom]">
+    <xsl:text> (</xsl:text>
+    <xsl:value-of select="rdom/rdommin" />
+    <xsl:text> to </xsl:text>
+    <xsl:value-of select="rdom/rdommax" />
+    <xsl:if test="rdom/attrunit">
+      <xsl:text> </xsl:text>
+      <xsl:value-of select="rdom/attrunit" />
+    </xsl:if>
+    <xsl:text>)</xsl:text>
+  </xsl:template>
+
+  <xsl:template match="attrdomv[udom]">
+    <xsl:text> (</xsl:text>
+    <xsl:value-of select="udom" />
+    <xsl:text>)</xsl:text>
+  </xsl:template>
+
   <!-- Distribution -->
   <xsl:template match="distinfo">
     <div id="fgdc-distribution-info">
@@ -1202,6 +1298,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Metadata -->
   <xsl:template match="metainfo">
     <div id="fgdc-metadata-reference-info">
@@ -1265,6 +1362,7 @@
       </dl>
     </div>
   </xsl:template>
+
   <!-- Citation -->
   <xsl:template match="citeinfo">
     <dl>
@@ -1362,6 +1460,7 @@
       </xsl:for-each>
     </dl>
   </xsl:template>
+
   <!-- Contact -->
   <xsl:template match="cntinfo">
     <dt>Contact Information</dt>
@@ -1478,6 +1577,7 @@
       </dl>
     </dd>
   </xsl:template>
+
   <!-- Time Period Info -->
   <xsl:template match="timeinfo">
     <dt>Time Period Information</dt>
@@ -1771,156 +1871,4 @@
       <xsl:value-of select="." />
     </dd>
   </xsl:template>
-  <xsl:template match="attr">
-    <dt>Attribute</dt>
-    <dd>
-      <dl>
-        <xsl:for-each select="attrlabl">
-          <dt>Attribute Label</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="attrdef">
-          <dt>Attribute Definition</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="attrdefs">
-          <dt>Attribute Definition Source</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="attrdomv">
-          <dt>Attribute Domain Values</dt>
-          <dd>
-            <dl>
-              <xsl:for-each select="edom">
-                <dt>Enumerated Domain</dt>
-                <dd>
-                  <dl>
-                    <xsl:for-each select="edomv">
-                      <dt>Enumerated Domain Value</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="edomvd">
-                      <dt>Enumerated Domain Value Definition</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="edomvds">
-                      <dt>Enumerated Domain Value Definition Source</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:apply-templates select="attr" />
-                  </dl>
-                </dd>
-              </xsl:for-each>
-              <xsl:for-each select="rdom">
-                <dt>Range Domain</dt>
-                <dd>
-                  <dl>
-                    <xsl:for-each select="rdommin">
-                      <dt>Range Domain Minimum</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="rdommax">
-                      <dt>Range Domain Maximum</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="attrunit">
-                      <dt>Attribute Units of Measure</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="attrmres">
-                      <dt>Attribute Measurement Resolution</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:apply-templates select="attr" />
-                  </dl>
-                </dd>
-              </xsl:for-each>
-              <xsl:for-each select="codesetd">
-                <dt>Codeset Domain</dt>
-                <dd>
-                  <dl>
-                    <xsl:for-each select="codesetn">
-                      <dt>Codeset Name</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                    <xsl:for-each select="codesets">
-                      <dt>Codeset Source</dt>
-                      <dd>
-                        <xsl:value-of select="." />
-                      </dd>
-                    </xsl:for-each>
-                  </dl>
-                </dd>
-              </xsl:for-each>
-              <xsl:for-each select="udom">
-                <dt>Unrepresentable Domain</dt>
-                <dd>
-                  <xsl:value-of select="." />
-                </dd>
-              </xsl:for-each>
-            </dl>
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="begdatea">
-          <dt>Beginning Date of Attribute Values</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="enddatea">
-          <dt>Ending Date of Attribute Values</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="attrvai">
-          <dt>Attribute Value Accuracy Information</dt>
-          <dd>
-            <dl>
-              <xsl:for-each select="attrva">
-                <dt>Attribute Value Accuracy</dt>
-                <dd>
-                  <xsl:value-of select="." />
-                </dd>
-              </xsl:for-each>
-              <xsl:for-each select="attrvae">
-                <dt>Attribute Value Accuracy Explanation</dt>
-                <dd>
-                  <xsl:value-of select="." />
-                </dd>
-              </xsl:for-each>
-            </dl>
-          </dd>
-        </xsl:for-each>
-        <xsl:for-each select="attrmfrq">
-          <dt>Attribute Measurement Frequency</dt>
-          <dd>
-            <xsl:value-of select="." />
-          </dd>
-        </xsl:for-each>
-      </dl>
-    </dd>
-  </xsl:template>
 </xsl:stylesheet>