geohydra 0.3

data/lib/geohydra/ogpcleanup.xsl

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- 
+     ogpcleanup.xsl - Fix FgdcText so that XML is properly escaped
+     
+     Copyright 2013, Stanford University Libraries.
+     
+     Created by Darren Hardy.
+     -->
+<xsl:stylesheet xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://lucene.apache.org/solr/4/document" xmlns:xi="http://www.w3.org/2001/XInclude" xmlns:solr="http://lucene.apache.org/solr/4/document" version="1.0" exclude-result-prefixes="xi xsl rdf solr">
+  <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>
+  <xsl:strip-space elements="*"/>
+  <xsl:template match="/">
+    <add>
+      <doc>
+        <xsl:for-each select="/solr:add/solr:doc/solr:field[not(@name='FgdcText')]">
+          <xsl:copy-of select="."/>
+        </xsl:for-each>
+        <xsl:for-each select="/solr:add/solr:doc/solr:field[(@name='FgdcText')]">
+          <field name="FgdcText">
+            <xsl:text disable-output-escaping="yes">&lt;![CDATA[</xsl:text>
+            <xsl:copy-of select="*"/>
+            <xsl:text disable-output-escaping="yes">]]&gt;</xsl:text>
+          </field>
+        </xsl:for-each>
+      </doc>
+    </add>
+  </xsl:template>
+</xsl:stylesheet>

data/lib/geohydra/solr.rb

@@ -0,0 +1,49 @@
+# encoding: UTF-8
+
+require 'dor-services'
+require 'rsolr'
+
+module GeoHydra
+  # API for uploading Solr documents
+  class Solr
+
+    # @param [String] url to solr server
+    def initialize(url)
+      @solr = url.nil? ? nil : RSolr.connect(:url => url)
+      ap @solr
+      @geo_metadata = []
+    end
+
+    def add(xml)
+      if xml.is_a? Dor::GeoMetadataDS
+        @geo_metadata << xml
+      else
+        @geo_metadata << Dor::GeoMetadataDS.from_xml(xml)
+      end
+    end
+
+    def upload(optimize = false)
+      @geo_metadata.each do |ds|
+        unless ds.nil?
+          puts "Uploading #{ds.title}"
+          @solr.add ds.to_solr
+        end
+      end
+      commit
+      @solr.optimize if optimize
+    end
+
+    def reset
+      @geo_metadata = []
+    end
+
+    def commit
+      @solr.commit
+    end
+
+    def delete_all
+      @solr.delete_by_query '*:*'
+      commit
+    end
+  end
+end

data/lib/geohydra/transform.rb

@@ -0,0 +1,175 @@
+require 'base64'
+require 'rgeo'
+require 'rgeo/shapefile'
+
+module GeoHydra
+  # Facilitates XSLT stylesheet transformations for ISO 19139 import/export
+  class Transform
+    # XXX hardcoded paths
+    def self.search_for_xsl(filename)
+      path = %w{
+        lib
+        lib/geomdtk
+        /usr/share/tomcat6/webapps/geonetwork/xsl/conversion/import
+        /var/geonetwork/2.8.0/lib
+        /opt/staging/s_gis_services
+        }
+      path.unshift(File.dirname(__FILE__))
+      path.each do |d|
+        fn = File.join(d, filename)
+        if File.exist?(fn)
+          return fn
+        end
+      end
+      nil
+    end
+    
+    # XSLT file locations
+    XSLT = {
+      :arcgis     => self.search_for_xsl('ArcGIS2ISO19139.xsl'),
+      :arcgis_fc  => self.search_for_xsl('arcgis_to_iso19139_fc.xsl')
+    }
+    
+    # XSLT processor
+    XSLTPROC = 'xsltproc --novalid --xinclude'
+    # XML cleaner
+    XMLLINT = 'xmllint --format --xinclude --nsclean'
+    
+    # Converts a ISO 19139 into MODS v3
+    # @param [String] fn with data as ISO 19139 XML.
+    # @return [Nokogiri::XML::Document] the MODS v3 metadata
+    # @deprecated Use to_geoMetadataDS instead
+    def self.to_mods fn
+      doc = Dor::GeoMetadataDS.from_xml File.open(fn)
+      doc.to_mods
+    end
+    
+    # Converts an ESRI ArcCatalog metadata.xml into ISO 19139
+    # @param [String] fn Input file
+    # @param [String] ofn Output file
+    # @param [String] ofn_fc Output file for the Feature Catalog (optional)
+    def self.from_arcgis fn, ofn, ofn_fc = nil
+      system("#{XSLTPROC} #{XSLT[:arcgis]} '#{fn}' | #{XMLLINT} -o '#{ofn}' -")
+      unless ofn_fc.nil?
+        system("#{XSLTPROC} #{XSLT[:arcgis_fc]} '#{fn}' | #{XMLLINT} -o '#{ofn_fc}' -")
+      end
+    end
+    
+    # @return [Hash]
+    # @deprecated
+    def self.to_solr fn
+      doc = Dor::GeoMetadataDS.from_xml File.read(fn)
+      doc.to_solr
+    end
+    
+    # Extracts an inline thumbnail from the ESRI ArcCatalog metadata format
+    # @param [String] fn the metadata
+    # @param [String] thumbnail_fn the file into which to write JPEG image
+    # @param [String] property_type is the EsriPropertyType to select
+    # @raise [ArgumentError] if cannot find a thumbnail
+    def self.extract_thumbnail fn, thumbnail_fn, property_type = 'PictureX'
+      doc = Nokogiri::XML(File.read(fn))
+      doc.xpath('//Binary/Thumbnail/Data').each do |node|
+        if property_type.nil? or node['EsriPropertyType'] == property_type
+          image = Base64.decode64(node.text)
+          File.open(thumbnail_fn, 'wb') {|f| f << image }
+          return
+        end
+      end
+      raise ArgumentError, "No thumbnail embedded within #{fn}"
+    end
+    
+    # Converts a ISO 19139 into RDF-bundled document geoMetadataDS
+    # @param [Nokogiri::XML::Document] isoXml ISO 19193 MD_Metadata node
+    # @param [Nokogiri::XML::Document] fcXml ISO 19193 feature catalog
+    # @param [String] flags flags['purl']
+    # @return [Nokogiri::XML::Document] the geoMetadataDS with RDF
+    def self.to_geoMetadataDS isoXml, fcXml, flags = {}
+      raise ArgumentError, "PURL is required" if flags['purl'].nil?
+      raise ArgumentError, "ISO 19139 is required" if isoXml.nil? or isoXml.root.nil?
+      Nokogiri::XML("
+<rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">
+  <rdf:Description rdf:about=\"#{flags['purl']}\">
+    #{isoXml.root.to_s}
+  </rdf:Description>
+  <rdf:Description rdf:about=\"#{flags['purl']}\">
+    #{fcXml.nil?? '' : fcXml.root.to_s}
+  </rdf:Description>
+</rdf:RDF>")
+    end
+
+    # @param zipfn [String] ZIP file
+    def self.reproject druid, zipfn, flags
+
+      puts "Extracting #{druid.id} data from #{zipfn}" if flags[:verbose]
+      tmp = "#{flags[:tmpdir]}/#{druid.id}"
+      FileUtils.rm_rf tmp if File.directory? tmp
+      FileUtils.mkdir_p tmp
+      system("unzip -q -j '#{zipfn}' -d '#{tmp}'")
+      
+      shpfn = nil
+      Dir.glob("#{tmp}/**/*.shp") do |fn|
+        shpfn = File.basename(fn)
+      end
+      
+      [4326].each do |srid|
+        ifn = File.join(tmp, shpfn)
+        raise ArgumentError, "#{ifn} is missing" unless File.exist? ifn
+        
+        odr = File.join(flags[:tmpdir], 'EPSG_' + srid.to_s)
+        ofn = File.join(odr, shpfn)
+        puts "Projecting #{ifn} -> #{odr}/#{ofn}" if flags[:verbose]
+
+        # reproject
+        FileUtils.mkdir_p odr unless File.directory? odr
+        system("ogr2ogr -progress -t_srs '#{flags[:wkt][srid.to_s]}' '#{ofn}' '#{ifn}'") 
+
+        # normalize prj file
+        if flags[:overwrite_prj] and not flags[:wkt][srid.to_s].nil?
+          prj_fn = ofn.gsub(%r{\.shp}, '.prj')
+          puts "Overwriting #{prj_fn}" if flags[:verbose]
+          File.open(prj_fn, 'w') {|f| f.write(flags[:wkt][srid.to_s])}
+        end
+
+        # package up reprojection
+         ozip = File.join(druid.content_dir, "data_EPSG_#{srid}.zip")
+        puts "Repacking #{ozip}" if flags[:verbose]
+        system("zip -q -Dj '#{ozip}' #{odr}/#{File.basename(shpfn, '.shp')}.*")
+
+        # cleanup
+        FileUtils.rm_rf odr
+      end
+
+      # cleanup
+      FileUtils.rm_rf tmp
+    end
+    
+    # Reads the shapefile to determine geometry type
+    #
+    # @return [String] Point, Polygon, LineString as appropriate
+    def self.geometry_type(shp_filename)
+      begin
+        RGeo::Shapefile::Reader.open(shp_filename) do |shp|
+          shp.each do |record|
+            return record.geometry.geometry_type.to_s.gsub(/^Multi/,'')
+          end
+        end
+      rescue RGeo::Error::RGeoError => e
+        puts e.message
+      end 
+      nil     
+    end
+    
+    private
+    def self.do_xslt xslt, fn, params = {}
+      cmd = XSLTPROC
+      params.each do |k,v|
+        cmd += " --stringparam '#{k}' '#{v}'"
+      end
+      # ap({:cmd => cmd, :xslt => xslt})
+      IO::popen("#{cmd} #{xslt} #{fn} | #{XMLLINT} -", 'r') do |f|
+        return Nokogiri::XML(f.read)
+      end      
+    end
+  end
+end

data/lib/geohydra/utils.rb

@@ -0,0 +1,19 @@
+require 'druid-tools'
+
+module GeoHydra
+  class Utils
+    def self.find_druid_folders(dir = '.')
+      # requires extra * for symlinks to directories
+      Dir.glob(File.join(dir, '*', '**', DruidTools::Druid.glob + '/')).sort.collect do |p|
+        yield p if block_given?
+        p
+      end
+    end
+    
+    # @see http://www.esri.com/library/whitepapers/pdfs/shapefile.pdf pg2
+    def self.shapefile?(fn)
+      File.basename(fn).downcase =~ /^([a-z0-9_-]+)\.shp$/
+      $1.present?
+    end
+  end
+end

data/lib/geohydra/version.rb

@@ -0,0 +1,3 @@
+module GeoHydra
+  VERSION = File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).strip
+end

data/scripts/correct_collections.rb

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require 'optparse'
+require 'druid-tools'
+
+def doit(druid, flags)
+  puts "Processing #{druid.id}"
+  
+  begin
+    item = Dor::Item.find(druid.druid)
+    ap({:item => item, :collections => item.collections}) if flags[:debug]
+    
+    # remove all collections
+    item.collections.dup.each {|c| item.remove_collection(c)}
+    
+    # add the new ones
+    flags[:collections].each do |k, collection|
+      item.add_collection(collection)
+    end
+    
+    ap({:item => item, :collections => item.collections}) if flags[:debug]
+    unless item.allows_modification?
+      puts "WARNING: Item not editable: #{druid.id}" 
+      item.open_new_version
+    end
+    item.save
+    item.close_version
+  rescue ActiveFedora::ObjectNotFoundError => e
+    puts "ERROR: #{e.message}"
+  end
+end
+
+# __MAIN__
+begin
+  File.umask(002)
+  flags = {
+    :tmpdir => 'tmp',
+    :verbose => false,
+    :debug => false,
+    :collections => {},
+    :workspacedir => '/var/geomdtk/current/workspace'
+  }
+
+  OptionParser.new do |opts|
+    opts.banner = <<EOM
+Usage: #{File.basename(__FILE__)} [options] [druid [druid...] | < druids]
+EOM
+    opts.on('--collection DRUID', 'Collection for accession') do |druid|
+      flags[:collections][DruidTools::Druid.new(druid).id] = nil
+    end
+    opts.on('--tmpdir DIR', "Temporary directory for assembly (default: #{flags[:tmpdir]})") do |d|
+      flags[:tmpdir] = d
+    end
+    opts.on('-v', '--verbose', 'Run verbosely, use multiple times for debug level output') do
+      flags[:debug] = true if flags[:verbose]  # -vv
+      flags[:verbose] = true
+    end
+    opts.on('--workspace DIR', "Workspace directory for assembly (default: #{flags[:workspacedir]})") do |d|
+      flags[:workspacedir] = d
+    end
+  end.parse!
+
+  [flags[:tmpdir], flags[:workspacedir]].each do |d|
+    raise ArgumentError, "Missing directory #{d}" unless File.directory? d
+  end
+
+  ap({:flags => flags}) if flags[:debug]
+  
+  # Validate collection druids
+  flags[:collections].each do |druid,v|
+    begin
+      flags[:collections][druid] = Dor::Collection.find(druid)
+    rescue ActiveFedora::ObjectNotFoundError => e
+      puts "ERROR: Invalid collection #{druid}: #{e.message}"
+      exit(-1)
+    end
+  end
+
+  (ARGV.empty?? STDIN : ARGV).each do |pid|
+    druid = DruidTools::Druid.new(pid.strip, flags[:workspacedir])
+    ap({:druid => druid}) if flags[:debug]
+    begin
+      doit(druid, flags)
+    rescue Exception => e
+      ap({:error => e})
+    end
+  end
+rescue SystemCallError => e
+  $stderr.puts "ERROR: #{e.message}"
+  $stderr.puts e.backtrace
+  exit(-1)
+end

data/scripts/dbfdump.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require 'dbf'
+require 'awesome_print'
+
+ARGV.each do |fn|
+  f = DBF::Table.new(fn)
+  f.each do |r|
+    puts r.class
+    r.keys.each do |k|
+      ap k
+    end
+  end
+end

data/scripts/extract.xsl

@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<xsl:stylesheet xmlns:gco="http://www.isotc211.org/2005/gco" xmlns:gmd="http://www.isotc211.org/2005/gmd" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="xml" version="1.0" encoding="UTF-8" indent="yes"/>
+  <xsl:template match="/">
+    <xsl:text>{</xsl:text>
+    <xsl:choose>
+      <xsl:when test="//gmd:CI_OnlineResource">
+        <xsl:apply-templates select="//gmd:CI_OnlineResource"/>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:text>"druid" : "</xsl:text>
+        <xsl:variable name="druid"/>
+        <xsl:value-of select="$druid"/>
+        <xsl:text>", </xsl:text>
+      </xsl:otherwise>
+    </xsl:choose>
+    <xsl:apply-templates select="//gmd:RS_Identifier"/>
+    <xsl:text>"version" : 1 }</xsl:text>
+  </xsl:template>
+  <xsl:template match="//gmd:CI_OnlineResource">
+    <xsl:text>"druid" : "</xsl:text>
+    <xsl:value-of select="substring-after(.//gmd:URL/text(), 'http://purl.stanford.edu/')"/>
+    <xsl:text>", "filename" : "</xsl:text>
+    <xsl:value-of select="substring-before(.//gmd:name/gco:CharacterString/text(), '.shp')"/>
+    <xsl:text>",</xsl:text>
+  </xsl:template>
+  <xsl:template match="//gmd:RS_Identifier">
+    <xsl:text>"crs" : "</xsl:text>
+    <xsl:copy-of select=".//gmd:codeSpace/gco:CharacterString/text()"/>
+    <xsl:text>:</xsl:text>
+    <xsl:copy-of select=".//gmd:code/gco:CharacterString/text()"/>
+    <xsl:text>",
+</xsl:text>
+  </xsl:template>
+</xsl:stylesheet>

data/scripts/geoserver_createdb.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -x
+export PATH="/usr/pgsql-9.2/bin:$PATH"
+
+createdb -e geoserver
+createuser -e -W --encrypted --replication geostaff
+createuser -e -W --encrypted georead

data/scripts/geoserver_createdb.sql

@@ -0,0 +1,29 @@
+CREATE EXTENSION plpgsql
+ SCHEMA pg_catalog
+ VERSION "1.0";
+
+CREATE EXTENSION postgis
+ SCHEMA public
+ VERSION "2.0.1";
+
+CREATE SCHEMA druid
+  AUTHORIZATION geostaff;
+
+GRANT ALL ON SCHEMA druid TO geostaff;
+GRANT USAGE ON SCHEMA druid TO georead;
+
+SET search_path TO public;
+
+CREATE TABLE registered_layers
+(
+  druid character varying(11) NOT NULL, -- aa111aa1111
+  layername character varying NOT NULL,
+  title character varying NOT NULL,
+  CONSTRAINT pk_registered_layers PRIMARY KEY (druid)
+)
+WITH (
+  OIDS=FALSE
+);
+ALTER TABLE registered_layers
+  OWNER TO geostaff;
+COMMENT ON COLUMN registered_layers.druid IS 'aa111aa1111';