geohydra 0.3

data/Rakefile

@@ -0,0 +1,34 @@
+require 'rubygems'
+require 'rspec/core/rake_task'
+require 'bundler'
+require 'yard'
+require 'version_bumper'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+desc "Run console; defaults to IRB='pry'"
+task :console, :IRB do |t, args|
+  irb = args[:IRB].nil?? 'pry' : args[:IRB]
+  sh irb, "-r", "#{File.dirname(__FILE__)}/config/boot.rb"
+end
+
+desc "Run tests"
+task :spec do
+  RSpec::Core::RakeTask.new
+end
+
+desc "Build documentation"
+task :yard do
+  YARD::Rake::YardocTask.new do |t|
+    t.files = ['lib/**/*.rb', 'bin/**/*.rb']
+  end
+end
+
+desc "Runs 'rake spec yard'"
+task :default => [:spec, :yard]

data/VERSION

@@ -0,0 +1 @@
+0.3

data/bin/accession.rb

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require 'optparse'
+require 'druid-tools'
+
+VERSION = '0.3'
+
+# __MAIN__
+begin
+  File.umask(002)
+  flags = {
+    :admin_policy => 'druid:gz830zp4734',
+    :rights => 'stanford',
+    :tags => [
+      "Registered By : #{%x{whoami}.strip} (GeoHydra)"
+    ],
+    :tmpdir => GeoHydra::Config.geohydra.tmpdir || 'tmp',
+    :verbose => false,
+    :configtest => false,
+    :purge => false,
+    :accessionWF => false,
+    :contentMetadata => true,
+    :debug => false,
+    :shelve => false,
+    :workspacedir => GeoHydra::Config.geohydra.workspace || 'workspace'
+  }
+
+  OptionParser.new do |opts|
+    opts.version = VERSION
+    opts.banner = <<EOM
+Usage: #{File.basename(__FILE__)} [options] [druid [druid...]]
+       #{File.basename(__FILE__)} [options] < druids
+EOM
+    opts.on('--apo DRUID', 'APO for collection to accession' + (flags[:admin_policy] ? " (default: #{flags[:admin_policy]})" : '')) do |druid|
+      flags[:admin_policy] = DruidTools::Druid.new(druid).druid
+    end
+    opts.on('--collection DRUID', 'Collection for accession' + (flags[:collection] ? " (default: #{flags[:collection]})" : '')) do |druid|
+      flags[:collection] = DruidTools::Druid.new(druid).druid
+    end
+    opts.on('--purge', 'Purge items before accessioning') do |b|
+      flags[:purge] = true
+    end
+    opts.on('--accessionWF', 'Kick off the accessionWF workflow') do |b|
+      flags[:accessionWF] = true
+    end
+    opts.on('--rights KEYWORD', "Rights keyword (default: #{flags[:rights]})") do |keyword|
+      flags[:rights] = keyword
+    end
+    opts.on('--tag TAG', 'Tag for each object - multiple tags allowed') do |tag|
+      flags[:tags] << tag
+    end
+    opts.on('--test', 'Verify configuration then exit') do
+      flags[:configtest] = true
+    end
+    opts.on('--tmpdir DIR', "Temporary directory for assembly (default: #{flags[:tmpdir]})") do |d|
+      flags[:tmpdir] = d
+    end
+    opts.on('-v', '--verbose', 'Run verbosely, use multiple times for debug level output') do
+      flags[:debug] = true if flags[:verbose]  # -vv
+      flags[:verbose] = true
+    end
+    opts.on('--workspace DIR', "Workspace directory for assembly (default: #{flags[:workspacedir]})") do |d|
+      flags[:workspacedir] = d
+    end
+  end.parse!
+
+  [flags[:tmpdir], flags[:workspacedir]].each do |d|
+    raise ArgumentError, "Missing directory #{d}" unless File.directory? d
+  end
+
+  ap({:flags => flags}) if flags[:debug]
+
+  # Verify configuation
+  if flags[:configtest]
+    errs = 0
+    %w{workflow.url fedora.safeurl solrizer.url ssl.cert_file ssl.key_pass dor.service_root}.each do |k|
+      begin
+        k = "Dor::Config.#{k}"
+        if eval(k).nil?
+          $stderr.puts "ERROR: Configuration requires #{k}" 
+          errs += 1
+        end
+      rescue Exception => e
+        $stderr.puts e, e.backtrace
+      end
+    end
+    puts "Configuration OK" if errs == 0
+    exit errs
+  end
+
+  if ARGV.empty?
+    STDIN.each do |line|
+      pid = line.strip
+      druid = DruidTools::Druid.new(pid, flags[:workspacedir])
+      GeoHydra::Accession.run(druid, flags)
+    end
+  else
+    ARGV.each do |pid|
+      druid = DruidTools::Druid.new(pid, flags[:workspacedir])
+      GeoHydra::Accession.run(druid, flags)
+    end
+  end
+rescue SystemCallError => e
+  $stderr.puts "ERROR: #{e.message}"
+  $stderr.puts e.backtrace
+  exit(-1)
+end

data/bin/assemble.rb

@@ -0,0 +1,307 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require 'druid-tools'
+require 'optparse'
+require 'json'
+
+def setup_druid(obj, flags)
+  druid = DruidTools::Druid.new(obj.druid, flags[:workspacedir])
+  raise ArgumentError unless DruidTools::Druid.valid?(druid.druid)
+  %w{path content_dir metadata_dir temp_dir}.each do |k|
+    d = druid.send(k.to_sym)
+    unless File.directory? d
+      $stderr.puts "Creating directory #{d}" if flags[:verbose]
+      FileUtils.mkdir_p d
+    end
+  end
+  druid
+end
+
+def find_mef(druid, uuid, flags)
+  # export MEF -- the .iso19139.xml file is preferred
+  puts "Exporting MEF for #{uuid}" if flags[:verbose]
+  client.export(uuid, flags[:tmpdir])
+  system(['unzip', 
+          '-oq',
+          "'#{flags[:tmpdir]}/#{uuid}.mef'",
+          '-d', 
+          "'#{flags[:tmpdir]}'"].join(' '))
+  found_metadata = false
+  %w{metadata.iso19139.xml metadata.xml}.each do |fn| # priority order as per MEF
+    unless found_metadata
+      fn = File.join(flags[:tmpdir], uuid, 'metadata', fn)
+      next unless File.exist? fn
+
+      found_metadata = true
+      # original ISO 19139
+      ifn = File.join(druid.temp_dir, 'iso19139.xml')
+      FileUtils.link fn, ifn, :verbose => flags[:verbose], :force => true
+    end
+  end
+  ifn
+end
+
+def find_local(druid, xml, fcXml, flags)
+  ifn = File.join(druid.temp_dir, 'iso19139.xml')
+  unless File.exist?(ifn)
+    File.open(ifn, "w") {|f| f << xml.to_s}
+  end
+  
+  fcfn = File.join(druid.temp_dir, 'iso19139-fc.xml')
+  unless File.exist?(fcfn)
+    File.open(fcfn, "w") {|f| f << fcXml.to_s}
+  end
+  
+  [ifn, fcfn]
+end
+
+def convert_iso2geo(druid, ifn, isoXml, fcXml, flags)
+  isoXml = Nokogiri::XML(isoXml)
+  if isoXml.nil? or isoXml.root.nil?
+    raise ArgumentError, "Empty ISO 19139" 
+  end
+  fcXml = Nokogiri::XML(fcXml)
+  ap({:ifn => ifn, :isoXml => isoXml, :fcXml => fcXml, :flags => flags}) if flags[:debug]
+  # GeoMetadataDS
+  gfn = File.join(druid.metadata_dir, 'geoMetadata.xml')
+  puts "Generating #{gfn}" if flags[:verbose]
+  xml = GeoHydra::Transform.to_geoMetadataDS(isoXml, fcXml, { 'purl' => "#{flags[:purl]}/#{druid.id}"}) 
+  File.open(gfn, 'w') {|f| f << xml.to_xml(:indent => 2) }
+  gfn
+end
+
+def convert_geo2mods(druid, geoMetadata, flags)
+  # MODS from GeoMetadataDS
+  ap({:geoMetadata => geoMetadata.ng_xml}) if flags[:debug]
+  dfn = File.join(druid.metadata_dir, 'descMetadata.xml')
+  puts "Generating #{dfn}" if flags[:verbose]
+  File.open(dfn, 'w') { |f| f << geoMetadata.to_mods.to_xml(:index => 2) }
+  dfn
+end
+
+def convert_geo2solrspatial(druid, geoMetadata, flags)
+  # Solr document from GeoMetadataDS
+  sfn = File.join(druid.temp_dir, 'spatialSolr.xml')
+  puts "Generating #{sfn}" if flags[:verbose]
+  h = geoMetadata.to_solr_spatial
+  ap({:to_solr_spatial => h}) if flags[:debug]
+  
+  doc = Nokogiri::XML::Builder.new do |xml|
+    xml.add {
+      xml.doc_ {
+        h.each do |k, v|
+          v.each do |s|
+            xml.field s, :name => k
+          end unless v.nil?
+        end unless h.nil?
+      }
+    }
+  end
+  File.open(sfn, 'w') { |f| f << doc.to_xml(:indent => 2) }
+  sfn
+end
+
+def convert_mods2ogpsolr(druid, dfn, flags)
+  raise ArgumentError, 'Missing required :geoserver flag' if flags[:geoserver].nil?
+  raise ArgumentError, 'Missing required :purl flag' if flags[:purl].nil?
+
+  geoserver = flags[:geoserver]
+  purl = "#{flags[:purl]}/#{druid.id}"
+
+  # OGP Solr document from GeoMetadataDS
+  sfn = File.join(druid.temp_dir, 'ogpSolr.xml')
+  FileUtils.rm_f(sfn) if File.exist?(sfn)
+  cmd = ['xsltproc',
+          "--stringparam geoserver_root '#{geoserver}'",
+          "--stringparam purl '#{purl}'",
+          "--output '#{sfn}'",
+          "'#{File.expand_path(File.dirname(__FILE__) + '/../lib/geohydra/mods2ogp.xsl')}'",
+          "'#{dfn}'"
+          ].join(' ')
+  puts "Generating #{sfn} using #{cmd}" if flags[:verbose]
+  ap({:cmd => cmd}) if flags[:debug]
+  system(cmd)
+
+  sfn
+end
+
+def convert_geo2dc(geoMetadata, flags)
+  # Solr document from GeoMetadataDS
+  dcfn = File.join(druid.temp_dir, 'dc.xml')
+  puts "Generating #{dcfn}" if flags[:verbose]
+  File.open(dcfn, 'w') { |f| f << geoMetadata.to_dublin_core.to_xml(:indent => 2) }
+  dcfn
+end
+
+def export_images(druid, uuid, flags)
+  # export any thumbnail images
+  %w{png jpg}.each do |fmt|
+    Dir.glob("#{flags[:tmpdir]}/#{uuid}/{private,public}/*.#{fmt}") do |fn|
+      ext = '.' + fmt
+      tfn = File.basename(fn, ext)
+      # convert _s to _small as per GeoNetwork convention
+      tfn = tfn.gsub(/_s$/, '_small')
+      imagefn = File.join(druid.content_dir, tfn + ext)
+      FileUtils.link fn, imagefn, :verbose => flags[:debug], :force => true
+      yield imagefn if block_given?
+    end
+  end
+end
+
+def export_local_images(druid, tempdir, flags)
+  # export any thumbnail images
+  n = 0
+  %w{png jpg}.each do |fmt|
+    Dir.glob("#{flags[:stagedir]}/#{druid.id}/content/*.#{fmt}") do |fn|
+      imagefn = File.join(druid.content_dir, 'preview' + '.' + fmt)
+      FileUtils.link fn, imagefn, :verbose => flags[:debug], :force => true
+      yield imagefn if block_given?
+      n = n + 1
+    end
+  end
+  puts "WARNING: Multiple images for #{druid.id}" if n > 1
+end
+
+def export_attachments(druid, flags)
+  Dir.glob("#{flags[:stagedir]}/#{druid.id}/content/*") do |fn|
+    unless %w{.png .jpg .zip}.include?(File.extname(fn))
+      afn = File.join(druid.content_dir, File.basename(fn))
+      FileUtils.link fn, afn, :verbose => flags[:debug], :force => true
+      yield afn if block_given?
+    end
+  end
+end
+
+def export_zip(druid, flags)
+  # export content into zip files
+  Dir.glob(File.join(flags[:stagedir], "#{druid.id}/content/*.zip")) do |fn|
+    ofn = File.join(druid.content_dir, File.basename(fn))
+    FileUtils.link fn, ofn, :verbose => flags[:verbose], :force => true
+    yield ofn if block_given?
+  end
+end
+
+def doit(client, uuid, obj, flags)
+  puts "Processing #{obj.druid}"
+  druid = setup_druid(obj, flags)
+
+  if flags[:geonetwork]
+    puts "Processing #{uuid} from geonetwork" if flags[:debug]
+    ifn = find_mef(druid, uuid, flags)
+  else
+    raise ArgumentError, druid if obj.content.empty?
+    ifn, fcfn = find_local(druid, obj.content.to_s, obj.fc.to_s, flags)
+  end
+  
+  puts "Processing #{ifn}" if flags[:verbose]
+
+  optfn = File.expand_path("#{flags[:stagedir]}/#{obj.druid}/temp/geoOptions.json")
+  puts "Loading extra out-of-band options #{optfn}" if flags[:debug]
+  if File.exist?(optfn)
+    h = JSON.parse(File.read(optfn))
+    flags = flags.merge(h).symbolize_keys
+    ap({:optfn => optfn, :h => h, :flags => flags}) if flags[:debug]
+  else
+    puts "WARNING: #{obj.druid} is missing options .json parameters: #{optfn}"
+    flags[:geometryType] ||= 'Polygon' # XXX: placeholder
+  end
+
+  gfn = convert_iso2geo(druid, ifn, obj.content, obj.fc, flags)
+  geoMetadata = Dor::GeoMetadataDS.from_xml File.read(gfn)
+  geoMetadata.geometryType = flags[:geometryType] || 'Polygon'
+  geoMetadata.zipName = 'data.zip'
+  geoMetadata.purl = File.join(flags[:purl], druid.id)
+
+  dfn = convert_geo2mods(druid, geoMetadata, flags)
+  sfn = convert_geo2solrspatial(druid, geoMetadata, flags)
+  
+  ofn = convert_mods2ogpsolr(druid, dfn, flags)
+  
+  if flags[:geonetwork]
+    export_images(druid, uuid, flags)
+  else
+    export_local_images(druid, File.expand_path(File.dirname(obj.zipfn) + '/../temp'), flags)
+  end
+  
+  export_attachments(druid, flags)
+  export_zip(druid, flags)
+end
+
+def main(flags)
+  File.umask(002)
+  if flags[:geonetwork]
+    client = GeoHydra::GeoNetwork.new
+    client.each do |uuid|
+      begin
+        puts "Processing #{uuid}"
+        obj = client.fetch(uuid)
+        doit client, uuid, obj, flags
+      rescue Exception => e
+        $stderr.puts e
+        $stderr.puts e.backtrace if flags[:debug]
+      end
+    end
+  else
+    puts "Searching for staged content..." if flags[:verbose]
+    puts flags[:stagedir] + '/' + DruidTools::Druid.glob + '/content/data.zip' if flags[:debug]
+    Dir.glob(flags[:stagedir] + '/' + DruidTools::Druid.glob + '/content/data.zip') do |zipfn|
+      Dir.glob(File.join(File.dirname(zipfn), '..', 'temp', '*iso19139.xml')) do |xmlfn|
+        druid = File.basename(File.dirname(File.dirname(zipfn)))
+        obj = Struct.new(:content, :status, :druid, :zipfn, :fc).new(File.read(xmlfn), nil, druid, zipfn, File.read(xmlfn.gsub('.xml', '-fc.xml')))
+        ap({:zipfn => zipfn, :obj => obj}) if flags[:debug]
+        doit client, nil, obj, flags
+      end
+    end
+  end
+end
+
+# __MAIN__
+begin
+  flags = {
+    :debug => false,
+    :verbose => false,
+    :geonetwork => false,
+    :geoserver => GeoHydra::Config.ogp.geoserver,
+    :solr => GeoHydra::Config.ogp.solr,
+    :purl => GeoHydra::Config.ogp.purl,
+    :stagedir => GeoHydra::Config.geohydra.stage || 'stage',
+    :xinclude => false,
+    :workspacedir => GeoHydra::Config.geohydra.workspace || 'workspace',
+    :tmpdir => GeoHydra::Config.geohydra.tmpdir || 'tmp'
+  }
+
+  OptionParser.new do |opts|
+    opts.banner = <<EOM
+Usage: #{File.basename(__FILE__)} [options]
+EOM
+    opts.on('--geonetwork', 'Run against GeoNetwork server') do |v|
+      flags[:geonetwork] = true
+    end
+    opts.on('-v', '--verbose', 'Run verbosely') do |v|
+      flags[:debug] = true if flags[:verbose]
+      flags[:verbose] = true
+    end
+    opts.on('--stagedir DIR', "Staging directory with ZIP files (default: #{flags[:stagedir]})") do |v|
+      flags[:stagedir] = v
+    end
+    opts.on('--workspace DIR', "Workspace directory for assembly (default: #{flags[:workspacedir]})") do |v|
+      flags[:workspacedir] = v
+    end
+    opts.on('--tmpdir DIR', "Temporary directory for assembly (default: #{flags[:tmpdir]})") do |v|
+      flags[:tmpdir] = v
+    end
+  end.parse!
+
+  %w{tmpdir stagedir workspacedir}.each do |k|
+    d = flags[k.to_sym]
+    raise ArgumentError, "Missing directory #{d}" unless d.nil? or File.directory? d
+  end
+
+  ap({:flags => flags}) if flags[:debug]
+  raise NotImplementError, 'geonetwork code is stale' if flags[:geonetwork]
+  main flags
+rescue SystemCallError => e
+  $stderr.puts "ERROR: #{e.message}"
+  exit(-1)
+end