geohydra 0.3

data/bin/loader.rb

@@ -0,0 +1,296 @@
+#!/usr/bin/env ruby
+# -*- encoding : utf-8 -*-
+#
+# RGeoServer Batch load layers (batch_demo.rb)
+# Usage: #{File.basename(__FILE__)} [input.yml]
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require 'optparse'
+require 'uri'
+require 'yaml'
+
+require 'druid-tools'
+require 'mods'
+
+ENV['RGEOSERVER_CONFIG'] ||= ENV_FILE + '_rgeoserver.yml'
+require 'rgeoserver'
+
+def do_vector(catalog, layername, format, ws, ds, v, flags)
+  # Create data stores for shapefiles
+  ds = RGeoServer::DataStore.new catalog, :workspace => ws, :name => (ds.nil?? v['druid'].id : ds)
+  puts "DataStore: #{ws.name}/#{ds.name} (#{v['remote']})" if flags[:verbose]
+  ap({:vector => v}) if flags[:debug]
+  case v['remote']
+  when 'serverfile'
+    puts "DataStore: Loading server-side file #{v['filename']}" if flags[:verbose]
+    ds.upload_external v['filename']
+  when 'localfile'
+    puts "DataStore: Uploading local file #{v['filename']}" if flags[:verbose]
+    ds.upload_file v['filename']
+  when 'postgis-db'
+    puts "DataStore: Connecting to PostGIS database #{flags[:host]}:#{flags[:port]}/#{flags[:database]}" if flags[:verbose]
+    ds.data_type = 'PostGIS'
+    ds.connection_parameters = ds.connection_parameters.merge({
+      "Connection timeout" => 20,
+      "port" => flags[:port],
+      "dbtype" => 'postgis',
+      "host" => flags[:host],
+      "validate connections" => true,
+      "encode functions" => false,
+      "max connections" => 16,
+      "database" => flags[:database],
+      "namespace" => flags[:namespace],
+      "schema" => flags[:schema],
+      "Loose bbox" => true,
+      "Expose primary keys" => false,
+      "fetch size" => 1000,
+      "Max open prepared statements" => 50,
+      "preparedStatements" => false,
+      "Estimated extends" => true,
+      "user" => flags[:user],
+      "min connections" => 4
+    })
+    ap({:connection_parameters => ds.connection_parameters}) if flags[:debug]
+  else
+    raise NotImplemented, "Unknown remote type: #{v['remote']}"
+  end
+  
+  if v['remote'] =~ /file$/
+    ds.connection_parameters = ds.connection_parameters.merge({
+      "namespace" => flags[:namespace],
+      "charset" => 'UTF-8',
+      "create spatial index" => 'true',
+      "cache and reuse memory maps" => 'true',
+      "enable spatial index" => 'true',
+      "filetype" => 'shapefile',
+      "memory mapped buffer" => 'false'
+    })
+    ap({:connection_parameters => ds.connection_parameters}) if flags[:debug]
+  end
+
+  # modify DataStore with rest of parameters
+  ds.enabled = 'true'
+  ds.description = v['description']
+  ds.save# unless v['remote'] =~ /-db$/
+  
+  ft = RGeoServer::FeatureType.new catalog, 
+    :workspace => ws, 
+    :data_store => ds, 
+    :name => layername 
+  puts "WARNING: FeatureType doesn't already exists #{ft}" if ft.new?
+  puts "FeatureType: #{ws.name}/#{ds.name}/#{ft.name}" if flags[:verbose]
+  ft.enabled = 'true'
+  ft.title = v['title'] 
+  ft.description = v['description']
+  ft.keywords = v['keywords']
+  ft.metadata_links = v['metadata_links']
+  puts(ft.message) if flags[:debug]
+  ft.save
+end
+
+def do_raster(catalog, layername, format, ws, v, flags)
+  # Create of a coverage store
+  puts "CoverageStore: #{ws.name}/#{layername} (#{format})" if flags[:verbose]
+  cs = RGeoServer::CoverageStore.new catalog, :workspace => ws, :name => layername
+  if v['filename'] =~ %r{^/}
+    cs.url = "file://" + v['filename']
+  else
+    cs.url = "file://" + File.join(flags[:datadir], v['filename'])
+  end
+  cs.description = v['description'] 
+  cs.enabled = 'true'
+  cs.data_type = format
+  cs.save
+
+  # Now create the actual coverage
+  puts "Coverage: #{ws.name}/#{cs.name}/#{layername}" if flags[:verbose]
+  cv = RGeoServer::Coverage.new catalog, :workspace => ws, :coverage_store => cs, :name => layername 
+  cv.enabled = 'true'
+  cv.title = v['title'] 
+  cv.keywords = v['keywords']
+  cv.metadata_links = v['metadata_links']
+  cv.save
+end
+
+#=  Input data. *See DATA section at end of file*
+# The input file is in YAML syntax with each record is a Hash with keys:
+# - layername
+# - filename
+# - format
+# - title
+# and optionally
+# - description
+# - keywords
+# - metadata_links
+
+def main catalog, ws, layers, flags = {}
+  raise ArgumentError, "Layer is malformed" unless not layers.nil? and layers.is_a? Hash and not layers.empty?
+
+  # Iterate over all records in YAML file and create stores in the catalog
+  layers.each do |k, v|
+    ['layername', 'format', 'filename', 'title'].each do |i|
+      raise ArgumentError, "Layer is missing #{i}" unless v.include?(i) and v[i] != nil
+    end
+    puts "Processing layer #{k}" if flags[:verbose]
+
+    layername = v['layername'].strip
+    format = v['format'].strip
+    
+    case format
+    when 'GeoTIFF'
+      do_raster catalog, layername, format, ws, v, flags
+    when 'Shapefile'
+      do_vector catalog, layername, format, ws, flags[:datastore], v, flags
+    else
+      raise NotImplementedError, "Unsupported format #{format}"    
+    end
+  end
+end
+
+def from_druid druid, flags
+  prj = flags[:projection] || "EPSG:4326"
+  prj = prj.split(':').join('_')
+  druid = DruidTools::Druid.new(druid, flags[:datadir])
+  mods_fn = druid.path('metadata/descMetadata.xml')
+  puts "Loading #{mods_fn}" if flags[:verbose]
+  mods = Mods::Record.new
+  mods.from_url(mods_fn)
+  zipfn = nil
+  layername = nil
+  Dir.glob(druid.content_dir + "/*_#{prj}.zip") do |fn|
+    puts "Found EPSG 4326 zip: #{fn}" if flags[:verbose]
+    zipfn = fn
+    layername = File.basename(zipfn, "_#{prj}.zip")
+    puts "Derived layername #{zipfn} -> #{layername}" if flags[:verbose]
+  end
+  if not zipfn
+    Dir.glob(druid.content_dir + "/*.zip") do |fn|
+      zipfn = fn
+      layername = File.basename(zipfn, '.zip')
+    end
+  end
+  raise ArgumentError, zipfn unless File.exist?(zipfn) and layername
+  ap({:zipfn => zipfn, :layername => layername}) if flags[:verbose]
+  r = { 
+    'vector' => {
+      'druid' => druid,
+      'remote' => flags[:remote],
+      'format' => 'Shapefile',
+      'layername' => layername,
+      'filename' => zipfn,
+      'title' => mods.full_titles.first,
+      'description' => mods.term_value(:abstract),
+      'keywords' => [mods.term_values([:subject, 'topic']),
+                     mods.term_values([:subject, 'geographic'])].flatten,
+      'metadata_links' => [{
+        'metadataType' => 'TC211',
+        'content' => "http://purl.stanford.edu/#{druid.id}.geoMetadata"
+      }]
+    }
+  }
+  r
+end
+
+# __MAIN__
+begin
+  flags = {
+    :debug => false,
+    :delete => false,
+    :verbose => true,
+    :datadir => '/var/geomdtk/current/workspace',
+    :format => 'MODS',
+    :remote => 'localfile',
+    :datastore => GeoHydra::Config.geoserver.datastore || nil,
+    :workspace => GeoHydra::Config.geoserver.workspace || 'druid',
+    :namespace => GeoHydra::Config.geoserver.namespace || 'http://purl.stanford.edu'
+    # :host => 'localhost',
+    # :port => 5432,
+    # :user => 'geostaff',
+    # :database => 'geoserver',
+    # :schema => 'druid'
+  }
+  
+  OptionParser.new do |opts|
+    opts.banner = "
+Usage: #{File.basename(__FILE__)} [-v] [--delete] [-f MODS] [druid ... | < druids]
+       #{File.basename(__FILE__)} [-v] [--delete] -f YAML [input.yaml ... | <input.yml]
+           
+    "
+    opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+      flags[:debug] = true if flags[:verbose]
+      flags[:verbose] = true
+    end
+    opts.on("--delete", "Delete workspaces recursively") do |v|
+      flags[:delete] = true
+    end
+    opts.on("-d DIR", "--datadir DIR", "Data directory on GeoServer (default: #{flags[:datadir]})") do |v|
+      raise ArgumentError, "Invalid directory #{v}" unless File.directory?(v)
+      flags[:datadir] = v
+    end
+    opts.on("-f FORMAT", "--format=FORMAT", "Input file format of MODS or YAML (default: #{flags[:format]})") do |v|
+      raise ArgumentError, "Invalid format #{v}" unless ['YAML', 'MODS'].include?(v.upcase)
+      flags[:format] = v.upcase
+    end
+    opts.on("--workspace NAME", "Workspace on GeoServer (default: #{flags[:workspace]})") do |v|
+      flags[:workspace] = v.to_s
+    end
+    opts.on("--namespace NAME", "Namespace on GeoServer (default: #{flags[:namespace]})") do |v|
+      flags[:namespace] = v.to_s
+    end
+    opts.on("--datastore NAME", "Datastore on GeoServer in which data are loaded") do |v|
+      flags[:datastore] = v.to_s
+    end
+    opts.on("--remote NAME", "Remote action (default: #{flags[:remote]})") do |v|
+      flags[:remote] = v.to_s
+    end
+    opts.on("--dburl URL", "Database URL") do |v|
+      url = URI(v)
+      raise ArgumentError, "Invalid database URL (#{v}) -- postgresql://u@h:p/db#s" unless url.scheme == 'postgresql'
+      flags[:host] = url.host
+      flags[:port] = url.port || flags[:port]
+      flags[:database] = url.path.gsub(%r{^/}, '') || flags[:database]
+      flags[:user] = url.user || flags[:user]
+      flags[:schema] = url.fragment || flags[:schema]
+      flags[:remote] = 'postgis-db'
+      flags[:datastore] = 'postgis'
+    end
+  end.parse!
+  
+  ap({:flags => flags}) if flags[:debug]
+  
+  # init
+  # Connect to the GeoServer catalog
+  puts "Connecting to catalog..." if flags[:verbose]
+  catalog = RGeoServer::catalog
+
+  # Obtain a handle to the workspace and clean it up. 
+  ws = RGeoServer::Workspace.new catalog, :name => flags[:workspace]
+  puts "Workspace: #{ws.name} new?=#{ws.new?}" if flags[:verbose]
+  ws.delete :recurse => true if flags[:delete] and not ws.new?
+  ws.enabled = 'true'
+  ws.save
+  puts "Workspace: #{ws.name} ready" if flags[:verbose]
+
+  if ARGV.size > 0
+    ARGV.each do |v|
+      case flags[:format]
+      when 'YAML' then
+        main(catalog, ws, YAML::load_file(v), flags)
+      when 'MODS' then
+        main(catalog, ws, from_druid(v, flags), flags)
+      end
+    end
+  else
+    case flags[:format]
+    when 'YAML' then
+      main(catalog, ws, YAML::load($stdin), flags)
+    when 'MODS' then
+      $stdin.readlines.each do |line|
+        main(catalog, ws, from_druid(line.strip, flags), flags)
+      end
+    end
+  end
+rescue SystemCallError => e
+  $stderr.puts "ERROR: #{e.message}"
+  $stderr.puts e.backtrace
+  exit(-1)
+end

data/bin/loader_postgis.rb

@@ -0,0 +1,259 @@
+#!/usr/bin/env ruby
+# -*- encoding : utf-8 -*-
+#
+# RGeoServer Batch load layers (batch_demo.rb)
+# Usage: #{File.basename(__FILE__)} [input.yml]
+require File.expand_path(File.dirname(__FILE__) + '/../config/boot')
+require 'optparse'
+require 'mods'
+require 'druid-tools'
+require 'dor-services'
+
+require 'active_record'
+require 'active_support'
+require 'tmpdir'
+
+class RegisteredLayer < ActiveRecord::Base
+  attr_accessible :druid, :layername, :title
+  # self.primary_key = 'druid'
+  
+  def find_by_druid(druid)
+    RegisteredLayer.find(:first, :conditions => [ "druid = ?", druid.to_s])
+  end
+end
+
+
+# ENV['RGEOSERVER_CONFIG'] ||= ENV_FILE + '_rgeoserver.yml'
+# require 'rgeoserver'
+
+#=  Input data. *See DATA section at end of file*
+# The input file is in YAML syntax with each record is a Hash with keys:
+# - layername
+# - filename
+# - format
+# - title
+# and optionally
+# - description
+# - keywords
+# - metadata_links
+
+# USAGE: shp2pgsql [<options>] <shapefile> [[<schema>.]<table>]
+# OPTIONS:
+#   -s [<from>:]<srid> Set the SRID field. Defaults to 0.
+#       Optionally reprojects from given SRID (cannot be used with -D).
+#  (-d|a|c|p) These are mutually exclusive options:
+#      -d  Drops the table, then recreates it and populates
+#          it with current shape file data.
+#      -a  Appends shape file into current table, must be
+#          exactly the same table schema.
+#      -c  Creates a new table and populates it, this is the
+#          default if you do not specify any options.
+#      -p  Prepare mode, only creates the table.
+#   -g <geocolumn> Specify the name of the geometry/geography column
+#       (mostly useful in append mode).
+#   -D  Use postgresql dump format (defaults to SQL insert statements).
+#   -e  Execute each statement individually, do not use a transaction.
+#       Not compatible with -D.
+#   -G  Use geography type (requires lon/lat data or -r to reproject).
+#   -k  Keep postgresql identifiers case.
+#   -i  Use int4 type for all integer dbf fields.
+#   -I  Create a spatial index on the geocolumn.
+#   -S  Generate simple geometries instead of MULTI geometries.
+#   -t <dimensionality> Force geometry to be one of '2D', '3DZ', '3DM', or '4D'
+#   -w  Output WKT instead of WKB.  Note that this can result in
+#       coordinate drift.
+#   -W <encoding> Specify the character encoding of Shape's
+#       attribute column. (default: "UTF-8")
+#   -N <policy> NULL geometries handling policy (insert*,skip,abort).
+#   -n  Only import DBF file.
+#   -T <tablespace> Specify the tablespace for the new table.
+#       Note that indexes will still use the default tablespace unless the
+#       -X flag is also used.
+#   -X <tablespace> Specify the tablespace for the table's indexes.
+#       This applies to the primary key, and the spatial index if
+#       the -I flag is used.
+#   -?  Display this help screen.
+
+def main conn, layers, flags = {}
+  ap({:layers => layers, :flags => flags}) if flags[:debug]
+  layers.each do |k, v|
+    %w{layername format filename title}.each do |i|
+      raise ArgumentError, "Layer is missing required '#{i}'" if v[i.to_sym].nil? or not v.include?(i.to_sym)
+    end
+    puts "Processing layer #{k}" if flags[:verbose]
+  
+    layername = v[:layername].strip
+    format = v[:format].downcase.strip.to_sym
+    druid = v[:druid]
+    title = v[:title].strip
+      
+    case format
+    when :shapefile      
+      ap({:v => v}) if flags[:debug]
+      Dir.mktmpdir('shp', druid.temp_dir) do |d|
+        begin
+          system("unzip -oj '#{v[:filename]}' -d '#{d}'")
+          system("ls -la #{d}")
+          Dir.glob("#{d}/*.shp") do |shp|
+            # XXX: HARD CODED projection here -- extract from MODS or ISO19139
+            # XXX: Perhaps put the .sql data into the content directory as .zip for derivative
+            # XXX: -G for the geography column causes some issues with GeoServer
+            system("shp2pgsql -d -D -I -W #{flags[:encoding]} '#{shp}' #{flags[:schema]}.#{druid.id} > '#{druid.temp_dir}/#{druid.id}.sql'")
+            system('psql --no-psqlrc --no-password --quiet ' +
+                   "--host='#{flags[:host.to_s]}' " +
+                   "--port='#{flags[:port.to_s]}' " +
+                   "--username='#{flags[:username.to_s]}' " + 
+                   "--dbname='#{flags[:database.to_s]}' " +
+                   "--file='#{druid.temp_dir}/#{druid.id}.sql' ")
+          end
+        rescue Exception => e
+          FileUtils.rm_rf(d) if File.exist?(d)
+        end
+      end
+      
+      if flags[:register]
+        puts "Registering layer #{druid.id}, #{layername}, #{title}" if flags[:verbose]
+        layer = RegisteredLayer.find_by_druid druid.id
+        ap({:found_layer => layer}) if flags[:debug]
+        if layer.nil?
+          layer = RegisteredLayer.new( 
+            :druid => druid.id,
+            :layername => layername,
+            :title => title
+          )
+        end
+        layer.layername = layername
+        layer.title = title
+        ap({:updated_layer => layer}) if flags[:debug]
+        layer.save
+      end
+    else
+      raise NotImplementedError, "Unsupported format #{format}"    
+    end
+  end
+end
+
+def from_druid druid, flags
+  ap({:druid => druid}) if flags[:debug]
+  prj = flags[:projection] || "EPSG:4326"
+  prj = prj.split(':').join('_')
+  druid = DruidTools::Druid.new(druid, flags[:datadir])
+  mods_fn = druid.path('metadata/descMetadata.xml')
+  puts "Loading #{mods_fn}" if flags[:verbose]
+  mods = Mods::Record.new
+  mods.from_url(mods_fn)
+  ap({:mods => mods}) if flags[:debug]
+
+  geo_fn = druid.path('metadata/geoMetadata.xml')
+  puts "Loading #{geo_fn}" if flags[:verbose]
+  geo = Dor::GeoMetadataDS.from_xml(File.read(geo_fn))
+  ap({:geo => geo}) if flags[:debug]
+
+  zipfn = nil
+  layername = nil
+  projection = nil
+  Dir.glob(druid.content_dir + "/*_#{prj}.zip") do |fn|
+    puts "Found EPSG 4326 zip: #{fn}" if flags[:verbose]
+    projection = '4326'
+    zipfn = fn
+    layername = File.basename(zipfn, "_#{prj}.zip")
+    puts "Derived layername #{zipfn} -> #{layername}" if flags[:verbose]
+  end
+  if not zipfn
+    Dir.glob(druid.content_dir + "/*.zip") do |fn|
+      zipfn = fn
+      layername = File.basename(zipfn, '.zip')
+    end
+  end
+  raise ArgumentError, zipfn unless File.exist?(zipfn) and layername
+  ap({:zipfn => zipfn, :layername => layername}) if flags[:debug]
+  r = { 
+    :vector => {
+      :druid => druid,
+      :format => 'Shapefile',
+      :layername => layername,
+      :filename => zipfn,
+      :title => mods.full_titles.first,
+      :projection => projection.nil?? '4326' : projection
+    }
+  }
+  r
+end
+
+# __MAIN__
+begin
+  flags = {
+    :debug => false,
+    :verbose => false,
+    :register => false,
+    :register_drop => false,
+    :register_table => 'registered_layers',
+    :encoding => 'UTF-8',
+    :datadir => '/var/geomdtk/current/workspace',
+    :schema => GeoHydra::Config.postgis.schema || 'public'
+  }
+  
+  OptionParser.new do |opts|
+    opts.banner = "
+Usage: #{File.basename(__FILE__)} [-v] [druid ... | < druids]
+"
+    opts.on("-v", "--verbose", "Run verbosely") do |v|
+      flags[:debug] = true if flags[:verbose]
+      flags[:verbose] = true
+    end
+    opts.on("-d DIR", "--datadir DIR", "Data directory on GeoServer (default: #{flags[:datadir]})") do |v|
+      raise ArgumentError, "Invalid directory #{v}" unless File.directory?(v)
+      flags[:datadir] = v
+    end    
+    opts.on("-R", "--register", "Register shapefile") do |v|
+      flags[:register] = true
+    end
+
+    %w{schema encoding}.each do |k|
+      opts.on("--#{k} #{k.upcase}", "PostGIS #{k} (default: #{flags[k.to_sym]})") do |v|
+        flags[k.to_sym] = v
+      end
+    end
+  end.parse!
+
+  ap({:flags => flags}) if flags[:debug]
+  dbfn = File.expand_path(File.dirname(__FILE__) + '/../config/database.yml')
+  puts "Loading #{dbfn}" if flags[:verbose]
+  dbconfig = YAML.load(File.read(dbfn))
+  raise ArgumentError, "Missing configuration for environment" unless dbconfig.include?(ENV['GEOHYDRA_ENVIRONMENT'])
+  flags.merge! dbconfig[ENV['GEOHYDRA_ENVIRONMENT']]
+  ap({:flags => flags}) if flags[:debug]
+  
+  ActiveRecord::Base.logger = ActiveSupport::BufferedLogger.new($stderr) if flags[:debug]
+  
+  conn = ActiveRecord::Base.establish_connection flags
+  conn.with_connection do |db|
+    # ap({:obj => db, :klass => db.class, :methods => db.public_methods, :schema_search_path => db.schema_search_path})
+    puts "Connected to PostgreSQL #{db.select_value("SHOW server_version")} " +
+         "using #{db.current_database} database" if flags[:verbose]
+    if db.select_value('select default_version from pg_catalog.pg_available_extensions where name = \'postgis\'') =~ /^(2\.[\.\d]*)$/
+      puts "Using PostGIS #{$1}" if flags[:verbose]
+    else
+      raise NotImplementedError, "Database does not have PostGIS support"
+    end
+    if flags[:register]
+      n = db.select_value("SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public' and tablename = '#{flags[:register_table]}'")
+      if n.to_i == 0 or flags[:register_drop]
+        puts "Creating registry table in public.#{flags[:register_table]}" if flags[:verbose]
+        db.execute("DROP TABLE public.#{flags[:register_table]}") if flags[:register_drop]
+        db.execute("
+          CREATE TABLE public.#{flags[:register_table]}
+          (
+            druid character varying NOT NULL PRIMARY KEY,
+            layername character varying NOT NULL,
+            title character varying NOT NULL
+          );      
+          ")
+      end
+    end
+  end
+    
+  (ARGV.size > 0 ? ARGV : $stdin).each do |s|
+      main(conn, from_druid(s.strip, flags), flags)
+  end
+end