geoblacklight-schema 0.2.0

data/tools/ogp/0_download.rb

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+require 'open-uri'
+
+class DownloadOgp
+  URL = {
+    'tufts' => 'http://geodata.tufts.edu/solr/select',
+    'mit' => 'http://arrowsmith.mit.edu/solr/select',
+    'berkeley' => 'http://gis.lib.berkeley.edu:9081/solr4/select',
+    'harvard' => 'http://geodata.tufts.edu/solr/select', # Harvard uses Tufts solr index
+    'ucla' => 'http://vizsla.oit.ucla.edu:8080/solr/select',
+    'columbia' => 'http://culspatial.cul.columbia.edu/solr/select',
+    'minnesota' => 'http://ec2-54-87-229-228.compute-1.amazonaws.com:8080/solr/collection1/select'
+  }
+  
+  FIELDS = %w{
+    Abstract
+    Access
+    Area
+    Availability
+    CenterX
+    CenterY
+    ContentDate
+    DataType
+    ExternalLayerId
+    FgdcText
+    GeoReferenced
+    HalfHeight
+    HalfWidth
+    Institution
+    LayerDisplayName
+    LayerId
+    Location
+    MaxX
+    MaxY
+    MinX
+    MinY
+    Name
+    PlaceKeywords
+    Publisher
+    SrsProjectionCode
+    ThemeKeywords
+    WorkspaceName
+  }.join(',')
+  
+  def download(src, i, n, w=50)
+    start = 0
+    i = src if i.nil?
+    while start < n do
+      fetch(src, i, start, w)
+      start += w
+    end
+  end
+  
+  # fetch a set of Solr records from the src provider about the target institution
+  #
+  # @param [String] src The source provider of the Solr records
+  # @param [String] target the target institution
+  # @param [Integer] start
+  # @param [Integer] rows
+  def fetch(src, target, start, rows, datadir = 'data')
+    fn = File.join(datadir, "#{src.downcase}_#{target.downcase}_#{sprintf('%05i', start)}_#{rows}.json")
+    unless File.exist?(fn)
+      raise "Unknown URL for #{src}" unless URL.include?(src.downcase)
+      puts "Downloading #{target} #{start} to #{start+rows}"
+      url = "#{URL[src.downcase]}?" + URI::encode_www_form(
+                  'q' => '*:*', 
+                  'fq' => "Institution:#{target}",
+                  'start' => start,
+                  'rows' => rows,
+                  'wt' => 'json',
+                  'indent' => 'on',
+                  'fl' => FIELDS
+                  )
+      puts "    #{url}" if $DEBUG
+      open(url) do |res|
+        File.open(fn, 'wb') do |f|
+          f.write(res.read())
+        end
+      end
+    else
+      puts "Using cache for #{target} #{start} to #{start+rows}"
+    end
+  end
+end
+
+# __MAIN__
+ogp = DownloadOgp.new
+ogp.download('Berkeley', 'Berkeley', 450)
+ogp.download('Tufts', 'MassGIS', 600)
+ogp.download('Tufts', 'Tufts', 2850)
+ogp.download('Tufts', 'Harvard', 10000)
+ogp.download('MIT', 'MIT', 9200)
+ogp.download('UCLA', 'UCLA', 200)
+ogp.download('Columbia', 'Columbia', 3600)
+ogp.download('Minnesota', 'Minnesota', 2300)

data/tools/ogp/1_validate.rb

@@ -0,0 +1,225 @@
+#!/usr/bin/env ruby
+#
+# Usage: validate_ogp [output.json]
+#
+#  Requires data/*.json as input and output to valid.json
+#
+require 'awesome_print'
+require 'json'
+require 'uri'
+require 'date'
+
+class ValidateOgp
+  def initialize(fn)
+    @wms_servers = {}
+    @output = File.open(fn, 'wb')
+    @output.write "[\n"
+    yield self
+    self.close
+  end
+
+  def validate_file(fn)
+    stats = { :accepted => 0, :rejected => 0 }
+    puts "Validating #{fn}"
+    json = JSON::parse(File.read(fn))
+    json['response']['docs'].each do |doc| # contains JSON Solr query results
+      begin
+        validate(doc)
+        stats[:accepted] += 1
+      rescue Exception => e
+        puts e
+        stats[:rejected] += 1
+      end
+    end
+    stats
+  end
+
+
+  def validate(layer)
+    id = layer['LayerId']
+
+    %w{LayerId Name Institution Access MinX MinY MaxX MaxY LayerDisplayName}.each do |k|
+      if layer[k].nil? || layer[k].to_s.empty?
+        raise ArgumentError, "ERROR: #{id} missing #{k}"
+      end
+    end
+    
+    k = 'LayerId'
+    if layer[k].is_a? Array
+      layer[k] = layer[k].first
+    end
+    raise ArgumentError, "ERROR: #{k} is not a String: #{layer[k]}" unless layer[k].is_a? String
+    
+    %w{MinX MaxX}.each do |lon|
+      raise ArgumentError, "ERROR: #{id}: Invalid longitude value: #{layer[lon]}" unless lon?(layer[lon])
+    end
+
+    %w{MinY MaxY}.each do |lat|
+      raise ArgumentError, "ERROR: #{id} Invalid latitude value: #{layer[lat]}" unless lat?(layer[lat])
+    end
+    
+    if layer['MinX'].to_s.to_f > layer['MaxX'].to_s.to_f
+      raise ArgumentError, "ERROR: #{id} has MinX > MaxX"
+    end
+
+    if layer['MinY'].to_s.to_f > layer['MaxY'].to_s.to_f
+      raise ArgumentError, "ERROR: #{id} has MinY > MaxY"
+    end
+    
+    k = 'Institution'
+    layer[k] = 'Columbia' if layer[k] == 'Columbia University'
+    if ([layer[k]] & %w{Berkeley Harvard MIT MassGIS Stanford Tufts UCLA Minnesota Columbia}).empty?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+
+    k = 'DataType'
+    layer[k] = 'Paper Map' if layer[k] == 'Paper'
+    if ([layer[k]] & %w{Line Paper\ Map Point Polygon Raster CD-ROM DVD-ROM}).empty?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+
+    k = 'Access'
+    if ([layer[k]] & %w{Public Restricted}).empty?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+
+    k = 'WorkspaceName'
+    if layer[k].nil?
+      layer[k] = layer['Institution']
+    end
+
+    k = 'Availability'
+    if layer[k].downcase == 'online' # cleanup 
+      layer[k] = 'Online'
+    end
+    if layer[k].downcase == 'offline'
+      layer[k] = 'Offline'
+    end
+    if ([layer[k]] & %w{Online Offline}).empty?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+
+    k = 'Location'
+    layer[k] = validate_location(id, layer[k])
+    if layer[k].nil?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+
+    k = 'GeoReferenced'
+    unless layer[k].nil? or layer[k] == true
+      puts "WARNING: #{id} has boundingbox but claims it is not georeferenced"
+      #layer[k] = true
+    end
+    
+    k = 'Area'
+    unless layer[k].to_i >= 0
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+    
+    k = 'ContentDate'
+    if layer[k].nil? || layer[k].to_s.strip.empty?
+      raise ArgumentError, "ERROR: #{id} has unsupported #{k}: #{layer[k]}"
+    end
+    dt = Date.rfc3339(layer[k])
+    if dt.year <= 1 or dt.year > 2100
+      raise ArgumentError, "ERROR: #{id} has invalid #{k}: #{layer[k]}: #{dt}"
+    end
+    
+    # k = 'FgdcText'
+    # unless layer[k].nil? or layer[k].empty?
+    #   layer[k] = ''
+    # end
+
+    @output.write JSON::pretty_generate(layer)
+    @output.write "\n,\n"
+  end
+
+  def close
+    @output.write "\n {} \n]\n"
+    @output.close
+    ap({:wms_servers => @wms_servers})
+  end
+  
+  private
+  
+  def validate_location(id, location)
+    begin
+      begin
+        x = JSON::parse(location)
+      rescue JSON::ParserError => e
+        x = JSON::parse("{ #{location} }") # wrap in dictionary
+      end
+      
+      unless x['externalDownload'].nil?
+        x['download'] = x['externalDownload']
+        x.delete('externalDownload')
+      end
+      unless x['libRecord'].nil?
+        x['url'] = x['libRecord']
+        x.delete('libRecord')
+      end
+      if x['download'].nil? && x['wms'].nil? && (x['wcs'].nil? && x['wfs'].nil?) && x['url'].nil?
+        puts "WARNING: #{id}: Missing Download or WMS or WCS/WFS or URL: #{x}"
+        return {}.to_json
+      end
+      
+      %w{download wms wcs wfs url}.each do |protocol|
+        begin
+          unless x[protocol].nil?
+            if x[protocol].is_a? String
+              if x[protocol].empty? || x[protocol] == "None available"
+                x[protocol] = nil
+                next
+              else
+                x[protocol] = [x[protocol]]
+              end
+            end
+            
+            unless x[protocol].is_a? Array
+              raise ArgumentError, "ERROR: #{id}: Unknown #{protocol} value: #{x}"
+            end
+            
+            x[protocol].each do |url|
+              uri = URI.parse(url)
+              raise ArgumentError, "ERROR: #{id}: Invalid URL: #{uri}" unless uri.kind_of?(URI::HTTP) or uri.kind_of?(URI::HTTPS) or uri.kind_of?(URI::FTP)
+            end
+            
+            # convert from Array to String
+            x[protocol] = x[protocol].first if x[protocol].is_a? Array
+          end
+        rescue URI::InvalidURIError => e
+          raise ArgumentError, "ERROR: #{id}: Invalid URL parsing: #{x}"
+        end        
+      end
+      
+      @wms_servers[x['wms']] = true unless x['wms'].nil?
+
+      return x.to_json
+    end
+    nil
+  end
+  
+  def lon? lon
+    lon >= -180 and lon <= 180
+  end
+  
+  def lat? lat
+    lat >= -90 and lat <= 90
+  end
+end
+
+
+# __MAIN__
+ValidateOgp.new(ARGV[0].nil?? 'valid.json' : ARGV[0]) do |ogp|
+  stats = { :accepted => 0, :rejected => 0 }
+  Dir.glob('data/*.json') do |fn|
+    begin
+      s = ogp.validate_file(fn)      
+    rescue Exception => e
+    end
+    
+    stats[:accepted] += s[:accepted]
+    stats[:rejected] += s[:rejected]
+  end
+  ap({:statistics => stats})
+end

data/tools/ogp/2_transform.rb

@@ -0,0 +1,438 @@
+#!/usr/bin/env ruby
+#
+# Usage: transform_ogp output.json
+#
+#  Reads valid*.json in current directory
+#
+
+require 'awesome_print'
+require 'json'
+require 'uri'
+require 'date'
+require 'nokogiri'
+require 'fileutils'
+require 'open-uri'
+
+# Transforms an OGP schema into GeoBlacklight. Requires input of a JSON array
+# of OGP hashs.
+class TransformOgp
+
+  def initialize(fn)
+    @output = File.open(fn, 'wb')
+    @output.write "[\n"
+    @fgdcdir = 'fgdc'
+    yield self
+    self.close
+  end
+  
+  # @param [String|Array] s the URI to clean up
+  # @return [String] a normalized URI
+  def clean_uri(s)
+    unless s.nil? or s.empty?
+      return (s.is_a?(Array) ? URI(s.first.to_s.strip) : URI(s.to_s.strip)).to_s
+    end
+    ''
+  end
+
+  # @param [String] fn filename of JSON array of OGP hash objects
+  # @return [Hash] stats about :accepted vs. :rejected records
+  def transform_file(fn, layers_json)
+    stats = { :accepted => 0, :rejected => 0 }
+    puts "Parsing #{fn}"
+    json = JSON::parse(File.open(fn, 'rb').read)
+    json.each do |doc| # contains JSON Solr query results
+      unless doc.empty?
+        begin
+          transform(doc, layers_json)
+          stats[:accepted] += 1
+        rescue ArgumentError => e
+          puts e, e.backtrace
+          stats[:rejected] += 1
+        end
+      end
+    end
+    stats
+  end
+
+  # Transforms a single OGP record into a GeoBlacklight record
+  # @param [Hash] layer an OGP hash for a given layer
+  def transform(layer, layers_json, skip_fgdc = false, skip_geoblacklight = false, skip_ogm_check = false)
+    id = layer['LayerId'].to_s.strip
+    puts "Tranforming #{id}"
+
+    # For URN style @see http://www.ietf.org/rfc/rfc2141.txt
+    # For ARK @see https://wiki.ucop.edu/display/Curation/ARK
+    layer['Institution'].strip!
+    prefix = case layer['Institution']
+    when 'Stanford'
+      'http://purl.stanford.edu/'
+    when 'Tufts'
+      'urn:geodata.tufts.edu:'
+    when 'MassGIS'
+      'urn:massgis.state.ma.us:'
+    when 'Berkeley'
+      'http://ark.cdlib.org/ark:/'
+    when 'MIT'
+      'urn:arrowsmith.mit.edu:'
+    when 'Harvard'
+      'urn:hul.harvard.edu:'
+    when 'Minnesota'
+      'urn:umn.edu:'
+    when 'UCLA'
+      'urn:ucla.edu:'
+    when 'Columbia'
+      'urn:columbia.edu:'
+    else
+      raise ArgumentError, 'ERROR: Skipping urn:UNKNOWN:'
+    end
+    uuid = prefix + URI.encode(id)
+    
+    # Parse out the Location to get the WMS/WFS/WCS URLs, if available
+    location = JSON::parse(layer['Location'])
+    raise ArgumentError, "ERROR: #{id} has malformed location" unless location.is_a? Hash
+    
+    # Parse out the bounding box
+    s = layer['MinY'].to_f
+    w = layer['MinX'].to_f
+    n = layer['MaxY'].to_f
+    e = layer['MaxX'].to_f
+    
+    # Parse out the ContentDate date/time
+    begin
+      dt = DateTime.rfc3339(layer['ContentDate']) 
+    rescue => e2
+      raise ArgumentError, "ERROR: #{id} has bad ContentDate: #{layer['ContentDate']}"
+    end
+    
+    # pub_dt = DateTime.rfc3339('2000-01-01T00:00:00Z') # XXX fake data, get from MODS
+    
+    access = layer['Access']
+    collection = nil
+    
+    # Parse out the PURL and other metadata for Stanford
+    if layer['Institution'] == 'Stanford'
+      purl = location['purl']
+      if purl.is_a? Array
+        purl = purl.first
+      end
+      if purl.nil? and uuid =~ /^http/
+        purl = uuid
+      end
+    else
+      purl = nil
+      # Because OGP does not deliminate keywords, we use a heuristic here
+      %w{PlaceKeywords ThemeKeywords}.each do |k|
+        # layer[k] = nil
+        # unless layer[k] =~ /[;,]/ or layer[k].split.size < 4
+          # layer[k] = layer[k].split.join(';')
+        # end
+      end
+    end
+    
+    slug = to_slug(id, layer)
+    
+    layer_geom_type = layer['DataType'].to_s.strip
+    if (layer_geom_type.downcase == 'raster' || layer['LayerDisplayName'] =~ /\(Raster Image\)/) ||
+       (layer['Institution'] == 'Harvard' && layer_geom_type.downcase == 'paper map')
+      format = 'GeoTIFF'
+      layer_geom_type = 'Raster'
+    elsif %w{Point Line Polygon}.include?(layer_geom_type)
+      format = 'Shapefile'
+    elsif layer_geom_type.downcase == 'paper map'
+      format = 'Paper'
+      layer_geom_type = 'Paper Map'
+    elsif layer_geom_type.downcase =~ /-rom$/
+      format = layer_geom_type
+      layer_geom_type = 'Mixed'
+    else
+      raise ArgumentError, "ERROR: Invalid layer_geom_type: #{layer_geom_type}"
+    end
+    
+    # if layer['LayerDisplayName'] =~ /Scanned Map/
+    #   layer_geom_type = 'Scanned Map'
+    #   format = 'Paper'
+    # end
+    
+    layer_id = layer['WorkspaceName'] + ':' + layer['Name']
+    
+    # @see https://github.com/OSGeo/Cat-Interop
+    %w{wcs wfs wms}.each do |k|
+      location[k] = location[k].first if location[k].is_a? Array
+    end
+    refs = {}
+    refs['http://www.opengis.net/def/serviceType/ogc/wcs'] = "#{location['wcs']}" if location['wcs']
+    refs['http://www.opengis.net/def/serviceType/ogc/wfs'] = "#{location['wfs']}" if location['wfs']
+    refs['http://www.opengis.net/def/serviceType/ogc/wms'] = "#{location['wms']}" if location['wms']
+    if layer['Institution'] == 'Harvard'
+      refs['http://schema.org/DownloadAction'] = "#{location['download']}" if location['download']
+      refs['http://schema.org/UserDownloads'] = "#{location['serviceStart']}" if location['serviceStart']
+      refs['http://tilecache.org'] = "#{location['tilecache'].first}" if location['tilecache']
+    else
+      refs['http://schema.org/downloadUrl'] = "#{location['download']}" if location['download']
+    end
+    refs['http://schema.org/url'] = "#{location['url']}" if location['url']
+    if purl
+      refs["http://schema.org/thumbnailUrl"] = "http://stacks.stanford.edu/file/druid:#{id}/preview.jpg"
+      refs["http://schema.org/url"] = "#{clean_uri(purl)}"
+      refs["http://schema.org/downloadUrl"] = "http://stacks.stanford.edu/file/druid:#{id}/data.zip"
+      refs["http://www.loc.gov/mods/v3"] = "#{purl}.mods"
+      refs["http://www.isotc211.org/schemas/2005/gmd/"] = "http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/#{layer_id}/iso19139.xml"
+      refs['http://www.w3.org/1999/xhtml'] = "http://opengeometadata.stanford.edu/metadata/edu.stanford.purl/#{layer_id}/iso19139.html"
+    else
+      refs['http://www.opengis.net/cat/csw/csdgm'] = "http://opengeometadata.stanford.edu/metadata/org.opengeoportal/#{URI::encode(layer_id)}/fgdc.xml"
+      begin
+        _f = open(refs['http://www.opengis.net/cat/csw/csdgm'])
+        refs['http://www.w3.org/1999/xhtml'] = "http://opengeometadata.stanford.edu/metadata/org.opengeoportal/#{URI::encode(layer_id)}/fgdc.html"
+      rescue OpenURI::HTTPError => e2
+        refs.delete('http://www.opengis.net/cat/csw/csdgm')
+      rescue URI::InvalidURIError => e2
+        raise ArgumentError, "ERROR: #{id} has bad LayerId: #{layer['layer_id']}"
+      end unless skip_ogm_check
+    end
+    
+    # If there's no homepage, use the HTML version of the Metadata if available
+    if refs['http://schema.org/url'].nil? && !refs['http://www.w3.org/1999/xhtml'].nil?
+      refs['http://schema.org/url'] = refs['http://www.w3.org/1999/xhtml']
+    end
+    
+    # Make the conversion from OGP to GeoBlacklight
+    #
+    # @see http://dublincore.org/documents/dcmi-terms/
+    # @see http://wiki.dublincore.org/index.php/User_Guide/Creating_Metadata
+    # @see http://www.ietf.org/rfc/rfc5013.txt
+    new_layer = {
+      :uuid               => uuid,
+      
+      # Dublin Core elements
+      :dc_creator_sm      => string2array(layer['Originator']),
+      :dc_description_s   => layer['Abstract'],
+      :dc_format_s        => format,
+      :dc_identifier_s    => uuid,
+      :dc_language_s      => 'English', # 'en', # XXX: fake data
+      :dc_publisher_s     => layer['Publisher'],
+      :dc_rights_s        => access,
+      :dc_subject_sm      => string2array(layer['ThemeKeywords'], true),
+      :dc_title_s         => layer['LayerDisplayName'],
+      :dc_type_s          => 'Dataset',  # or 'Image' for non-georectified, 
+                                         # or 'PhysicalObject' for non-digitized maps
+      # Dublin Core terms
+      :dct_isPartOf_sm    => collection.nil?? nil : [collection],
+      :dct_references_s   => refs.to_json.to_s,
+      :dct_spatial_sm     => string2array(layer['PlaceKeywords'], true),
+      :dct_temporal_sm    => [dt.year.to_s],
+      # :dct_issued_s       => pub_dt.year.to_s,
+      :dct_provenance_s   => layer['Institution'],
+
+     #
+     # xmlns:georss="http://www.georss.org/georss"
+     # A bounding box is a rectangular region, often used to define the extents of a map or a rough area of interest. A box contains two space seperate latitude-longitude pairs, with each pair separated by whitespace. The first pair is the lower corner, the second is the upper corner.
+      :georss_box_s       => "#{s} #{w} #{n} #{e}",
+      :georss_polygon_s   => "#{n} #{w} #{n} #{e} #{s} #{e} #{s} #{w} #{n} #{w}",
+     
+      # Layer-specific schema
+      :layer_slug_s       => slug,
+      :layer_id_s         => layer_id,
+      # :layer_srs_s        => 'EPSG:4326', # XXX: fake data
+      :layer_geom_type_s  => layer_geom_type,
+      :layer_modified_dt  => Time.now.utc.strftime('%FT%TZ'),
+      
+      # derived fields used only by solr, for which copyField is insufficient
+      :solr_bbox  => "#{w} #{s} #{e} #{n}", # minX minY maxX maxY
+      # :solr_ne_pt => "#{n},#{e}",
+      # :solr_sw_pt => "#{s},#{w}",
+      :solr_geom  => "ENVELOPE(#{w}, #{e}, #{n}, #{s})",
+      :solr_year_i => dt.year,
+      # :solr_issued_dt => pub_dt.strftime('%FT%TZ') # Solr requires 1995-12-31T23:59:59Z
+      # :solr_wms_url => location['wms'],
+      # :solr_wfs_url => location['wfs'],
+      # :solr_wcs_url => location['wcs']
+      
+      # :layer_year_i       => dt.year#, # XXX: migrate to copyField
+      # :ogp_area_f         => layer['Area'],
+      # :ogp_center_x_f     => layer['CenterX'],
+      # :ogp_center_y_f     => layer['CenterY'],
+      # :ogp_georeferenced_b   => (layer['GeoReferenced'].to_s.downcase == 'true'),
+      # :ogp_halfheight_f   => layer['HalfHeight'],
+      # :ogp_halfwidth_f    => layer['HalfWidth'],
+      # :ogp_layer_id_s     => layer['LayerId'],
+      # :ogp_name_s         => layer['Name'],
+      # :ogp_location_s     => layer['Location'],
+      # :ogp_workspace_s    => layer['WorkspaceName']
+    }
+    
+    # Remove any fields that are blank
+    new_layer.each do |k, v| 
+      new_layer.delete(k) if v.nil? or (v.respond_to?(:empty?) and v.empty?)
+    end
+    
+    # Write the JSON record for the GeoBlacklight layer
+    @output.write JSON::pretty_generate(new_layer)
+    @output.write "\n,\n"
+    
+    # export into OGM
+    ogm_dir = new_layer[:dct_provenance_s] + '/' + new_layer[:layer_id_s][-2,2].downcase.gsub(/[^a-z0-9]/, '_') + '/' + new_layer[:layer_id_s]
+    unless skip_fgdc or layer['FgdcText'].nil? or layer['FgdcText'].empty?
+      _fn = 'opengeometadata/org.opengeoportal/' + ogm_dir + '/fgdc.xml'
+      unless File.size?(_fn)
+        FileUtils.mkdir_p(File.dirname(_fn)) unless File.directory?(File.dirname(_fn))
+        xml = Nokogiri::XML(layer['FgdcText'])
+        xml.write_xml_to(File.open(_fn, 'wb'), encoding: 'UTF-8', indent: 2)
+      end
+    end
+
+    unless skip_geoblacklight
+      _fn = 'opengeometadata/org.opengeoportal/' + ogm_dir + '/geoblacklight.json'
+      layers_json[new_layer[:layer_id_s]] = ogm_dir
+      unless File.size?(_fn)
+        FileUtils.mkdir_p(File.dirname(_fn)) unless File.directory?(File.dirname(_fn))
+        _s = JSON::pretty_generate(new_layer)
+        File.open(_fn, 'wb') {|f| f.write(_s) }
+      end
+    end
+  end
+
+  def close
+    @output.write "\n {} \n]\n"
+    @output.close
+  end
+    
+  # @param [String] s has semi-colon/comma/gt delimited array
+  # @return [Array] results as array
+  def string2array(s, clean_only = false)
+    return nil if s.nil?
+    if clean_only
+      if s.strip.size > 0 && !s.strip.index(' ') && s.strip.downcase != 'none'
+        [s.strip]
+      else
+        nil
+      end
+    else
+      if s.to_s =~ /[;,>]/
+        s.split(/\s*[;,>]\s*/).uniq.collect {|i| i.strip}
+      elsif s.is_a?(String) && s.size > 0
+        [s.strip]
+      else
+        nil
+      end
+    end
+  end
+  
+  @@slugs = {}
+  def to_slug(id, layer)
+    # strip out schema and usernames
+    name = layer['Name'].sub('SDE_DATA.', '').sub('SDE.', '').sub('SDE2.', '').sub('GISPORTAL.GISOWNER01.', '').sub('GISDATA.', '').sub('MORIS.', '')
+    unless name.size > 1 
+      # use first word of title is empty name
+      name = layer['LayerDisplayName'].split.first 
+    end
+    slug = layer['Institution'] + '-' + name
+    
+    # slugs should only have a-z, A-Z, 0-9, and -
+    slug.gsub!(/[^a-zA-Z0-9\-]/, '-')
+    slug.gsub!(/[\-]+/, '-')
+    
+    # only lowercase
+    slug.downcase!
+    
+    # ensure slugs are unique for this pass
+    if @@slugs.include?(slug)
+      slug += '-' + sprintf("%06d", Random.rand(999999))
+    end
+    @@slugs[slug] = true
+
+    slug
+  end
+
+  # Ensure that the WMS/WFS/WCS location values are as expected
+  def validate_location(id, location)
+    begin
+      x = JSON::parse(location)      
+      %w{download url wms wcs wfs}.each do |protocol|
+        begin
+          unless x[protocol].nil?
+            if x[protocol].is_a? String
+              x[protocol] = [x[protocol]]
+            end
+            
+            unless x[protocol].is_a? Array
+              raise ArgumentError, "ERROR: #{id}: Unknown #{protocol} value: #{x}"
+            end
+            
+            x[protocol].each do |url|
+              uri = clean_uri.parse(url)
+              raise ArgumentError, "ERROR: #{id}: Invalid URL: #{uri}" unless uri.kind_of?(clean_uri::HTTP) or uri.kind_of?(clean_uri::HTTPS)
+            end
+            x[protocol] = x[protocol].first
+          end
+        rescue Exception => e
+          raise ArgumentError, "ERROR: #{id}: Invalid #{k}: #{x}"
+        end        
+      end
+      
+      return x.to_json
+    rescue JSON::ParserError => e
+      raise ArgumentError, "ERROR: #{id}: Invalid JSON: #{location}"
+    end
+    nil
+  end
+  
+  def lon? lon
+    lon >= -180 and lon <= 180
+  end
+  
+  def lat? lat
+    lat >= -90 and lat <= 90
+  end
+end
+
+
+# __MAIN__
+#
+TransformOgp.new(ARGV[0].nil?? 'transformed.json' : ARGV[0]) do |ogp|
+  stats = { :accepted => 0, :rejected => 0 }
+  layers_json = {}
+  
+  Dir.glob('valid*.json') do |fn|
+    s = ogp.transform_file(fn, layers_json)
+    stats[:accepted] += s[:accepted]
+    stats[:rejected] += s[:rejected]
+  end
+  
+  File.open('opengeometadata/org.opengeoportal/layers.json', 'wb') do |f|
+    f << JSON::pretty_generate(layers_json)
+  end
+  
+  ap({:statistics => stats})
+end
+
+# example input data
+__END__
+[
+{
+  "Abstract": "The boundaries of each supervisorial district in Sonoma County based on 2000 census. Redrawn in 2001 using Autobound.",
+  "Access": "Public",
+  "Area": 0.9463444815860053,
+  "Availability": "Online",
+  "CenterX": -122.942159,
+  "CenterY": 38.4580755,
+  "ContentDate": "2000-01-01T01:01:01Z",
+  "DataType": "Polygon",
+  "FgdcText": "...",
+  "GeoReferenced": true,
+  "HalfHeight": 0.39885650000000084,
+  "HalfWidth": 0.593161000000002,
+  "Institution": "Berkeley",
+  "LayerDisplayName": "SCGISDB2_BASE_ADM_SUPERVISOR",
+  "LayerId": "28722/bk0012h5s52",
+  "Location": "{\"wms\":[\"http://gis.lib.berkeley.edu:8080/geoserver/wms\"],\"tilecache\":[\"http://gis.lib.berkeley.edu:8080/geoserver/gwc/service/wms\"],\"download\":\"\",\"wfs\":[\"http://gis.lib.berkeley.edu:8080/geoserver/wfs\"]}",
+  "MaxX": -122.348998,
+  "MaxY": 38.856932,
+  "MinX": -123.53532,
+  "MinY": 38.059219,
+  "Name": "ADM_SUPERVISOR",
+  "PlaceKeywords": "Sonoma County County of Sonoma Sonoma California Bay Area",
+  "Publisher": "UC Berkeley Libraries",
+  "ThemeKeywords": "Supervisorial districts 1st District 2nd District 3rd District 4th District 5th District",
+  "WorkspaceName": "UCB"
+}
+]