geoblacklight-schema 0.2.0

data/tools/ogp/3_stanford.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+require 'awesome_print'
+require 'nokogiri'
+require 'json'
+
+srcdir = '/var/cache/opengeometadata/edu.stanford.purl'
+
+layers = JSON.parse(File.open('transformed.json', 'rb').read)
+raise RuntimeError unless layers.is_a?(Array) && layers.first.is_a?(Hash)
+
+Dir.glob("#{srcdir}/**/geoblacklight.xml").each do |fn|
+  puts "Loading #{fn}"
+  doc = Nokogiri::XML(File.open(fn, 'rb').read)
+  # ap({:doc => doc})
+  new_layer = {}
+  doc.xpath('//solr:field', 'xmlns:solr' => 'http://lucene.apache.org/solr/4/document').each do |field|
+    k = field['name'].to_s.strip.to_sym
+    if k.to_s =~ /_sm$/
+      if new_layer[k].nil?
+        new_layer[k] = []
+      end
+      new_layer[k] << field.content
+    else
+      new_layer[k] = field.content
+    end
+  end
+  # ap({:new_layer => new_layer})
+  layers << new_layer
+end
+
+File.open('transformed.json', 'wb') do |f|
+  f << JSON::pretty_generate(layers)
+end
+
+puts "Transformed now has #{layers.size} layers"

data/tools/ogp/4_select.rb

@@ -0,0 +1,189 @@
+#!/usr/bin/env ruby
+#
+# Usage: select.rb
+
+require 'awesome_print'
+require 'json'
+
+# __MAIN__
+selected = []
+srand Time.now.to_i
+fn = 'transformed.json'
+valid = JSON::parse(File.read(fn))
+
+# select random records
+valid.each do |i|
+  if rand < 0.01
+    selected << i
+  end
+end
+
+puts "Selected #{selected.size} random records"
+
+# iterate through specific layer_id's
+%w{
+  http://arks.princeton.edu/ark:/88435/02870w62c
+  http://purl.stanford.edu/jf841ys4828
+  http://purl.stanford.edu/jr127wd5810
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.EZ_F7DISTRICTS_2008
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.INT_A1CNTRY_2005
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.INT_A8MISPOP_2004
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.IN_HYDERABAD_P2ROAD_2007
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.NA_N3NOREGESP_2008
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.NA_P61AIRPORTS_2009
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.SF_WC_G46LEASEAREAS_1998
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.SZ_E29VAC_2002
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.US_F7MPO_2006
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.US_MA_CAMBRIDGE_C3BASINS_2007
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.US_MA_CAMBRIDGE_G46PORCH_2010
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.US_MA_CAMBRIDGE_S1CONSV_2007
+  urn:arrowsmith.mit.edu:MIT.SDE_DATA.US_MA_E25ZCTA5DCT_2000
+  urn:geodata.tufts.edu:Tufts.AlgeriaLakes97
+  urn:geodata.tufts.edu:Tufts.BahrainRoads94
+  urn:geodata.tufts.edu:Tufts.BulgariaRegions06
+  urn:geodata.tufts.edu:Tufts.CambridgeParkingLot03
+  urn:geodata.tufts.edu:Tufts.CambridgeStairs03
+  urn:geodata.tufts.edu:Tufts.CambridgeTaxingDistFY11
+  urn:geodata.tufts.edu:Tufts.ChinaProvinces90
+  urn:geodata.tufts.edu:Tufts.Denmark4DigPostCode06
+  urn:geodata.tufts.edu:Tufts.Ecuador1MLake08
+  urn:geodata.tufts.edu:Tufts.EgyptLakes97
+  urn:geodata.tufts.edu:Tufts.IrelandRivers06
+  urn:geodata.tufts.edu:Tufts.IsraelPalestineGenevaBord03
+  urn:geodata.tufts.edu:Tufts.JerusalemBuildings08
+  urn:geodata.tufts.edu:Tufts.MarlboroughRoads09
+  urn:geodata.tufts.edu:Tufts.TogoRivers97
+  urn:hul.harvard.edu:HARVARD.SDE.AMS7810_S250_U54_NG47_11
+  urn:hul.harvard.edu:HARVARD.SDE.AMS7810_S250_U54_NG47_16
+  urn:hul.harvard.edu:HARVARD.SDE.ARCBIKE
+  urn:hul.harvard.edu:HARVARD.SDE.ARCHHYDROLN
+  urn:hul.harvard.edu:HARVARD.SDE.ARCMARTASTN
+  urn:hul.harvard.edu:HARVARD.SDE.BRLBOS
+  urn:hul.harvard.edu:HARVARD.SDE.ESRIEURCANALS
+  urn:hul.harvard.edu:HARVARD.SDE.ESRIMIZIP
+  urn:hul.harvard.edu:HARVARD.SDE.KNG_POLICE
+  urn:hul.harvard.edu:HARVARD.SDE.MEACOMMSTAT95
+  urn:hul.harvard.edu:HARVARD.SDE.TG00ALCTY00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00ALKGL
+  urn:hul.harvard.edu:HARVARD.SDE.TG00CAAIR00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00COMSA00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00HIUNI
+  urn:hul.harvard.edu:HARVARD.SDE.TG00IDPLC
+  urn:hul.harvard.edu:HARVARD.SDE.TG00ILCTY00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00INLKA
+  urn:hul.harvard.edu:HARVARD.SDE.TG00MATRT
+  urn:hul.harvard.edu:HARVARD.SDE.TG00MILKD
+  urn:hul.harvard.edu:HARVARD.SDE.TG00MNUNI
+  urn:hul.harvard.edu:HARVARD.SDE.TG00MTLKF
+  urn:hul.harvard.edu:HARVARD.SDE.TG00MTPUMA
+  urn:hul.harvard.edu:HARVARD.SDE.TG00NJPUMA
+  urn:hul.harvard.edu:HARVARD.SDE.TG00NMVOT00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00ORLPT
+  urn:hul.harvard.edu:HARVARD.SDE.TG00PRPMS00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00VAAIR00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00VATRT00
+  urn:hul.harvard.edu:HARVARD.SDE.TG00VILKD
+  urn:hul.harvard.edu:HARVARD.SDE.TG00VTSEC
+  urn:hul.harvard.edu:HARVARD.SDE.TG00WAPUMA
+  urn:hul.harvard.edu:HARVARD.SDE.TG95ALCDCPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95AZELMPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95CACTYPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95COCCDPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95HISECPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95ILUNIPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95KSPLCPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95MILKBLN
+  urn:hul.harvard.edu:HARVARD.SDE.TG95MTAIRCUPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95MTURBPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95NMTRTPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95NYCTYPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95OHLPTPT
+  urn:hul.harvard.edu:HARVARD.SDE.TG95ORTRTPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95SCCTYPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95VALKDLN
+  urn:hul.harvard.edu:HARVARD.SDE.TG95VALKHLN
+  urn:hul.harvard.edu:HARVARD.SDE.TG95VATAZPY
+  urn:hul.harvard.edu:HARVARD.SDE.TG95WVLKALN
+  urn:hul.harvard.edu:HARVARD.SDE.USGS15MA_GREENFIE_1890
+  urn:hul.harvard.edu:HARVARD.SDE.VMAP1AQUEDCTL
+  urn:hul.harvard.edu:HARVARD.SDE.VMAP1EMBANKL
+  urn:hul.harvard.edu:HARVARD.SDE.VMAP1SEAICEA
+  urn:hul.harvard.edu:HARVARD.SDE.VMAP1VEGTXT
+  urn:hul.harvard.edu:HARVARD.SDE2.AFRICOVER_TZ_SPAT_AGG
+  urn:hul.harvard.edu:HARVARD.SDE2.AM_ONC_L04L
+  urn:hul.harvard.edu:HARVARD.SDE2.CH2000_L_F
+  urn:hul.harvard.edu:HARVARD.SDE2.DMA50K_46751L
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI04EURREGDEMOG
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI04LAKES
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI04MXURBAN
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI04USLALNDMRK
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI06EURPLACES
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI06USBLKPOP_ME
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI07USBLKPOP_AK
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI07USBLKPOP_ME
+  urn:hul.harvard.edu:HARVARD.SDE2.ESRI12USTRACTS
+  urn:hul.harvard.edu:HARVARD.SDE2.EURATLAS_CITIES_1900
+  urn:hul.harvard.edu:HARVARD.SDE2.G3774_P9_1849_C8
+  urn:hul.harvard.edu:HARVARD.SDE2.G3802_E44_1821_N4_SH2
+  urn:hul.harvard.edu:HARVARD.SDE2.G3804_N4C2_1961_G4_SH_6
+  urn:hul.harvard.edu:HARVARD.SDE2.G3804_N4_2M3_1845_S7_SH_2
+  urn:hul.harvard.edu:HARVARD.SDE2.G6714_N2_1826_J6
+  urn:hul.harvard.edu:HARVARD.SDE2.G7064_S2_1834_K3
+  urn:hul.harvard.edu:HARVARD.SDE2.H008768589_V07_0007
+  urn:hul.harvard.edu:HARVARD.SDE2.MADRG_K42073B1
+  urn:hul.harvard.edu:HARVARD.SDE2.MATWN_3764_B6P3_1850_P4
+  urn:hul.harvard.edu:HARVARD.SDE2.ME3732_P4_1798_C3
+  urn:hul.harvard.edu:HARVARD.SDE2.MGISBIOCHP
+  urn:hul.harvard.edu:HARVARD.SDE2.TG10NHVTD
+  urn:hul.harvard.edu:HARVARD.SDE2.TG10USAIANNH
+  urn:hul.harvard.edu:HARVARD.SDE2.USGS_GT_CONCEPCION_MINAS_DNN
+  urn:hul.harvard.edu:HARVARD.SDE2.USGS_GT_GUATEMALA_CITY_HYP
+  urn:hul.harvard.edu:HARVARD.SDE2.USGS_NU_ESTELI_RDL
+  urn:hul.harvard.edu:HARVARD.SDE2.USGS_NU_LEON_RRL
+  urn:hul.harvard.edu:HARVARD.SDE2.USGS_NU_POSOLTEGA_DSL
+  urn:hul.harvard.edu:HARVARD.SDE2.VT3754_H52G46_1836_M3
+  urn:massgis.state.ma.us:MassGIS.EOPS.UCR_00_S_V
+  urn:massgis.state.ma.us:MassGIS.GISDATA.ACECS_ARC
+  urn:massgis.state.ma.us:MassGIS.GISDATA.FISHTRAPS_PT
+  urn:massgis.state.ma.us:MassGIS.GISDATA.REGDPHEPC_POLY
+}.each do |i|
+  valid.each do |j|
+    selected << j if j['dc_identifier_s'] == i
+  end
+end
+puts "Selected #{selected.size} specific IDs and random records"
+
+# iterate through specific conditions
+found_case = {}
+valid.each do |j|
+  if !found_case['1'] && j['dc_rights_s'] == 'Public' && j['dct_provenance_s'] == 'Stanford'
+    selected << j
+    found_case['1'] = true
+  end
+  if !found_case['2'] && j['dct_references_s'] == '{}'
+    selected << j
+    found_case['2'] = true
+  end
+  %w{Berkeley Columbia Harvard Minnesota MIT MassGIS Tufts UCLA}.each do |i|
+    if !found_case[i] && j['dct_provenance_s'] == i
+      selected << j
+      found_case[i] = true
+    end
+  end
+  %w{Line Point Polygon Raster Scanned\ Map Paper\ Map Digital}.each do |i|
+    if !found_case[i] && j['layer_geom_type_s'] == i
+      selected << j
+      found_case[i] = true
+    end
+  end
+  %w{Shapefile GeoTIFF Paper CD-ROM DVD-ROM}.each do |i|
+    if !found_case[i] && j['dc_format_s'] == i
+      selected << j
+      found_case[i] = true
+    end
+  end
+end
+puts "Selected #{selected.size} specific cases and IDs and random records"
+
+
+File.open('selected.json', 'wb') {|f| f << JSON.pretty_generate(selected)}

data/tools/ogp/5_ingest.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'json'
+require 'rsolr'
+
+
+class IngestOgp
+  def initialize(collection, url, skip = 0)
+    @skip = skip
+    raise ArgumentError, 'Collection not defined' unless collection.is_a? String
+    @solr = RSolr.connect(:url => (url + '/' + collection))
+    yield self
+    close
+  end
+  
+  def ingest(fn)
+    puts "Ingesting #{fn}"
+    json = JSON::parse(File.read(fn))
+    n = 0
+    json.each do |doc|
+      next unless doc.is_a? Hash and not doc.empty?
+      doc.delete('_version_')
+      doc.delete('timestamp')
+      putc "."
+      begin
+        @solr.add doc      
+      rescue Exception => e
+        puts e
+      end unless n < @skip
+      
+      n += 1
+      if n % 100 == 0
+        @solr.commit 
+        puts "\ncommit 100 records, #{n} total\n"
+      end
+    end
+    puts "\n#{n} records\n"
+    @solr.commit
+  end
+  
+  def close
+    @solr.commit
+    #@solr.optimize
+    @solr = nil
+  end
+  
+end
+
+
+# __MAIN__
+IngestOgp.new(ARGV[0], (ARGV[1].nil?? 'http://localhost:8080/solr' : ARGV[1]), ARGV[2].nil?? 0 : ARGV[2].to_i) do |ogp|
+  Dir.glob("transformed*.json") do |fn|
+    ogp.ingest(fn)
+  end
+end

data/tools/ogp/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gem 'rsolr'

data/tools/solr/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gem 'rsolr'
+gem 'nokogiri'

data/tools/solr/purge.rb

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# 
+# Usage: purge.rb my-collection http://localhost:8080/solr query
+#
+require 'rsolr'
+
+class SolrPurger
+  def initialize(collection, url)
+    raise ArgumentError, 'Collection not defined' unless collection.is_a? String
+    @solr = RSolr.connect(:url => (url + '/' + collection))
+    yield self
+    close
+  end
+  
+  def purge q
+    @solr.delete_by_query(q)
+  end
+  
+  def close
+    @solr.commit
+    @solr = nil
+  end
+  
+end
+
+# __MAIN__
+if ARGV[0] == '--help' || ARGV.size != 2
+  puts 'Usage: purge.rb my-collection http://localhost:8080/solr'
+  exit -1
+end
+SolrPurger.new(ARGV[0], ARGV[1]) do |sp|
+  sp.purge ARGV[2]
+end

data/tools/solr/upload.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# 
+# Usage: upload.rb http://localhost:8080/my-collection file1.xml [file2.json...]
+#
+require 'rsolr'
+require 'nokogiri'
+
+if ARGV.size < 2
+  puts 'Usage: upload.rb http://localhost:8080/my-collection file1.xml [file2.json...]'
+  exit -1
+end
+
+stop_on_error = false
+
+solr = RSolr.connect :url => ARGV.delete_at(0) 
+
+ARGV.each do |fn|
+  puts "Processing #{fn}"
+  begin
+    if fn =~ /.xml$/
+      doc = Nokogiri::XML(File.open(fn, 'rb').read)
+      solr.update :data => doc.to_xml    
+    elsif fn =~ /.json$/
+      doc = JSON.parse(File.open(fn, 'rb').read)
+      solr.add doc
+    else
+      raise RuntimeError, "Unknown file type: #{fn}"
+    end
+  rescue => e
+    puts "ERROR: #{e}: #{e.backtrace}"
+    raise e if stop_on_error
+  end
+end
+
+solr.commit

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: geoblacklight-schema
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Darren Hardy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+description: 
+email:
+- drh@stanford.edu
+executables:
+- fgdc2mods.rb
+- mods2geoblacklight.rb
+- xsltproc-saxon
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE
+- README.md
+- bin/fgdc2mods.rb
+- bin/mods2geoblacklight.rb
+- bin/xsltproc-saxon
+- conf/protwords.txt
+- conf/schema.xml
+- conf/solrconfig.xml
+- conf/stopwords_en.txt
+- conf/synonyms.txt
+- examples/Gemfile
+- examples/generate-example-doc.rb
+- examples/selected.json
+- examples/upload-to-solr.rb
+- geoblacklight-schema.gemspec
+- lib/geoblacklight/gazetteer.csv
+- lib/geoblacklight/gazetteer.rb
+- lib/xslt/arcgis_to_iso19110.xsl
+- lib/xslt/fgdc2mods.xsl
+- lib/xslt/iso2mods.xsl
+- lib/xslt/mods2geoblacklight.xsl
+- lib/xslt/mods2ogp.xsl
+- tools/fgdc2html/Gemfile
+- tools/fgdc2html/fgdc2html.css
+- tools/fgdc2html/fgdc2html.js
+- tools/fgdc2html/fgdc2html.xsl
+- tools/fgdc2html/render.rb
+- tools/iso2html/Gemfile
+- tools/iso2html/iso-html.xsl
+- tools/iso2html/render.rb
+- tools/iso2html/utils/convert-enumerations.xsl
+- tools/iso2html/utils/convert-latlong.xsl
+- tools/iso2html/utils/decode-uri/base.css
+- tools/iso2html/utils/decode-uri/index.html
+- tools/iso2html/utils/elements-fgdc.xml
+- tools/iso2html/utils/elements-iso.xml
+- tools/iso2html/utils/printFormatted.xsl
+- tools/iso2html/utils/printTextLines.xsl
+- tools/iso2html/utils/replace-newlines.xsl
+- tools/iso2html/utils/replace-string.xsl
+- tools/iso2html/utils/strip-digits.xsl
+- tools/iso2html/utils/url-decode.xsl
+- tools/iso2html/utils/wrap-text.xsl
+- tools/ogp/0_download.rb
+- tools/ogp/1_validate.rb
+- tools/ogp/2_transform.rb
+- tools/ogp/3_stanford.rb
+- tools/ogp/4_select.rb
+- tools/ogp/5_ingest.rb
+- tools/ogp/Gemfile
+- tools/solr/Gemfile
+- tools/solr/purge.rb
+- tools/solr/upload.rb
+- vendor/.keep
+homepage: http://github.com/geoblacklight/geoblacklight-schema
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Schema Tools for GeoBlacklight
+test_files: []
+has_rdoc: