geohydra 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +2 -1
- data/.travis.yml +1 -2
- data/Gemfile +1 -8
- data/Gemfile.lock +87 -102
- data/README.md +2 -2
- data/VERSION +1 -1
- data/bin/accession.rb +99 -89
- data/bin/assemble.rb +288 -247
- data/bin/assemble_data.rb +54 -51
- data/bin/assemble_placenames.rb +85 -85
- data/bin/build_stage_options.rb +24 -18
- data/bin/derive_wgs84.rb +65 -66
- data/bin/extract_thumbnail.rb +38 -37
- data/bin/geo2mods.rb +78 -0
- data/bin/geohydra +14 -5
- data/bin/ingest_arcgis.rb +80 -60
- data/bin/iso2geo.rb +64 -0
- data/bin/loader_postgis.rb +121 -227
- data/bin/run_task.rb +23 -0
- data/bin/sync_geoserver_metadata.rb +132 -127
- data/bin/xsltproc-saxon +6 -0
- data/geohydra.gemspec +6 -4
- data/lib/geohydra.rb +5 -0
- data/lib/geohydra/accession.rb +24 -13
- data/lib/geohydra/{arcgis_to_iso19139_fc.xsl → arcgis_to_iso19110.xsl} +0 -0
- data/lib/geohydra/gazetteer.csv +842 -36
- data/lib/geohydra/gazetteer.rb +48 -24
- data/lib/geohydra/mods2geoblacklight.xsl +248 -0
- data/lib/geohydra/mods2ogp.xsl +5 -8
- data/lib/geohydra/transform.rb +8 -2
- data/lib/geohydra/utils.rb +6 -0
- data/lib/geohydra/workflow/gisAssemblyWF.rb +109 -0
- data/lib/geohydra/workflow/gisAssemblyWF.xml +85 -0
- data/lib/geohydra/workflow/gisDeliveryWF.rb +33 -0
- data/lib/geohydra/workflow/gisDeliveryWF.xml +36 -0
- data/lib/geohydra/workflow/gisDiscoveryWF.rb +55 -0
- data/lib/geohydra/workflow/gisDiscoveryWF.xml +28 -0
- data/lib/geohydra/workflow/task.rb +82 -0
- data/ogp/README.md +350 -0
- data/ogp/download.rb +92 -0
- data/ogp/fgdc2mods.sh +9 -0
- data/ogp/fgdc2mods.xsl +884 -0
- data/ogp/ingest.rb +48 -0
- data/ogp/select.rb +20 -0
- data/ogp/transform.rb +354 -0
- data/ogp/validate.rb +182 -0
- data/{bin → scripts}/ingest_tufts.rb +0 -0
- data/scripts/iso2html/doit.sh +15 -0
- data/scripts/iso2html/main.css +66 -0
- data/scripts/iso2html/pacioos-iso-html.xsl +1749 -0
- data/scripts/iso2html/utils/replace-newlines.xsl +97 -0
- data/scripts/iso2html/utils/replace-string.xsl +80 -0
- data/scripts/iso2html/utils/strip-digits.xsl +60 -0
- data/{bin → scripts}/loader.rb +0 -0
- data/scripts/rename_shapefiles.rb +5 -0
- data/scripts/render_gazetteer.rb +36 -0
- data/{bin → scripts}/seed.rb +0 -0
- data/{bin → scripts}/solr_indexer.rb +0 -0
- data/scripts/status.csv +253 -0
- data/scripts/status.rb +32 -0
- data/{bin → scripts}/validate_data.rb +1 -1
- data/solr/kurma-app-dev/conf/lang/stopwords_en.txt +34 -0
- data/solr/kurma-app-dev/conf/protwords.txt +21 -0
- data/solr/kurma-app-dev/conf/schema.xml +156 -0
- data/solr/kurma-app-dev/conf/solrconfig.xml +161 -0
- data/solr/kurma-app-dev/conf/synonyms.txt +29 -0
- data/solr/kurma-app-dev/purge.sh +8 -0
- data/solr/kurma-app-test/conf/lang/stopwords_en.txt +34 -0
- data/solr/kurma-app-test/conf/protwords.txt +21 -0
- data/solr/kurma-app-test/conf/schema.xml +158 -0
- data/solr/kurma-app-test/conf/solrconfig.xml +161 -0
- data/solr/kurma-app-test/conf/synonyms.txt +29 -0
- data/solr/kurma-app-test/deploy.sh +15 -0
- data/solr/kurma-app-test/purge.sh +8 -0
- data/solr/ogp-dev/purge.sh +1 -2
- data/spec/fixtures/bw938nk9584/temp/{TRIPURA-iso19139-fc.xml → TRIPURA-iso19110.xml} +0 -0
- data/spec/fixtures/cc142xj8436/temp/{HARYANA-iso19139-fc.xml → HARYANA-iso19110.xml} +0 -0
- data/spec/fixtures/cg716wc7949/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/cm007pv9601/temp/{MEGHALAYA-iso19139-fc.xml → MEGHALAYA-iso19110.xml} +0 -0
- data/spec/fixtures/cp055nb0189/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/cs838pw3418/temp/{OIL_GAS_FIELDS-iso19139-fc.xml → OIL_GAS_FIELDS-iso19110.xml} +0 -0
- data/spec/fixtures/dd308sy5843/temp/{ORISSA-iso19139-fc.xml → ORISSA-iso19110.xml} +0 -0
- data/spec/fixtures/dd452vk1873/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/dg850pt1796/temp/{STATE1951-iso19139-fc.xml → STATE1951-iso19110.xml} +0 -0
- data/spec/fixtures/dn744tf5427/temp/{DISTRICT1991-iso19139-fc.xml → DISTRICT1991-iso19110.xml} +0 -0
- data/spec/fixtures/dq603nz8402/temp/{STATE2001-iso19139-fc.xml → STATE2001-iso19110.xml} +0 -0
- data/spec/fixtures/dv609zt4699/temp/{ASSAM-iso19139-fc.xml → ASSAM-iso19110.xml} +0 -0
- data/spec/fixtures/dz222hw0585/temp/{PUNJAB-iso19139-fc.xml → PUNJAB-iso19110.xml} +0 -0
- data/spec/fixtures/fd673qb9705/temp/{STATE1971-iso19139-fc.xml → STATE1971-iso19110.xml} +0 -0
- data/spec/fixtures/fg451wp8917/temp/{SIKKIM-iso19139-fc.xml → SIKKIM-iso19110.xml} +0 -0
- data/spec/fixtures/fh247yz0156/temp/{RAJASTHAN-iso19139-fc.xml → RAJASTHAN-iso19110.xml} +0 -0
- data/spec/fixtures/fs487vd1465/temp/{CHHATTISGARH-iso19139-fc.xml → CHHATTISGARH-iso19110.xml} +0 -0
- data/spec/fixtures/fs591bn3317/temp/{HIMACHAL_PRADESH-iso19139-fc.xml → HIMACHAL_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/fw920bc5473/temp/{PLSS_TWN-iso19139-fc.xml → PLSS_TWN-iso19110.xml} +0 -0
- data/spec/fixtures/gj831wj3625/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/gp075nv3265/temp/{PONDICHERRY-iso19139-fc.xml → PONDICHERRY-iso19110.xml} +0 -0
- data/spec/fixtures/gv800hj8141/temp/{BIHAR-iso19139-fc.xml → BIHAR-iso19110.xml} +0 -0
- data/spec/fixtures/gw520gz6339/temp/{DADRA_NAGAR_HAVELI_PT-iso19139-fc.xml → DADRA_NAGAR_HAVELI_PT-iso19110.xml} +0 -0
- data/spec/fixtures/gy054hz1045/temp/{HARYANA-iso19139-fc.xml → HARYANA-iso19110.xml} +0 -0
- data/spec/fixtures/gz352mw6982/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/hb489vm9892/temp/{DISTRICT1981-iso19139-fc.xml → DISTRICT1981-iso19110.xml} +0 -0
- data/spec/fixtures/hw125dq0418/temp/{DELHI-iso19139-fc.xml → DELHI-iso19110.xml} +0 -0
- data/spec/fixtures/hw892mn4587/temp/{KERALA-iso19139-fc.xml → KERALA-iso19110.xml} +0 -0
- data/spec/fixtures/jb371hz3868/temp/{INCOME-iso19139-fc.xml → INCOME-iso19110.xml} +0 -0
- data/spec/fixtures/jc017yk9928/temp/{KARNATAKA-iso19139-fc.xml → KARNATAKA-iso19110.xml} +0 -0
- data/spec/fixtures/jf841ys4828/temp/{ANDHRA_PRADESH-iso19139-fc.xml → ANDHRA_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/jh802mp2160/temp/{DELHI_PT-iso19139-fc.xml → DELHI_PT-iso19110.xml} +0 -0
- data/spec/fixtures/jj806fc3801/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/jq835yn7161/temp/{HIMACHAL_PRADESH-iso19139-fc.xml → HIMACHAL_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/jr455pt6676/temp/{TAMILNADU-iso19139-fc.xml → TAMILNADU-iso19110.xml} +0 -0
- data/spec/fixtures/js637zp2537/temp/{BIHAR-iso19139-fc.xml → BIHAR-iso19110.xml} +0 -0
- data/spec/fixtures/jv502wg9611/temp/{GOA-iso19139-fc.xml → GOA-iso19110.xml} +0 -0
- data/spec/fixtures/jw462ck6560/temp/{JAMMU_KASHMIR-iso19139-fc.xml → JAMMU_KASHMIR-iso19110.xml} +0 -0
- data/spec/fixtures/kj800fb6273/temp/{STATE2011-iso19139-fc.xml → STATE2011-iso19110.xml} +0 -0
- data/spec/fixtures/km504zq3948/temp/{HIMACHAL_PRADESH-iso19139-fc.xml → HIMACHAL_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/ks297fy1411/temp/{OFFSH_BLOCKS-iso19139-fc.xml → OFFSH_BLOCKS-iso19110.xml} +0 -0
- data/spec/fixtures/md358hy5049/temp/{MIZORAM-iso19139-fc.xml → MIZORAM-iso19110.xml} +0 -0
- data/spec/fixtures/mg745bq0193/temp/{MADHYA_PRADESH-iso19139-fc.xml → MADHYA_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/mh187yx3536/temp/{WEST_BENGAL-iso19139-fc.xml → WEST_BENGAL-iso19110.xml} +0 -0
- data/spec/fixtures/mk488yn6694/temp/{GUJARAT-iso19139-fc.xml → GUJARAT-iso19110.xml} +0 -0
- data/spec/fixtures/my216kp3008/temp/{DELHI-iso19139-fc.xml → DELHI-iso19110.xml} +0 -0
- data/spec/fixtures/my504nz9827/temp/{JAMMU_KASHMIR-iso19139-fc.xml → JAMMU_KASHMIR-iso19110.xml} +0 -0
- data/spec/fixtures/ng819jm8700/temp/{MANIPUR-iso19139-fc.xml → MANIPUR-iso19110.xml} +0 -0
- data/spec/fixtures/np020jq2139/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/ns377mt1608/temp/{STATE1991-iso19139-fc.xml → STATE1991-iso19110.xml} +0 -0
- data/spec/fixtures/nw926np8508/temp/{metadata.iso19139-fc.xml → metadata.iso19110.xml} +0 -0
- data/spec/fixtures/ny358rm8559/temp/{TRIPURA-iso19139-fc.xml → TRIPURA-iso19110.xml} +0 -0
- data/spec/fixtures/nz176rm8192/temp/{DISTRICT2011-iso19139-fc.xml → DISTRICT2011-iso19110.xml} +0 -0
- data/spec/fixtures/nz252rq2252/temp/{UTTAR_PRADESH-iso19139-fc.xml → UTTAR_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/pd902kb3348/temp/{MADHYA_PRADESH-iso19139-fc.xml → MADHYA_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/pz792fz1776/temp/{MAHARASHTRA-iso19139-fc.xml → MAHARASHTRA-iso19110.xml} +0 -0
- data/spec/fixtures/qb767ss4042/temp/{UTTAR_PRADESH-iso19139-fc.xml → UTTAR_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/qc091qw0570/temp/{GUJARAT-iso19139-fc.xml → GUJARAT-iso19110.xml} +0 -0
- data/spec/fixtures/qc652vr7204/temp/{ANDHRA_PRADESH_PT-iso19139-fc.xml → ANDHRA_PRADESH_PT-iso19110.xml} +0 -0
- data/spec/fixtures/qk786js7484/temp/{DISTRICT1961-iso19139-fc.xml → DISTRICT1961-iso19110.xml} +0 -0
- data/spec/fixtures/qn676pg6767/temp/{GOA-iso19139-fc.xml → GOA-iso19110.xml} +0 -0
- data/spec/fixtures/qr255jh4074/temp/{LOKSABHA_14-iso19139-fc.xml → LOKSABHA_14-iso19110.xml} +0 -0
- data/spec/fixtures/qr374kj4827/temp/{ASSAM-iso19139-fc.xml → ASSAM-iso19110.xml} +0 -0
- data/spec/fixtures/qy162js1748/temp/{CHHATTISGARH-iso19139-fc.xml → CHHATTISGARH-iso19110.xml} +0 -0
- data/spec/fixtures/rd446vf2633/temp/{NAGALAND-iso19139-fc.xml → NAGALAND-iso19110.xml} +0 -0
- data/spec/fixtures/rf389hf2983/temp/{CHHATTISGARH_PT-iso19139-fc.xml → CHHATTISGARH_PT-iso19110.xml} +0 -0
- data/spec/fixtures/rf859ff4582/temp/{JHARKHAND-iso19139-fc.xml → JHARKHAND-iso19110.xml} +0 -0
- data/spec/fixtures/rh343ds8931/temp/{BIHAR-iso19139-fc.xml → BIHAR-iso19110.xml} +0 -0
- data/spec/fixtures/rn815xk8157/temp/{SIKKIM-iso19139-fc.xml → SIKKIM-iso19110.xml} +0 -0
- data/spec/fixtures/rq653sz4470/temp/{CHHATTISGARH-iso19139-fc.xml → CHHATTISGARH-iso19110.xml} +0 -0
- data/spec/fixtures/rt625ws6022/temp/{GULF_FAIRWAYS-iso19139-fc.xml → GULF_FAIRWAYS-iso19110.xml} +0 -0
- data/spec/fixtures/sc330vf4259/temp/{JHARKHAND-iso19139-fc.xml → JHARKHAND-iso19110.xml} +0 -0
- data/spec/fixtures/sq479mx3086/temp/{OFFSH_PLATF-iso19139-fc.xml → OFFSH_PLATF-iso19110.xml} +0 -0
- data/spec/fixtures/sr686bm4098/temp/{DAMAN_DIU_PT-iso19139-fc.xml → DAMAN_DIU_PT-iso19110.xml} +0 -0
- data/spec/fixtures/sv303sh5583/temp/{ARUNACHAL_PRADESH-iso19139-fc.xml → ARUNACHAL_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/sy319nh8520/temp/{GUJARAT-iso19139-fc.xml → GUJARAT-iso19110.xml} +0 -0
- data/spec/fixtures/td363vx2792/temp/{HIMACHAL_PRADESH_PT-iso19139-fc.xml → HIMACHAL_PRADESH_PT-iso19110.xml} +0 -0
- data/spec/fixtures/tf374bd2484/temp/{DISTRICT1951-iso19139-fc.xml → DISTRICT1951-iso19110.xml} +0 -0
- data/spec/fixtures/tj797mj7877/temp/{LOKSABHA_15-iso19139-fc.xml → LOKSABHA_15-iso19110.xml} +0 -0
- data/spec/fixtures/tv060wq5179/temp/{ASSAM-iso19139-fc.xml → ASSAM-iso19110.xml} +0 -0
- data/spec/fixtures/tv536bn1915/temp/{ARUNACHAL_PRADESH-iso19139-fc.xml → ARUNACHAL_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/tz359cc2977/temp/{MANIPUR-iso19139-fc.xml → MANIPUR-iso19110.xml} +0 -0
- data/spec/fixtures/vb525my6511/temp/{UTTARAKHAND-iso19139-fc.xml → UTTARAKHAND-iso19110.xml} +0 -0
- data/spec/fixtures/vh802fs4240/temp/{PONDICHERRY-iso19139-fc.xml → PONDICHERRY-iso19110.xml} +0 -0
- data/spec/fixtures/vk120xn2474/temp/{PLSS_SEC-iso19139-fc.xml → PLSS_SEC-iso19110.xml} +0 -0
- data/spec/fixtures/vn439bc7316/temp/{KERALA-iso19139-fc.xml → KERALA-iso19110.xml} +0 -0
- data/spec/fixtures/vq745jk0695/temp/{MEGHALAYA-iso19139-fc.xml → MEGHALAYA-iso19110.xml} +0 -0
- data/spec/fixtures/vr593vj7147/temp/{ANDHRA_PRADESH-iso19139-fc.xml → ANDHRA_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/vw911qb5271/temp/{DISTRICT2001-iso19139-fc.xml → DISTRICT2001-iso19110.xml} +0 -0
- data/spec/fixtures/wg680pz0365/temp/{ANDHRA_PRADESH-iso19139-fc.xml → ANDHRA_PRADESH-iso19110.xml} +0 -0
- data/spec/fixtures/wg761xn1926/temp/{HARYANA-iso19139-fc.xml → HARYANA-iso19110.xml} +0 -0
- data/spec/fixtures/wh870qw1934/temp/{PUNJAB-iso19139-fc.xml → PUNJAB-iso19110.xml} +0 -0
- data/spec/fixtures/wk775mm4673/temp/{MAHARASHTRA-iso19139-fc.xml → MAHARASHTRA-iso19110.xml} +0 -0
- data/spec/fixtures/ws171yz2165/temp/{ARUNACHAL_PRADESH_PT-iso19139-fc.xml → ARUNACHAL_PRADESH_PT-iso19110.xml} +0 -0
- data/spec/fixtures/wt473hz7153/temp/{CHANDIGARH_PT-iso19139-fc.xml → CHANDIGARH_PT-iso19110.xml} +0 -0
- data/spec/fixtures/ww217dj0457/temp/{CO2_PIPE-iso19139-fc.xml → CO2_PIPE-iso19110.xml} +0 -0
- data/spec/fixtures/wy875pk9849/temp/{STATE1961-iso19139-fc.xml → STATE1961-iso19110.xml} +0 -0
- data/spec/fixtures/xb018tk2042/temp/{STATE1981-iso19139-fc.xml → STATE1981-iso19110.xml} +0 -0
- data/spec/fixtures/xg539vw8586/temp/{ORISSA-iso19139-fc.xml → ORISSA-iso19110.xml} +0 -0
- data/spec/fixtures/xv475kp4644/temp/{ASSAM_PT-iso19139-fc.xml → ASSAM_PT-iso19110.xml} +0 -0
- data/spec/fixtures/xy096gc2959/temp/{GOA-iso19139-fc.xml → GOA-iso19110.xml} +0 -0
- data/spec/fixtures/xz518gz3362/temp/{UTTARAKHAND-iso19139-fc.xml → UTTARAKHAND-iso19110.xml} +0 -0
- data/spec/fixtures/yh986wy4737/temp/{NAGALAND-iso19139-fc.xml → NAGALAND-iso19110.xml} +0 -0
- data/spec/fixtures/yn187fq4474/temp/{KARNATAKA-iso19139-fc.xml → KARNATAKA-iso19110.xml} +0 -0
- data/spec/fixtures/yn236mw3250/temp/{TAMILNADU-iso19139-fc.xml → TAMILNADU-iso19110.xml} +0 -0
- data/spec/fixtures/yz596nz0112/temp/{WEST_BENGAL-iso19139-fc.xml → WEST_BENGAL-iso19110.xml} +0 -0
- data/spec/fixtures/zk596gy7380/temp/{DISTRICT1971-iso19139-fc.xml → DISTRICT1971-iso19110.xml} +0 -0
- data/spec/fixtures/zn452hh7431/temp/{RAJASTHAN-iso19139-fc.xml → RAJASTHAN-iso19110.xml} +0 -0
- data/spec/fixtures/zt093fw6519/temp/{MIZORAM-iso19139-fc.xml → MIZORAM-iso19110.xml} +0 -0
- data/spec/fixtures/zv925hd6723/temp/{OGWELLS-iso19139-fc.xml → OGWELLS-iso19110.xml} +0 -0
- data/spec/fixtures/zy658cr1728/temp/{ANDAMAAN_NICOBAR_PT-iso19139-fc.xml → ANDAMAAN_NICOBAR_PT-iso19110.xml} +0 -0
- data/spec/fixtures/zz943vx1492/temp/{BASINS-iso19139-fc.xml → BASINS-iso19110.xml} +0 -0
- data/spec/unit/gazetteer_spec.rb +100 -35
- data/spec/unit/task_spec.rb +68 -0
- data/spec/unit/transform_spec.rb +1 -1
- data/spec/unit/utils_spec.rb +17 -3
- data/workflow.rb +35 -0
- metadata +323 -316
@@ -0,0 +1,109 @@
|
|
1
|
+
module GeoHydra
|
2
|
+
module GisAssemblyWF
|
3
|
+
|
4
|
+
class RegisterDruidTask < ManualTask; end
|
5
|
+
|
6
|
+
class AuthorMetadataTask < ManualTask; end
|
7
|
+
|
8
|
+
class ApproveMetadataTask < ManualTask; end
|
9
|
+
|
10
|
+
class WrangleDataTask < ManualTask; end
|
11
|
+
|
12
|
+
class ApproveDataTask < ManualTask; end
|
13
|
+
|
14
|
+
class NormalizeDataTask < Task
|
15
|
+
def perform(data)
|
16
|
+
raise NotImplementedError
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
class PackageDataTask
|
21
|
+
def perform(data)
|
22
|
+
raise NotImplementedError
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class FinishDataTask < NoopTask; end
|
27
|
+
|
28
|
+
class ExtractThumbnailTask < Task
|
29
|
+
def perform(data)
|
30
|
+
raise NotImplementedError
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
class ExtractIso19139Task < Task
|
35
|
+
def perform(data)
|
36
|
+
raise NotImplementedError
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class GenerateGeoMetadataTask < Task
|
41
|
+
def perform(data)
|
42
|
+
initialize(data)
|
43
|
+
isoFn = File.join(druid.temp_dir, 'iso19139.xml')
|
44
|
+
fcFn = File.join(druid.temp_dir, 'iso19110.xml')
|
45
|
+
geoFn = File.join(druid.metadata_dir, 'geoMetadata.xml')
|
46
|
+
log_debug({:isoFn => isoFn, :fcFn => fcFn, :geoFn => geoFn, :flags => flags})
|
47
|
+
|
48
|
+
unless FileUtils.uptodate?(geoFn, [isoFn, fcFn])
|
49
|
+
isoXml = Nokogiri::XML(File.read(isoFn))
|
50
|
+
fcXml = Nokogiri::XML(File.read(fcFn))
|
51
|
+
log_verbose("Generating geoMetadata: #{geoFn} <- #{isoFn}, #{fcFn}")
|
52
|
+
xml = GeoHydra::Transform.to_geoMetadataDS(isoXml, fcXml, { 'purl' => to_purl})
|
53
|
+
File.open(geoFn, 'w') {|f| f << xml.to_xml(:indent => 2) }
|
54
|
+
end
|
55
|
+
'SUCCESS'
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
class GenerateModsTask < Task
|
60
|
+
def perform(data)
|
61
|
+
initialize(data)
|
62
|
+
geoFn = File.join(druid.metadata_dir, 'geoMetadata.xml')
|
63
|
+
modsFn = File.join(druid.metadata_dir, 'descMetadata.xml')
|
64
|
+
log_debug({:geoFn => geoFn, :modsFn => modsFn, :flags => flags})
|
65
|
+
|
66
|
+
unless FileUtils.uptodate?(modsFn, [geoFn])
|
67
|
+
# MODS from GeoMetadataDS
|
68
|
+
geoMetadata = Dor::GeoMetadataDS.from_xml File.read(geoFn)
|
69
|
+
geoMetadata.geometryType = data[:geometryType] || 'Polygon'
|
70
|
+
geoMetadata.zipName = 'data.zip'
|
71
|
+
geoMetadata.purl = to_purl
|
72
|
+
|
73
|
+
log_debug({:geoMetadata => geoMetadata.ng_xml})
|
74
|
+
File.open(modsFn, 'w') { |f| f << geoMetadata.to_mods.to_xml(:index => 2) }
|
75
|
+
end
|
76
|
+
'SUCCESS'
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
class AssignPlacenamesTask < Task
|
81
|
+
def perform(data)
|
82
|
+
raise NotImplementedError
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
class FinishMetadataTask < NoopTask; end
|
87
|
+
|
88
|
+
class GenerateContentMetadataTask < Task
|
89
|
+
def perform(data)
|
90
|
+
raise NotImplementedError
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
class FinishGisAssemblyPipeline < NoopTask; end
|
95
|
+
|
96
|
+
class StartAssemblyWorkflowTask < Task
|
97
|
+
def perform(data)
|
98
|
+
raise NotImplementedError
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class StartDeliveryWorkflowTask < Task
|
103
|
+
def perform(data)
|
104
|
+
raise NotImplementedError
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<workflow-def id="gisAssemblyWF" repository="dor">
|
3
|
+
<process name="start-gis-assembly-pipeline" sequence="1" status="completed">
|
4
|
+
<label>Initiate assembly pipeline for the object</label>
|
5
|
+
</process>
|
6
|
+
<process name="register-druid" sequence="2">
|
7
|
+
<prereq>start-gis-assembly-pipeline</prereq>
|
8
|
+
<label>Ensure proper registration of druid, source ID, and label</label>
|
9
|
+
</process>
|
10
|
+
<process name="author-metadata" sequence="3">
|
11
|
+
<prereq>register-druid</prereq>
|
12
|
+
<label>Author metadata using ArcCatalog</label>
|
13
|
+
</process>
|
14
|
+
<process name="approve-metadata" sequence="4">
|
15
|
+
<prereq>author-metadata</prereq>
|
16
|
+
<label>Approve metadata quality and release for workflow pipeline</label>
|
17
|
+
</process>
|
18
|
+
<process name="wrangle-data" sequence="5">
|
19
|
+
<prereq>register-druid</prereq>
|
20
|
+
<prereq>approve-metadata</prereq><!-- XXX: force linear -->
|
21
|
+
<label>Wrangle the data into the digital work</label>
|
22
|
+
</process>
|
23
|
+
<process name="approve-data" sequence="6">
|
24
|
+
<prereq>wrangle-data</prereq>
|
25
|
+
<label>Approve data quality for digital work and release for workflow pipeline</label>
|
26
|
+
</process>
|
27
|
+
<process name="normalize-data" sequence="7">
|
28
|
+
<prereq>approve-data</prereq>
|
29
|
+
<label>Reproject the data into common SRS projection and/or file format</label>
|
30
|
+
</process>
|
31
|
+
<process name="package-data" sequence="8">
|
32
|
+
<prereq>normalize-data</prereq>
|
33
|
+
<label>Package the digital work and derivative data files</label>
|
34
|
+
</process>
|
35
|
+
<process name="finish-data" sequence="9">
|
36
|
+
<prereq>package-data</prereq>
|
37
|
+
<label>Finalize the data preparation</label>
|
38
|
+
</process>
|
39
|
+
<process name="extract-thumbnail" sequence="10">
|
40
|
+
<prereq>approve-metadata</prereq>
|
41
|
+
<prereq>finish-data</prereq><!-- XXX: force linear -->
|
42
|
+
<label>Extract thumbnail preview</label>
|
43
|
+
</process>
|
44
|
+
<process name="extract-iso19139" sequence="11">
|
45
|
+
<prereq>approve-metadata</prereq>
|
46
|
+
<prereq>finish-data</prereq> <!-- requires shapefile -->
|
47
|
+
<prereq>extract-thumbnail</prereq><!-- XXX: force linear -->
|
48
|
+
<label>Extract ISO 19139 metadata from ArcCatalog</label>
|
49
|
+
</process>
|
50
|
+
<process name="generate-geo-metadata" sequence="12">
|
51
|
+
<prereq>extract-iso19139</prereq>
|
52
|
+
<label>Convert ISO 19139 metadata into geoMetadata</label>
|
53
|
+
</process>
|
54
|
+
<process name="generate-mods" sequence="13">
|
55
|
+
<prereq>generate-geoMetadata</prereq>
|
56
|
+
<label>Convert geoMetadata into MODS</label>
|
57
|
+
</process>
|
58
|
+
<process name="assign-placenames" sequence="14">
|
59
|
+
<prereq>generate-mods</prereq>
|
60
|
+
<label>Insert linked data into MODS record from gazetteer</label>
|
61
|
+
</process>
|
62
|
+
<process name="finish-metadata" sequence="15">
|
63
|
+
<prereq>extract-thumbnail</prereq>
|
64
|
+
<prereq>assign-placenames</prereq>
|
65
|
+
<label>Finalize the data preparation</label>
|
66
|
+
</process>
|
67
|
+
<process name="generate-content-metadata" sequence="16">
|
68
|
+
<prereq>finish-metadata</prereq>
|
69
|
+
<prereq>finish-data</prereq>
|
70
|
+
<label>Generate contentMetadata manifest</label>
|
71
|
+
</process>
|
72
|
+
<process name="finish-gis-assembly-pipeline" sequence="17">
|
73
|
+
<prereq>generate-content-metadata</prereq>
|
74
|
+
<label>Finalize assembly pipeline to prepare for assembly/delivery/discovery</label>
|
75
|
+
</process>
|
76
|
+
<process name="start-assembly-workflow" sequence="18">
|
77
|
+
<prereq>finish-gis-assembly-pipeline</prereq>
|
78
|
+
<label>Kickstart the core assembly workflow at assemblyWF</label>
|
79
|
+
</process>
|
80
|
+
<process name="start-delivery-workflow" sequence="19">
|
81
|
+
<prereq>finish-gis-assembly-pipeline</prereq>
|
82
|
+
<prereq>start-assembly-workflow</prereq><!-- XXX: force linear -->
|
83
|
+
<label>Kickstart the GIS delivery workflow at gisDeliveryWF</label>
|
84
|
+
</process>
|
85
|
+
</workflow-def>
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module GeoHydra
|
2
|
+
module GisDeliveryWF
|
3
|
+
|
4
|
+
class StartGisDeliveryPipelineTask < NoopTask; end
|
5
|
+
|
6
|
+
class LoadVectorTask < Task
|
7
|
+
|
8
|
+
end
|
9
|
+
|
10
|
+
class LoadRasterTask < Task
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
class LoadGeoserverTask < Task
|
15
|
+
|
16
|
+
end
|
17
|
+
|
18
|
+
class LoadGeowebcacheTask < Task
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
class SeedGeowebcacheTask < Task
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
class FinishGisDeliveryPipelineTask < NoopTask; end
|
27
|
+
|
28
|
+
class StartGisDiscoveryWorkflowTask < Task
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<workflow-def id="gisDeliveryWF" repository="dor">
|
3
|
+
<process name="start-gis-delivery-pipeline" sequence="1" status="completed">
|
4
|
+
<label>Initiate delivery pipeline for the object</label>
|
5
|
+
</process>
|
6
|
+
<process name="load-vector" sequence="2">
|
7
|
+
<prereq>start-gis-delivery-pipeline</prereq>
|
8
|
+
<label>Load vector data into PostGIS database</label>
|
9
|
+
</process>
|
10
|
+
<process name="load-raster" sequence="3">
|
11
|
+
<prereq>start-delivery-pipeline</prereq>
|
12
|
+
<prereq>load-vector</prereq><!-- XXX: force linear -->
|
13
|
+
<label>Load raster into GeoTIFF data store</label>
|
14
|
+
</process>
|
15
|
+
<process name="load-geoserver" sequence="4">
|
16
|
+
<prereq>load-vector</prereq>
|
17
|
+
<prereq>load-raster</prereq>
|
18
|
+
<label>Load layers into GeoServer</label>
|
19
|
+
</process>
|
20
|
+
<process name="load-geowebcache" sequence="5">
|
21
|
+
<prereq>load-geoserver</prereq>
|
22
|
+
<label>Load layers into GeoWebCache</label>
|
23
|
+
</process>
|
24
|
+
<process name="seed-geowebcache" sequence="6">
|
25
|
+
<prereq>load-geowebcache</prereq>
|
26
|
+
<label>Generate tiles for GeoWebCache</label>
|
27
|
+
</process>
|
28
|
+
<process name="finish-gis-delivery-pipeline" sequence="7">
|
29
|
+
<prereq>seed-geowebcache</prereq>
|
30
|
+
<label>Finalize delivery pipeline for the object</label>
|
31
|
+
</process>
|
32
|
+
<process name="start-gis-discovery-workflow" sequence="8">
|
33
|
+
<prereq>finish-gis-delivery-pipeline</prereq>
|
34
|
+
<label>Kickstart GIS discovery workflow with gisDiscoveryWF</label>
|
35
|
+
</process>
|
36
|
+
</workflow-def>
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module GeoHydra
|
2
|
+
module GisDiscoveryWF
|
3
|
+
class GenerateSolr < Task
|
4
|
+
def perform_solr(app)
|
5
|
+
modsFn = File.join(druid.metadata_dir, 'descMetadata.xml')
|
6
|
+
solrFn = File.join(druid.temp_dir, "#{app}Solr.xml")
|
7
|
+
log_debug({:modsFn => modsFn, :solrFn => solrFn, :flags => flags})
|
8
|
+
|
9
|
+
unless FileUtils.uptodate?(solrFn, [modsFn])
|
10
|
+
# Solr document from descMetadataDS
|
11
|
+
cmd = [ 'xsltproc',
|
12
|
+
"--stringparam geoserver_root '#{flags[:geoserver]}'",
|
13
|
+
"--stringparam purl '#{to_purl}'",
|
14
|
+
"--output '#{solrFn}'",
|
15
|
+
"'#{File.expand_path(File.dirname(__FILE__) + '/../mods2#{app}.xsl')}'",
|
16
|
+
"'#{modsFn}'"
|
17
|
+
].join(' ')
|
18
|
+
log_debug({:cmd => cmd})
|
19
|
+
system(cmd)
|
20
|
+
end
|
21
|
+
'SUCCESS'
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
|
26
|
+
class StartGisDiscoveryPipelineTask < NoopTask; end
|
27
|
+
|
28
|
+
class GenerateOgpTask < GenerateSolr
|
29
|
+
def perform(data)
|
30
|
+
initialize(data)
|
31
|
+
perform_solr('ogp')
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
class LoadOgpTask < Task
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
class GenerateGeosearchTask < Task
|
41
|
+
def perform(data)
|
42
|
+
initialize(data)
|
43
|
+
perform_solr('geoblacklight')
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class LoadGeosearchTask < Task
|
48
|
+
|
49
|
+
end
|
50
|
+
|
51
|
+
class FinishGisDiscoveryPipeline < Task
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
<?xml version="1.0"?>
|
2
|
+
<workflow-def id="gisDiscoveryWF" repository="dor">
|
3
|
+
<process name="start-gis-discovery-pipeline" sequence="1" status="completed">
|
4
|
+
<label>Initiate GIS discovery pipeline for the object</label>
|
5
|
+
</process>
|
6
|
+
<process name="generate-ogp" sequence="2">
|
7
|
+
<prereq>start-gis-discovery-pipeline</prereq>
|
8
|
+
<label>Generate Solr document for OpenGeoPortal</label>
|
9
|
+
</process>
|
10
|
+
<process name="load-ogp" sequence="3">
|
11
|
+
<prereq>generate-ogp</prereq>
|
12
|
+
<label>Load Solr document into OpenGeoPortal</label>
|
13
|
+
</process>
|
14
|
+
<process name="generate-geosearch" sequence="4">
|
15
|
+
<prereq>start-gis-discovery-pipeline</prereq>
|
16
|
+
<prereq>load-ogp</prereq><!-- XXX: force linear -->
|
17
|
+
<label>Generate Solr document for GeoSearch</label>
|
18
|
+
</process>
|
19
|
+
<process name="load-geosearch" sequence="5">
|
20
|
+
<prereq>generate-geosearch</prereq>
|
21
|
+
<label>Load Solr document into GeoSearch</label>
|
22
|
+
</process>
|
23
|
+
<process name="finish-gis-discovery-pipeline" sequence="6">
|
24
|
+
<prereq>load-ogp</prereq>
|
25
|
+
<prereq>load-geosearch</prereq>
|
26
|
+
<label>Finalize GIS discovery pipeline for the object</label>
|
27
|
+
</process>
|
28
|
+
</workflow-def>
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'druid-tools'
|
2
|
+
|
3
|
+
module GeoHydra
|
4
|
+
class Task
|
5
|
+
STATUS = %w{HOLD READY RUNNING DEFERRED ERROR COMPLETED}
|
6
|
+
# @param [String] status
|
7
|
+
# @return [Boolean] true if the given status is one of the valid Task.STATUS values
|
8
|
+
def valid_status? status
|
9
|
+
STATUS.include? status.upcase
|
10
|
+
end
|
11
|
+
|
12
|
+
attr_accessor :druid, :flags
|
13
|
+
|
14
|
+
def initialize(data = {})
|
15
|
+
@flags = {}
|
16
|
+
@flags[:debug] = data[:debug] || false
|
17
|
+
@flags[:geoserver] = data[:geoserver] || GeoHydra::Config.ogp.geoserver || 'http://127.0.0.1:8080/geoserver'
|
18
|
+
@flags[:purl] = data[:purl] || GeoHydra::Config.ogp.purl || 'http://purl.stanford.edu'
|
19
|
+
@flags[:solr] = data[:solr] || GeoHydra::Config.ogp.solr || 'http://127.0.0.1:8983/solr'
|
20
|
+
@flags[:stagedir] = data[:stagedir] || GeoHydra::Config.geohydra.stage || 'stage'
|
21
|
+
@flags[:tmpdir] = data[:tmpdir] || GeoHydra::Config.geohydra.tmpdir || 'tmp'
|
22
|
+
@flags[:verbose] = data[:verbose] || false
|
23
|
+
@flags[:workspacedir] = data[:workspacedir] || GeoHydra::Config.geohydra.workspace || 'workspace'
|
24
|
+
|
25
|
+
@druid = nil
|
26
|
+
@druid = _setup_druid(data[:druid]) unless data[:druid].nil?
|
27
|
+
end
|
28
|
+
|
29
|
+
# Perform the task
|
30
|
+
# @param [Hash] data
|
31
|
+
# @return [String] one of the `Task.STATUS` values
|
32
|
+
def perform(data = {})
|
33
|
+
raise NotImplementedError, 'abstract method'
|
34
|
+
end
|
35
|
+
|
36
|
+
# @param [String] druid `aa111bb2222`
|
37
|
+
def druid= druid
|
38
|
+
_init_druid druid
|
39
|
+
end
|
40
|
+
|
41
|
+
# Converts the current @druid into a PURL
|
42
|
+
# @return [String] `http://purl.stanford.edu/aa111bb2222`
|
43
|
+
def to_purl
|
44
|
+
File.join(flags[:purl], @druid.id)
|
45
|
+
end
|
46
|
+
|
47
|
+
def log_info(args)
|
48
|
+
puts args
|
49
|
+
end
|
50
|
+
|
51
|
+
def log_verbose(args)
|
52
|
+
ap({:verbose => args}) if flags[:verbose]
|
53
|
+
end
|
54
|
+
|
55
|
+
def log_debug(args)
|
56
|
+
ap({:debug => args}) if flags[:debug]
|
57
|
+
end
|
58
|
+
|
59
|
+
def log_error(args)
|
60
|
+
ap({:error => args})
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
def _setup_druid s
|
65
|
+
druid = DruidTools::Druid.new(s, flags[:workspacedir])
|
66
|
+
raise ArgumentError if druid.nil? or not DruidTools::Druid.valid?(druid.druid)
|
67
|
+
druid
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
class NoopTask < Task
|
72
|
+
def perform(data)
|
73
|
+
'COMPLETED'
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
class ManualTask < Task
|
78
|
+
def perform(data)
|
79
|
+
'HOLD'
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/ogp/README.md
ADDED
@@ -0,0 +1,350 @@
|
|
1
|
+
{toc:maxLevel=2}
|
2
|
+
|
3
|
+
h1. Solr schema
|
4
|
+
|
5
|
+
The schema XML is on Github here: https://github.com/sul-dlss/geohydra/blob/master/solr/kurma-app-test/conf/schema.xml.
|
6
|
+
|
7
|
+
h2. Primary key
|
8
|
+
|
9
|
+
* *uuid*: Unique Identifier. Examples:
|
10
|
+
** [http://purl.stanford.edu/vr593vj7147],
|
11
|
+
** [http://ark.cdlib.org/ark:/28722/bk0012h535q],
|
12
|
+
** urn:geodata.tufts.edu:Tufts.CambridgeGrid100_04.
|
13
|
+
|
14
|
+
h2. Dublin Core
|
15
|
+
|
16
|
+
{note}
|
17
|
+
See the [Dublin Core Elements Guide|http://dublincore.org/documents/dcmi-terms/] for semantic descriptions of all of these fields. We're using both DC Elements and DC Terms
|
18
|
+
{note}
|
19
|
+
|
20
|
+
* *dct_spatial_sm*: Coverage, placenames. Multiple values allowed. Example: "Paris, France".
|
21
|
+
* *dct_temporal_sm*: Coverage, years. Multiple values allowed. Example: "2010".
|
22
|
+
* *dc_creator_sm*: Author(s). Example: "Washington, George".
|
23
|
+
* *dct_issued_dt*: Date in Solr syntax. Example: "2001-01-01T00:00:00Z".
|
24
|
+
* *dc_description_s*: Description.
|
25
|
+
* *dc_format_s*: File format (not MIME types). Valid values:
|
26
|
+
** "Shapefile"
|
27
|
+
** "GeoTIFF"
|
28
|
+
* *dc_identifier_s*: Unique identifier. Same as UUID.
|
29
|
+
* *dc_language_s*: Language. Example: "English".
|
30
|
+
* *dc_publisher_s*: Publisher. Example: "ML InfoMap (Firm)".
|
31
|
+
* *dct_references_sm*: URLs to referenced resources. Used scheme and url parameters. scheme values are based on [CatInterop|https://github.com/OSGeo/Cat-Interop/blob/master/link_types.csv] Multiple values allowed. Example:
|
32
|
+
scheme="urn:ogc:serviceType:WebFeatureService" url="http://geowebservices-restricted.stanford.edu/geoserver
|
33
|
+
/wfs"
|
34
|
+
* *dc_rights_s*: Rights for access. Valid values:
|
35
|
+
** "Restricted"
|
36
|
+
** "Public"
|
37
|
+
* *dct_provenance_s*: Source institution: Examples:
|
38
|
+
** Berkeley
|
39
|
+
** Harvard
|
40
|
+
** MassGIS
|
41
|
+
** MIT
|
42
|
+
** Stanford
|
43
|
+
** Tufts
|
44
|
+
* *dc_subject_sm*: Subject. Multiple values allowed. Example: "Human settlements", "Census".
|
45
|
+
* *dc_title_s*: Title.
|
46
|
+
* *dc_type_s*: Resource type. dc:type=Dataset for georectified images, dc:type=Image for digitaized, non-georectified images, or dc:type=PhysicalObject for paper maps (no digitization).
|
47
|
+
* *dct_isPartOf_sm*: Collection to which the layer belongs.
|
48
|
+
|
49
|
+
h2. GeoRSS metadata
|
50
|
+
|
51
|
+
* *georss_point_s*: Point representation for layer -- i.e., centroid?
|
52
|
+
* *georss_box_s*: Bounding box as maximum values for S W N E. Example: "12.62309 76.76 19.91705 84.76618"
|
53
|
+
* *georss_polygon_s*: Shape of the layer as a Polygon.
|
54
|
+
Example: "n w n e s e s w n w"
|
55
|
+
|
56
|
+
h2. Layer-specific metadata
|
57
|
+
|
58
|
+
* *layer_slug_s*. Unique identifier visible to the user, used for Permalinks.
|
59
|
+
* Example: stanford-vr593vj7147.
|
60
|
+
* *layer_id_s*. The complete identifier for the WMS/WFS/WCS layer.
|
61
|
+
Example: "druid:vr593vj7147",
|
62
|
+
* *layer_geom_type_s*. Valid values are: "Point", "Line", "Polygon", and "Raster".
|
63
|
+
|
64
|
+
h2. Derived metadata used by Solr index
|
65
|
+
|
66
|
+
* *solr_bbox*: Bounding box as maximum values for W S E N. Example: "76.76 12.62309 84.76618 19.91705"
|
67
|
+
* *solr_geom*: Shape of the layer as a Point, LineString, or Polygon WKT.
|
68
|
+
Example: "POLYGON((76.76 19.91705, 84.76618 19.91705, 84.76618 12.62309, 76.76 12.62309, 76.76 19.91705))"
|
69
|
+
* *solr_ne_pt* (from solr_bbox). North-eastern most point of the bounding box, as (y, x). Example: "83.1,-128.5"
|
70
|
+
* *solr_sw_pt* (from solr_bbox). South-western most point of the bounding box, as (y, x). Example: "81.2,-130.1"
|
71
|
+
* *solr_year_i* (from dc_coverage_temporal_sm): Year for which layer is valid. Example: 2012.
|
72
|
+
|
73
|
+
h2. Solr schema syntax
|
74
|
+
|
75
|
+
See complete schema on https://github.com/sul-dlss/geomdtk/blob/master/solr/kurma-app-test/conf/schema.xml
|
76
|
+
|
77
|
+
Note on the types:
|
78
|
+
|
79
|
+
|| Suffix || Solr data type using dynamicField ||
|
80
|
+
| \_s | String |
|
81
|
+
| \_sm | String, multivalued |
|
82
|
+
| \_t | Text, English |
|
83
|
+
| \_i | Integer |
|
84
|
+
| \_dt | Date time |
|
85
|
+
| \_url | URL as a non-indexed String |
|
86
|
+
| \_bbox | Spatial bounding box, Rectangle as (w, s, e, n) |
|
87
|
+
| \_pt | Spatial point as (y,x) |
|
88
|
+
| \_geom | Spatial shape as WKT |
|
89
|
+
|
90
|
+
|
91
|
+
{code:xml}
|
92
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
93
|
+
<schema name="GeoBlacklight" version="1.5">
|
94
|
+
<uniqueKey>uuid</uniqueKey>
|
95
|
+
<fields>
|
96
|
+
...
|
97
|
+
<!-- Spatial field types:
|
98
|
+
|
99
|
+
Solr3:
|
100
|
+
<field name="my_pt">83.1,-117.312</field>
|
101
|
+
as (y,x)
|
102
|
+
|
103
|
+
Solr4:
|
104
|
+
|
105
|
+
<field name="my_bbox">-117.312 83.1 -115.39 84.31</field>
|
106
|
+
as (W S E N)
|
107
|
+
|
108
|
+
<field name="my_geom">POLYGON((1 8, 1 9, 2 9, 2 8, 1 8))</field>
|
109
|
+
as WKT for point, linestring, polygon
|
110
|
+
|
111
|
+
-->
|
112
|
+
<dynamicField name="*_pt" type="location" stored="true" indexed="true"/>
|
113
|
+
<dynamicField name="*_bbox" type="location_rpt" stored="true" indexed="true"/>
|
114
|
+
<dynamicField name="*_geom" type="location_jts" stored="true" indexed="true"/>
|
115
|
+
</fields>
|
116
|
+
<types>
|
117
|
+
...
|
118
|
+
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_d"/>
|
119
|
+
<fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
|
120
|
+
distErrPct="0.025"
|
121
|
+
maxDistErr="0.000009"
|
122
|
+
units="degrees"
|
123
|
+
/>
|
124
|
+
<fieldType name="location_jts" class="solr.SpatialRecursivePrefixTreeFieldType"
|
125
|
+
spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory"
|
126
|
+
distErrPct="0.025"
|
127
|
+
maxDistErr="0.000009"
|
128
|
+
units="degrees"
|
129
|
+
/>
|
130
|
+
</types>
|
131
|
+
</schema>
|
132
|
+
{code}
|
133
|
+
|
134
|
+
|
135
|
+
----
|
136
|
+
h1. Solr queries
|
137
|
+
|
138
|
+
* Use the Solr query interface with LatLon data on [sul-solr-a|http://sul-solr-a/solr/#/] to try these using ogp core.
|
139
|
+
* For the polygon or JTS queries use [ogpapp-test|http://localhost:8983/solr/#/] via ssh tunnel to jetty 8983.
|
140
|
+
|
141
|
+
h2. Solr 3: Pseudo-spatial using _solr.LatLon_
|
142
|
+
|
143
|
+
{warning}
|
144
|
+
solr.LatLonType does not correctly work across the international dateline in these queries. _latlon in these examples are assumed to be solr.LatLonType.
|
145
|
+
{warning}
|
146
|
+
|
147
|
+
h3. Search for point within 50 km of N40 W114
|
148
|
+
|
149
|
+
Note: Solr _bbox_ uses circle with radius not rectangles.
|
150
|
+
|
151
|
+
{code:xml}
|
152
|
+
<str name="d">50</str>
|
153
|
+
<str name="q">*:*</str>
|
154
|
+
<str name="sfield">solr_latlon</str>
|
155
|
+
<str name="pt">40,-114</str>
|
156
|
+
<str name="fq">{!geofilt}</str>
|
157
|
+
{code}
|
158
|
+
|
159
|
+
|
160
|
+
h3. Search for single point _within_ a bounding box of SW=40,-120 NE=50,-110
|
161
|
+
|
162
|
+
{code:xml}
|
163
|
+
<str name="q">*:*</str>
|
164
|
+
<str name="fq">solr_latlon:[40,-120 TO 50,-110]</str>
|
165
|
+
{code}
|
166
|
+
|
167
|
+
h3. Search for bounding box _within_ a bounding box of SW=20,-160 NE=70,-70
|
168
|
+
|
169
|
+
{code:xml}
|
170
|
+
<str name="q">*:*</str>
|
171
|
+
<str name="fq">solr_sw_latlon:[20,-160 TO 70,-70] AND solr_ne_latlon:[20,-160 TO 70,-70]</str>
|
172
|
+
{code}
|
173
|
+
|
174
|
+
h2. Solr 4 Spatial -- non JTS
|
175
|
+
|
176
|
+
{warning}
|
177
|
+
_pt and _bbox in these examples are assumed to be solr.SpatialRecursivePrefixTreeFieldType.
|
178
|
+
{warning}
|
179
|
+
|
180
|
+
h3. Search for point _within_ a bounding box of SW=20,-160 NE=70,-70
|
181
|
+
|
182
|
+
{code:xml}
|
183
|
+
<str name="q">*:*</str>
|
184
|
+
<str name="fq">solr_pt:"Intersects(-160 20 -70 70)"</str>
|
185
|
+
{code}
|
186
|
+
|
187
|
+
h3. Search for bounding box _within_ a bounding box of SW=20,-160 NE=70,-70
|
188
|
+
|
189
|
+
{code:xml}
|
190
|
+
<str name="q">*:*</str>
|
191
|
+
<str name="fq">solr_sw_pt:[20,-160 TO 70,-70] AND solr_ne_pt:[20,-160 TO 70,-70]</str>
|
192
|
+
{code}
|
193
|
+
|
194
|
+
|
195
|
+
h3. Solr 4: ... using polygon intersection
|
196
|
+
|
197
|
+
{code:xml}
|
198
|
+
<str name="q">*:*</str>
|
199
|
+
<str name="fq">solr_bbox:"Intersects(-160 20 -70 70)"</str>
|
200
|
+
{code}
|
201
|
+
|
202
|
+
|
203
|
+
h3. Solr 4: ... using polygon containment
|
204
|
+
|
205
|
+
{code:xml}
|
206
|
+
<str name="q">*:*</str>
|
207
|
+
<str name="fq">solr_bbox:"IsWithin(-160 20 -150 30)"</str>
|
208
|
+
{code}
|
209
|
+
|
210
|
+
h3. Solr 4: ... using polygon containment for spatial relevancy
|
211
|
+
|
212
|
+
{code:xml}
|
213
|
+
<str name="q">solr_bbox:"IsWithin(-160 20 -150 30)"^10 railroads</str>
|
214
|
+
<str name="fq">solr_bbox:"Intersects(-160 20 -150 30)"</str>
|
215
|
+
{code}
|
216
|
+
|
217
|
+
|
218
|
+
h2. Solr 4 Spatial -- JTS
|
219
|
+
|
220
|
+
{warning}
|
221
|
+
This query requires [JTS|http://tsusiatsoftware.net/jts/main.html] installed in Solr 4, where the spatialContextFactory="com.spatial4j.core.context.jts.JtsSpatialContextFactory" for the solr.SpatialRecursivePrefixTreeFieldType field class.
|
222
|
+
{warning}
|
223
|
+
|
224
|
+
|
225
|
+
h3. Search for bbox _intersecting_ bounding box of SW=20,-160 NE=70,-70 using polygon intersection
|
226
|
+
|
227
|
+
|
228
|
+
{code:xml}
|
229
|
+
<str name="q">*:*</str>
|
230
|
+
<str name="fq">solr_bbox:"Intersects(POLYGON((-160 20, -160 70, -70 70, -70 20, -160 20)))"</str>
|
231
|
+
{code}
|
232
|
+
|
233
|
+
|
234
|
+
h2. Scoring formula
|
235
|
+
|
236
|
+
{code}
|
237
|
+
text^1
|
238
|
+
dc_description_ti^2
|
239
|
+
dc_creator_tmi^3
|
240
|
+
dc_publisher_ti^3
|
241
|
+
dct_isPartOf_tmi^4
|
242
|
+
dc_subject_tmi^5
|
243
|
+
dct_spatial_tmi^5
|
244
|
+
dct_temporal_tmi^5
|
245
|
+
dc_title_ti^6
|
246
|
+
dc_rights_ti^7
|
247
|
+
dct_provenance_ti^8
|
248
|
+
layer_geom_type_ti^9
|
249
|
+
layer_slug_ti^10
|
250
|
+
dc_identifier_ti^10
|
251
|
+
{code}
|
252
|
+
|
253
|
+
h2. Facets
|
254
|
+
|
255
|
+
{code:xml}
|
256
|
+
<str name="facet.field">dct_spatial_sm</str>
|
257
|
+
<str name="facet.field">dc_format_s</str>
|
258
|
+
<str name="facet.field">dc_language_s</str>
|
259
|
+
<str name="facet.field">dc_publisher_s</str>
|
260
|
+
<str name="facet.field">dc_rights_s</str>
|
261
|
+
<str name="facet.field">dct_provenance_s</str>
|
262
|
+
<str name="facet.field">dc_subject_sm</str>
|
263
|
+
<str name="facet.field">dct_isPartOf_sm</str>
|
264
|
+
<str name="facet.field">layer_geom_type_s</str>
|
265
|
+
<str name="facet.field">solr_year_i</str>
|
266
|
+
{code}
|
267
|
+
|
268
|
+
|
269
|
+
----
|
270
|
+
h1. Solr example documents
|
271
|
+
|
272
|
+
See [https://github.com/sul-dlss/geohydra/blob/master/ogp/transform.rb].
|
273
|
+
|
274
|
+
These metadata would be generated from the OGP Schema, or MODS, or FGDC, or ISO 19139.
|
275
|
+
|
276
|
+
{code}
|
277
|
+
|
278
|
+
"uuid": "http://purl.stanford.edu/zy658cr1728",
|
279
|
+
"dc_description_s": "This point dataset shows village locations with socio-demographic and economic Census data f
|
280
|
+
or 2001 for the Union Territory of Andaman and Nicobar Islands, India linked to the 2001 Census. Includes village s
|
281
|
+
ocio-demographic and economic Census attribute data such as total population, population by sex, household, literac
|
282
|
+
y and illiteracy rates, and employment by industry. This layer is part of the VillageMap dataset which includes soc
|
283
|
+
io-demographic and economic Census data for 2001 at the village level for all the states of India. This data layer
|
284
|
+
is sourced from secondary government sources, chiefly Survey of India, Census of India, Election Commission, etc. T
|
285
|
+
his map Includes data for 547 villages, 3 towns, 2 districts, and 1 union territory.; This dataset is intended for
|
286
|
+
researchers, students, and policy makers for reference and mapping purposes, and may be used for village level demo
|
287
|
+
graphic analysis within basic applications to support graphical overlays and analysis with other spatial data.; ",
|
288
|
+
"dc_format_s": "Shapefile",
|
289
|
+
"dc_identifier_s": "http://purl.stanford.edu/zy658cr1728",
|
290
|
+
"dc_language_s": "English",
|
291
|
+
"dc_publisher_s": "ML InfoMap (Firm)",
|
292
|
+
"dc_rights_s": "Restricted",
|
293
|
+
"dc_subject_sm": [
|
294
|
+
"Human settlements",
|
295
|
+
"Villages",
|
296
|
+
"Census",
|
297
|
+
"Demography",
|
298
|
+
"Population",
|
299
|
+
"Sex ratio",
|
300
|
+
"Housing",
|
301
|
+
"Labor supply",
|
302
|
+
"Caste",
|
303
|
+
"Literacy",
|
304
|
+
"Society",
|
305
|
+
"",
|
306
|
+
"Location"
|
307
|
+
],
|
308
|
+
"dc_title_s": "Andaman and Nicobar, India: Village Socio-Demographic and Economic Census Data, 2001",
|
309
|
+
"dc_type_s": "Dataset",
|
310
|
+
"dct_isPartOf_sm": "My Collection",
|
311
|
+
"dct_references_sm": [
|
312
|
+
"scheme=\"urn:ogc:serviceType:WebFeatureService\" url=\"http://geowebservices-restricted.stanford.edu/geoserver/wfs\"",
|
313
|
+
"scheme=\"urn:ogc:serviceType:WebMapService\" url=\"http://geowebservices-restricted.stanford.edu/geoserver/wms\"",
|
314
|
+
"scheme=\"urn:iso:dataFormat:19139\" url=\"http://purl.stanford.edu/zy658cr1728.iso19139\"",
|
315
|
+
"scheme=\"urn:x-osgeo:link:www\" url=\"http://purl.stanford.edu/zy658cr1728\"",
|
316
|
+
"scheme=\"urn:loc:dataFormat:MODS\" url=\"http://purl.stanford.edu/zy658cr1728.mods\"",
|
317
|
+
"scheme=\"urn:x-osgeo:link:www-thumbnail\", url=\"http://example.com/preview.jpg\""
|
318
|
+
],
|
319
|
+
"dct_spatial_sm": [
|
320
|
+
"Andaman and Nicobar Islands",
|
321
|
+
"Andaman",
|
322
|
+
"Nicobar",
|
323
|
+
"Car Nicobar Island",
|
324
|
+
"Port Blair",
|
325
|
+
"Indira Point",
|
326
|
+
"Diglipur",
|
327
|
+
"Nancowry Island"
|
328
|
+
],
|
329
|
+
"dct_temporal_sm": "2001-01-01T00:00:00Z",
|
330
|
+
"dct_issued_dt": "2000-01-01T00:00:00Z",
|
331
|
+
"dct_provenance_s": "Stanford",
|
332
|
+
"georss_box_s": "6.761581 92.234924 13.637013 94.262535",
|
333
|
+
"georss_polygon_s": "13.637013 92.234924 13.637013 94.262535 6.761581 94.262535 6.761581 92.234924 13.637013 92.234924",
|
334
|
+
"layer_slug_s": "stanford-zy658cr1728",
|
335
|
+
"layer_id_s": "druid:zy658cr1728",
|
336
|
+
"layer_srs_s": "EPSG:4326",
|
337
|
+
"layer_geom_type_s": "Point",
|
338
|
+
"solr_bbox": "92.234924 6.761581 94.262535 13.637013",
|
339
|
+
"solr_ne_pt": "13.637013,94.262535",
|
340
|
+
"solr_sw_pt": "6.761581,92.234924",
|
341
|
+
"solr_geom": "POLYGON((92.234924 13.637013, 94.262535 13.637013, 94.262535 6.761581, 92.234924 6.761581, 92.234924 13.637013))"
|
342
|
+
"score": 1.6703978
|
343
|
+
}
|
344
|
+
{code}
|
345
|
+
|
346
|
+
h1. Links
|
347
|
+
|
348
|
+
* Solr 4: [http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4]
|
349
|
+
* Solr 3: [http://wiki.apache.org/solr/SpatialSearch]
|
350
|
+
* JTS: [http://tsusiatsoftware.net/jts/main.html]
|