search_solr_tools 6.1.0 → 6.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +4 -0
  3. data/bin/search_solr_tools +1 -13
  4. data/lib/search_solr_tools/config/environments.yaml +0 -32
  5. data/lib/search_solr_tools/harvesters/base.rb +0 -1
  6. data/lib/search_solr_tools/helpers/solr_format.rb +0 -15
  7. data/lib/search_solr_tools/helpers/translate_spatial_coverage.rb +0 -1
  8. data/lib/search_solr_tools/version.rb +1 -1
  9. data/lib/search_solr_tools.rb +1 -2
  10. metadata +2 -44
  11. data/lib/search_solr_tools/harvesters/adc.rb +0 -49
  12. data/lib/search_solr_tools/harvesters/ade_auto_suggest.rb +0 -46
  13. data/lib/search_solr_tools/harvesters/bcodmo.rb +0 -64
  14. data/lib/search_solr_tools/harvesters/data_one.rb +0 -49
  15. data/lib/search_solr_tools/harvesters/echo.rb +0 -52
  16. data/lib/search_solr_tools/harvesters/eol.rb +0 -51
  17. data/lib/search_solr_tools/harvesters/gtnp.rb +0 -67
  18. data/lib/search_solr_tools/harvesters/ices.rb +0 -58
  19. data/lib/search_solr_tools/harvesters/ncdc_paleo.rb +0 -62
  20. data/lib/search_solr_tools/harvesters/nmi.rb +0 -34
  21. data/lib/search_solr_tools/harvesters/nodc.rb +0 -75
  22. data/lib/search_solr_tools/harvesters/oai.rb +0 -62
  23. data/lib/search_solr_tools/harvesters/pdc.rb +0 -40
  24. data/lib/search_solr_tools/harvesters/r2r.rb +0 -61
  25. data/lib/search_solr_tools/harvesters/rda.rb +0 -35
  26. data/lib/search_solr_tools/harvesters/tdar.rb +0 -71
  27. data/lib/search_solr_tools/harvesters/usgs.rb +0 -76
  28. data/lib/search_solr_tools/helpers/csw_iso_query_builder.rb +0 -29
  29. data/lib/search_solr_tools/helpers/data_one_format.rb +0 -74
  30. data/lib/search_solr_tools/helpers/iso_to_solr.rb +0 -97
  31. data/lib/search_solr_tools/helpers/iso_to_solr_format.rb +0 -197
  32. data/lib/search_solr_tools/helpers/ncdc_paleo_format.rb +0 -61
  33. data/lib/search_solr_tools/helpers/query_builder.rb +0 -13
  34. data/lib/search_solr_tools/helpers/r2r_format.rb +0 -25
  35. data/lib/search_solr_tools/helpers/selectors.rb +0 -22
  36. data/lib/search_solr_tools/helpers/tdar_format.rb +0 -70
  37. data/lib/search_solr_tools/helpers/usgs_format.rb +0 -50
  38. data/lib/search_solr_tools/selectors/adc.rb +0 -96
  39. data/lib/search_solr_tools/selectors/data_one.rb +0 -96
  40. data/lib/search_solr_tools/selectors/echo_iso.rb +0 -112
  41. data/lib/search_solr_tools/selectors/ices_iso.rb +0 -108
  42. data/lib/search_solr_tools/selectors/ncdc_paleo.rb +0 -90
  43. data/lib/search_solr_tools/selectors/nmi.rb +0 -107
  44. data/lib/search_solr_tools/selectors/nodc_iso.rb +0 -108
  45. data/lib/search_solr_tools/selectors/pdc_iso.rb +0 -109
  46. data/lib/search_solr_tools/selectors/r2r.rb +0 -115
  47. data/lib/search_solr_tools/selectors/rda.rb +0 -107
  48. data/lib/search_solr_tools/selectors/tdar_opensearch.rb +0 -91
  49. data/lib/search_solr_tools/selectors/usgs_iso.rb +0 -107
  50. data/lib/search_solr_tools/translators/bcodmo_json.rb +0 -89
  51. data/lib/search_solr_tools/translators/eol_to_solr.rb +0 -84
  52. data/lib/search_solr_tools/translators/gtnp_json.rb +0 -59
@@ -1,84 +0,0 @@
1
- require 'search_solr_tools'
2
- require_relative '../helpers/solr_format'
3
- require_relative '../helpers/translate_temporal_coverage'
4
- require_relative '../helpers/translate_spatial_coverage'
5
- require_relative '../helpers/bounding_box_util'
6
-
7
- module SearchSolrTools
8
- module Translators
9
- # Translates an EOL THREDDS dataset link set into a SOLR json ingest record
10
- class EolToSolr
11
- # rubocop:disable Metrics/AbcSize
12
- # rubocop:disable Metrics/MethodLength
13
-
14
- def translate(title_metadata, dataset_metadata)
15
- temporal_coverage_values = Helpers::TranslateTemporalCoverage.translate_coverages get_time_coverages(dataset_metadata)
16
- rev_date = dataset_metadata.xpath('//xmlns:date[@type="metadataCreated"]').text
17
- geospatial_coverage = parse_geospatial_coverages(dataset_metadata)
18
- {
19
- 'title' => title_metadata.xpath('//xmlns:dataset').first['name'],
20
- 'authoritative_id' => title_metadata.xpath('//xmlns:dataset').first['ID'],
21
- 'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name],
22
- 'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:EOL][:short_name]}",
23
- 'summary' => dataset_metadata.xpath('//xmlns:documentation[@type="summary"]').text,
24
- 'temporal_coverages' => temporal_coverage_values['temporal_coverages'],
25
- 'temporal_duration' => temporal_coverage_values['temporal_duration'],
26
- 'temporal' => temporal_coverage_values['temporal'],
27
- 'facet_temporal_duration' => temporal_coverage_values['facet_temporal_duration'],
28
- 'last_revision_date' => rev_date.empty? ? Helpers::SolrFormat.date_str(DateTime.now) : Helpers::SolrFormat.date_str(rev_date),
29
- 'source' => 'ADE',
30
- 'keywords' => dataset_metadata.xpath('//xmlns:keyword').map(&:text),
31
- 'authors' => dataset_metadata.xpath('//xmlns:contributor[@role="author"]').map { |node| parse_eol_authors(node.text) }.join(', '),
32
- 'dataset_url' => eol_dataset_url(dataset_metadata),
33
- 'facet_spatial_coverage' => Helpers::BoundingBoxUtil.box_global?(geospatial_coverage),
34
- 'facet_spatial_scope' => Helpers::SolrFormat.get_spatial_scope_facet_with_bounding_box(geospatial_coverage),
35
- 'spatial_coverages' => %i(south west north east).map { |d| geospatial_coverage[d] }.join(' '),
36
- 'spatial_area' => spatial_coverage_to_spatial_area(geospatial_coverage),
37
- 'spatial' => %i(west south east north).map { |d| geospatial_coverage[d] }.join(' ')
38
- }
39
- end
40
-
41
- def eol_dataset_url(node)
42
- node.xpath('//xmlns:documentation[@xlink:href]').each do |doc|
43
- return doc['xlink:href'] if doc['xlink:href'].match('http://data.eol.ucar.edu/codiac/dss/id=(\S*)')
44
- end
45
- rescue Nokogiri::XML::XPath::SyntaxError
46
- puts "Warning - no documentation URL found in the following node: #{node.to_html}"
47
- end
48
-
49
- def parse_eol_authors(author)
50
- if author.include?(' AT ') && author.include?(' dot ')
51
- author = author[0..author.rindex(',') - 1]
52
- end
53
- author
54
- end
55
-
56
- def get_time_coverages(doc)
57
- doc.xpath('//xmlns:timeCoverage').map do |node|
58
- { 'start' => node.xpath('./xmlns:start').text,
59
- 'end' => node.xpath('./xmlns:end').text }
60
- end
61
- end
62
-
63
- def spatial_coverage_to_spatial_area(coverage)
64
- return if [:north, :south].any? { |x| coverage[x].nil? }
65
- coverage[:north].abs - coverage[:south].abs
66
- end
67
-
68
- def parse_geospatial_coverages(doc)
69
- node = doc.xpath('//xmlns:geospatialCoverage')
70
- south = node.xpath('./xmlns:northsouth/xmlns:start').text.to_f
71
- north = south + node.xpath('./xmlns:northsouth/xmlns:size').text.to_f
72
- west = node.xpath('./xmlns:eastwest/xmlns:start').text.to_f
73
- east = west + node.xpath('./xmlns:eastwest/xmlns:size').text.to_f
74
- # EOL uses out-of-range east-west values to represent bounding boxes
75
- # that cross the date line. For any box with a value out of range,
76
- # adjust the east/west value to lie within the -180 to 180 range.
77
- east -= 360 if east > 180
78
- west += 360 if west < -180
79
-
80
- { east: east, west: west, north: north, south: south }
81
- end
82
- end
83
- end
84
- end
@@ -1,59 +0,0 @@
1
- require 'json'
2
- require 'rest-client'
3
- require 'rgeo/geo_json'
4
-
5
- require 'search_solr_tools'
6
- require_relative '../helpers/solr_format'
7
- require_relative '../helpers/translate_temporal_coverage'
8
- require_relative '../helpers/translate_spatial_coverage'
9
-
10
- module SearchSolrTools
11
- module Translators
12
- # Translates GTN-P json to solr json format
13
- class GtnpJsonToSolr
14
- # rubocop:disable AbcSize
15
- def translate(json_doc, json_record)
16
- json_geo = json_doc['geo'].nil? ? json_doc['coordinates'] : json_doc['geo']['coordinates']
17
- concatenated_name = "#{json_record['title']} - #{json_doc['name']}"
18
- spatial_values = translate_geometry json_geo
19
- {
20
- 'title' => concatenated_name,
21
- 'authoritative_id' => concatenated_name,
22
- 'data_centers' => Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name],
23
- 'facet_data_center' => "#{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:long_name]} | #{Helpers::SolrFormat::DATA_CENTER_NAMES[:GTNP][:short_name]}",
24
- 'summary' => json_record['abstract'].to_s,
25
- 'dataset_url' => json_doc['link'],
26
- 'source' => 'ADE',
27
- 'facet_spatial_scope' => spatial_values[:spatial_scope_facet],
28
- 'spatial_coverages' => spatial_values[:spatial_display],
29
- 'spatial_area' => spatial_values[:spatial_area],
30
- 'spatial' => spatial_values[:spatial_index],
31
- 'temporal_coverages' => Helpers::SolrFormat::NOT_SPECIFIED,
32
- 'facet_temporal_duration' => Helpers::SolrFormat::NOT_SPECIFIED,
33
- 'authors' => parse_people(json_doc)
34
- }
35
- end
36
-
37
- def parse_people(json_doc)
38
- people_found = []
39
- return people_found unless json_doc.key?('citation') && json_doc['citation'].key?('contacts')
40
- citation = json_doc['citation']
41
- citation['contacts'].each do |person|
42
- people_found << "#{person['givenName']} #{person['familyName']}"
43
- end
44
- people_found
45
- end
46
-
47
- def translate_geometry(json_geom)
48
- geo_string = "{\"type\":\"Point\",\"coordinates\":[#{json_geom['longitude']},#{json_geom['latitude']}]}"
49
- geometry = RGeo::GeoJSON.decode(geo_string, json_parser: :json)
50
- {
51
- spatial_display: Helpers::TranslateSpatialCoverage.geojson_to_spatial_display_str([geometry]),
52
- spatial_index: Helpers::TranslateSpatialCoverage.geojson_to_spatial_index_str([geometry]),
53
- spatial_area: Helpers::TranslateSpatialCoverage.geojson_to_spatial_area([geometry]),
54
- spatial_scope_facet: Helpers::TranslateSpatialCoverage.geojson_to_spatial_scope_facet([geometry])
55
- }
56
- end
57
- end
58
- end
59
- end