relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/relaton/3gpp/bibliography.rb +19 -6
- data/lib/relaton/3gpp/processor.rb +6 -0
- data/lib/relaton/adobe/processor.rb +9 -0
- data/lib/relaton/bib/converter/csl.rb +110 -0
- data/lib/relaton/bib/converter/ris.rb +104 -0
- data/lib/relaton/bib/converter/titles.rb +24 -0
- data/lib/relaton/bib/item_data.rb +23 -0
- data/lib/relaton/bib/model/item.rb +6 -6
- data/lib/relaton/bib/sanitizer.rb +71 -30
- data/lib/relaton/bib.rb +10 -0
- data/lib/relaton/bipm/bibliography.rb +24 -21
- data/lib/relaton/bipm/processor.rb +1 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
- data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
- data/lib/relaton/bsi/bibliography.rb +26 -23
- data/lib/relaton/bsi/processor.rb +5 -0
- data/lib/relaton/calconnect/bibliography.rb +6 -6
- data/lib/relaton/calconnect/hit_collection.rb +4 -1
- data/lib/relaton/ccsds/bibliography.rb +16 -10
- data/lib/relaton/ccsds/hit_collection.rb +1 -1
- data/lib/relaton/ccsds/processor.rb +13 -0
- data/lib/relaton/cen/bibliography.rb +27 -24
- data/lib/relaton/cen/hit_collection.rb +1 -1
- data/lib/relaton/cen/processor.rb +5 -0
- data/lib/relaton/cen/scraper.rb +9 -9
- data/lib/relaton/cie/bibliography.rb +11 -9
- data/lib/relaton/cie/data_fetcher.rb +18 -18
- data/lib/relaton/cie/processor.rb +1 -0
- data/lib/relaton/cie/scrapper.rb +15 -8
- data/lib/relaton/cie.rb +1 -1
- data/lib/relaton/cloud.rb +127 -0
- data/lib/relaton/core/hit_collection.rb +6 -10
- data/lib/relaton/core/processor.rb +121 -0
- data/lib/relaton/core/request_error.rb +21 -3
- data/lib/relaton/db/cache.rb +474 -148
- data/lib/relaton/db/cache_entry.rb +42 -0
- data/lib/relaton/db/registry.rb +152 -0
- data/lib/relaton/db.rb +275 -102
- data/lib/relaton/doi/crossref.rb +23 -6
- data/lib/relaton/doi/processor.rb +1 -0
- data/lib/relaton/easc/processor.rb +1 -0
- data/lib/relaton/ecma/bibliography.rb +18 -13
- data/lib/relaton/ecma/data_fetcher.rb +1 -1
- data/lib/relaton/ecma/data_parser.rb +1 -1
- data/lib/relaton/ecma/edition_parser.rb +2 -2
- data/lib/relaton/ecma/memento_parser.rb +4 -4
- data/lib/relaton/ecma/standard_parser.rb +6 -6
- data/lib/relaton/etsi/bibliography.rb +8 -7
- data/lib/relaton/etsi/processor.rb +1 -0
- data/lib/relaton/gb/bibliography.rb +20 -12
- data/lib/relaton/gb/gb_scraper.rb +5 -5
- data/lib/relaton/gb/scraper.rb +16 -16
- data/lib/relaton/gb/sec_scraper.rb +8 -8
- data/lib/relaton/gb/t_scraper.rb +5 -5
- data/lib/relaton/gost/processor.rb +1 -0
- data/lib/relaton/iala/bibliography.rb +15 -12
- data/lib/relaton/iala/processor.rb +1 -0
- data/lib/relaton/iana/bibliography.rb +12 -12
- data/lib/relaton/iana/data_fetcher.rb +3 -3
- data/lib/relaton/iana/parser.rb +12 -5
- data/lib/relaton/iana/processor.rb +9 -0
- data/lib/relaton/iec/bibliography.rb +18 -7
- data/lib/relaton/iec/data_parser.rb +24 -9
- data/lib/relaton/iec/processor.rb +11 -0
- data/lib/relaton/iec.rb +1 -1
- data/lib/relaton/ieee/bibliography.rb +19 -13
- data/lib/relaton/ieee/data_fetcher.rb +1 -1
- data/lib/relaton/ieee/processor.rb +24 -0
- data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
- data/lib/relaton/ietf/bibliography.rb +11 -9
- data/lib/relaton/ietf/data_fetcher.rb +1 -1
- data/lib/relaton/ietf/processor.rb +1 -0
- data/lib/relaton/ietf/rfc/entry.rb +15 -19
- data/lib/relaton/ietf/scraper.rb +5 -4
- data/lib/relaton/iho/processor.rb +1 -0
- data/lib/relaton/index/file_io.rb +2 -2
- data/lib/relaton/index/pool.rb +4 -3
- data/lib/relaton/index/shard_source.rb +1 -1
- data/lib/relaton/index/type.rb +2 -5
- data/lib/relaton/isbn/open_library.rb +11 -7
- data/lib/relaton/isbn/processor.rb +10 -0
- data/lib/relaton/iso/bibliography.rb +11 -9
- data/lib/relaton/iso/data_parser.rb +2 -2
- data/lib/relaton/iso/processor.rb +5 -0
- data/lib/relaton/iso/scraper.rb +20 -20
- data/lib/relaton/itu/bibliography.rb +107 -16
- data/lib/relaton/itu/data_crawler_r.rb +2 -2
- data/lib/relaton/itu/hit_collection.rb +64 -52
- data/lib/relaton/itu/processor.rb +1 -0
- data/lib/relaton/itu/scraper.rb +13 -3
- data/lib/relaton/itu.rb +0 -2
- data/lib/relaton/jis/bibliography.rb +22 -11
- data/lib/relaton/jis/data_fetcher.rb +4 -4
- data/lib/relaton/jis/processor.rb +1 -0
- data/lib/relaton/jis/scraper.rb +8 -8
- data/lib/relaton/nist/bibliography.rb +25 -21
- data/lib/relaton/oasis/bibliography.rb +16 -13
- data/lib/relaton/oasis/browser_agent.rb +2 -2
- data/lib/relaton/oasis/data_parser.rb +5 -5
- data/lib/relaton/oasis/data_parser_utils.rb +2 -2
- data/lib/relaton/oasis/data_part_parser.rb +7 -7
- data/lib/relaton/ogc/bibliography.rb +15 -14
- data/lib/relaton/ogc/hit_collection.rb +9 -7
- data/lib/relaton/ogc/processor.rb +13 -0
- data/lib/relaton/oiml/processor.rb +1 -0
- data/lib/relaton/omg/bibliography.rb +9 -9
- data/lib/relaton/omg/scraper.rb +14 -13
- data/lib/relaton/omg.rb +1 -1
- data/lib/relaton/plateau/bibliography.rb +9 -7
- data/lib/relaton/plateau/hit_collection.rb +2 -1
- data/lib/relaton/plateau/processor.rb +1 -0
- data/lib/relaton/un/bibliography.rb +21 -10
- data/lib/relaton/un/processor.rb +1 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/processor.rb +1 -0
- data/lib/relaton/xsf/bibliography.rb +11 -6
- data/lib/relaton.rb +13 -0
- metadata +60 -14
- data/lib/relaton/itu/pubid.rb +0 -199
|
@@ -5,18 +5,20 @@ module Relaton::Bipm
|
|
|
5
5
|
GH_ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-bipm/refs/heads/v2/".freeze
|
|
6
6
|
|
|
7
7
|
class << self
|
|
8
|
-
# @param
|
|
8
|
+
# @param ref [String, Pubid::Bipm::Identifier] the reference, or its
|
|
9
|
+
# parse from Relaton::Db (relaton#205)
|
|
9
10
|
# @return [RelatonBipm::BipmBibliographicItem]
|
|
10
|
-
def search(
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
def search(ref, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
12
|
+
key = ref.to_s
|
|
13
|
+
Util.info "Fetching from Relaton repository ...", key: key
|
|
14
|
+
query = ref.is_a?(String) ? ref.sub(/^BIPM\s/, "") : ref
|
|
15
|
+
item = get_bipm query
|
|
14
16
|
unless item
|
|
15
|
-
Util.info "Not found.", key:
|
|
17
|
+
Util.info "Not found.", key: key
|
|
16
18
|
return
|
|
17
19
|
end
|
|
18
20
|
|
|
19
|
-
Util.info "Found: `#{item.docidentifier[0].content}`", key:
|
|
21
|
+
Util.info "Found: `#{item.docidentifier[0].content}`", key: key
|
|
20
22
|
item
|
|
21
23
|
rescue Mechanize::ResponseCodeError => e
|
|
22
24
|
raise Relaton::RequestError, e.message unless e.response_code == "404"
|
|
@@ -40,13 +42,13 @@ module Relaton::Bipm
|
|
|
40
42
|
# end
|
|
41
43
|
|
|
42
44
|
#
|
|
43
|
-
# @param
|
|
45
|
+
# @param ref [String, Pubid::Bipm::Identifier]
|
|
44
46
|
#
|
|
45
47
|
# @return [RelatonBipm::BipmBibliographicItem]
|
|
46
48
|
#
|
|
47
|
-
def get_bipm(
|
|
48
|
-
pubid = parse_ref
|
|
49
|
-
rows = search_index pubid,
|
|
49
|
+
def get_bipm(ref) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
50
|
+
pubid = ref.is_a?(String) ? parse_ref(ref) : ref
|
|
51
|
+
rows = search_index pubid, ref.to_s
|
|
50
52
|
return if rows.empty?
|
|
51
53
|
|
|
52
54
|
# Latest edition wins. Metrologia and SI Brochure rows carry no year,
|
|
@@ -78,9 +80,9 @@ module Relaton::Bipm
|
|
|
78
80
|
# a rescan cannot help — only re-reading the tail as number-plus-year.
|
|
79
81
|
#
|
|
80
82
|
# @param pubid [Pubid::Bipm::Identifier] the parsed reference
|
|
81
|
-
# @param
|
|
83
|
+
# @param ref [String, nil] the raw reference, for the retry
|
|
82
84
|
# @return [Array<Hash>] the matching index rows
|
|
83
|
-
def search_index(pubid,
|
|
85
|
+
def search_index(pubid, ref)
|
|
84
86
|
# Reduce the query once, not once per row. `#exclude` copies the
|
|
85
87
|
# identifier, so rebuilding it per candidate dominated the lookup.
|
|
86
88
|
query_stem = stem pubid, pubid
|
|
@@ -88,7 +90,7 @@ module Relaton::Bipm
|
|
|
88
90
|
rows = index.search { |r| pubid_match? r[:id], pubid, query_stem } if rows.empty?
|
|
89
91
|
return rows unless rows.empty?
|
|
90
92
|
|
|
91
|
-
retry_pubid = year_number_retry
|
|
93
|
+
retry_pubid = year_number_retry ref
|
|
92
94
|
retry_pubid ? search_index(retry_pubid, nil) : rows
|
|
93
95
|
end
|
|
94
96
|
|
|
@@ -101,10 +103,10 @@ module Relaton::Bipm
|
|
|
101
103
|
# reference, so a probe that does not parse is a miss (nil), never a
|
|
102
104
|
# malformed-reference error -- the same rule as ISO's type/stage probe.
|
|
103
105
|
#
|
|
104
|
-
# @param
|
|
106
|
+
# @param ref [String, nil]
|
|
105
107
|
# @return [Pubid::Bipm::Identifier, nil]
|
|
106
|
-
def year_number_retry(
|
|
107
|
-
match =
|
|
108
|
+
def year_number_retry(ref)
|
|
109
|
+
match = ref&.match(/\A(?<stem>.+)\s(?<year>\d{4})-(?<number>\d+)\z/)
|
|
108
110
|
return unless match
|
|
109
111
|
|
|
110
112
|
parse_ref "#{match[:stem]} #{match[:number].sub(/\A0+(?=\d)/, '')} (#{match[:year]})"
|
|
@@ -205,11 +207,11 @@ module Relaton::Bipm
|
|
|
205
207
|
# Pubid::Errors::Error and renders "... is not a recognized standards
|
|
206
208
|
# identifier".
|
|
207
209
|
#
|
|
208
|
-
# @param
|
|
210
|
+
# @param ref [String]
|
|
209
211
|
# @return [Pubid::Bipm::Identifier]
|
|
210
212
|
# @raise [Pubid::Errors::ParseError]
|
|
211
|
-
def parse_ref(
|
|
212
|
-
::Pubid::Bipm.parse
|
|
213
|
+
def parse_ref(ref)
|
|
214
|
+
::Pubid::Bipm.parse ref
|
|
213
215
|
end
|
|
214
216
|
|
|
215
217
|
def index
|
|
@@ -219,7 +221,8 @@ module Relaton::Bipm
|
|
|
219
221
|
)
|
|
220
222
|
end
|
|
221
223
|
|
|
222
|
-
# @param ref [String] the BIPM standard Code to
|
|
224
|
+
# @param ref [String, Pubid::Bipm::Identifier] the BIPM standard Code to
|
|
225
|
+
# look up (e..g "BIPM B-11"), or its parse from Relaton::Db
|
|
223
226
|
# @param year [String] not used
|
|
224
227
|
# @param opts [Hash] not used
|
|
225
228
|
# @return [RelatonBipm::BipmBibliographicItem]
|
|
@@ -8,6 +8,7 @@ module Relaton
|
|
|
8
8
|
def initialize
|
|
9
9
|
@short = :relaton_bipm
|
|
10
10
|
@prefix = "BIPM"
|
|
11
|
+
@pubid_identifier = :Bipm # Db cache key
|
|
11
12
|
@defaultprefix = %r{^(?:BIPM|CCTF|CCDS|CGPM|CIPM|JCRB)(?!\w)}
|
|
12
13
|
@idtype = "BIPM"
|
|
13
14
|
@datasets = %w[bipm-data-outcomes bipm-si-brochure rawdata-bipm-metrologia]
|
|
@@ -19,9 +19,9 @@ module Relaton::Bipm
|
|
|
19
19
|
#
|
|
20
20
|
def self.parse(dir)
|
|
21
21
|
affiliations = Dir["#{dir}/*.xml"].each_with_object([]) do |path, m|
|
|
22
|
-
doc =
|
|
22
|
+
doc = Moxml.parse(File.read(path, encoding: "UTF-8"))
|
|
23
23
|
doc.xpath("//aff").each do |aff|
|
|
24
|
-
m << parse_affiliation(aff) if aff.
|
|
24
|
+
m << parse_affiliation(aff) if aff.at_xpath("institution")
|
|
25
25
|
end
|
|
26
26
|
end.uniq { |a| a.organization.name.first.content }
|
|
27
27
|
new affiliations
|
|
@@ -31,18 +31,18 @@ module Relaton::Bipm
|
|
|
31
31
|
# Parse affiliation organization
|
|
32
32
|
# https://github.com/relaton/relaton-data-bipm/issues/17#issuecomment-1367035444
|
|
33
33
|
#
|
|
34
|
-
# @param [
|
|
34
|
+
# @param [Moxml::Element] aff
|
|
35
35
|
#
|
|
36
36
|
# @return [Relaton::Bib::Affiliation] Organization name, country, division, street address
|
|
37
37
|
#
|
|
38
38
|
def self.parse_affiliation(aff)
|
|
39
|
-
text = aff.
|
|
39
|
+
text = aff.at_xpath("text()").text
|
|
40
40
|
return if text.include? "Permanent address:" || text.include?("1005 Southover Lane") ||
|
|
41
41
|
text == "Germany" || text.starts_with?("Guest") || text.starts_with?("Deceased") ||
|
|
42
42
|
text.include?("Author to whom any correspondence should be addressed")
|
|
43
43
|
|
|
44
44
|
args = {}
|
|
45
|
-
institution = aff.
|
|
45
|
+
institution = aff.at_xpath('institution')
|
|
46
46
|
if institution
|
|
47
47
|
name = institution.text
|
|
48
48
|
return if name == "1005 Southover Lane"
|
|
@@ -72,7 +72,7 @@ module Relaton::Bipm
|
|
|
72
72
|
address = []
|
|
73
73
|
addr = aff.xpath("text()[preceding-sibling::institution]").text.gsub(/^\W*|\W*$/, "")
|
|
74
74
|
address << addr unless addr.empty?
|
|
75
|
-
country = aff.
|
|
75
|
+
country = aff.at_xpath('country')
|
|
76
76
|
address << country.text if country && !country.text.empty?
|
|
77
77
|
address = address.join(", ")
|
|
78
78
|
return [] if address.empty?
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
require "
|
|
1
|
+
require "moxml"
|
|
2
2
|
|
|
3
3
|
module Relaton::Bipm
|
|
4
4
|
class SiBrochureParser
|
|
@@ -129,7 +129,7 @@ module Relaton::Bipm
|
|
|
129
129
|
end
|
|
130
130
|
|
|
131
131
|
def extract_editorialgroup(xml)
|
|
132
|
-
doc =
|
|
132
|
+
doc = Moxml.parse(xml)
|
|
133
133
|
doc.xpath("//editorialgroup/committee").map do |committee|
|
|
134
134
|
acronym = committee["acronym"]
|
|
135
135
|
names = committee.xpath("variant").map do |v|
|
|
@@ -7,10 +7,11 @@ module Relaton::Bsi
|
|
|
7
7
|
Ids = ::Pubid::Bsi::Identifiers
|
|
8
8
|
|
|
9
9
|
class << self
|
|
10
|
-
# @param
|
|
10
|
+
# @param ref [String] the printed reference
|
|
11
11
|
# @return [Relaton::Bsi::HitCollection]
|
|
12
|
-
def search(
|
|
13
|
-
|
|
12
|
+
def search(ref, year = nil)
|
|
13
|
+
# A pubid from Relaton::Db (relaton#205) prints "Expert Commentary".
|
|
14
|
+
code = ref.sub(/^BSI\s/, "").sub(/ExComm|Expert commentary/i, "Ex")
|
|
14
15
|
HitCollection.search(code, year)
|
|
15
16
|
rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
|
|
16
17
|
EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
|
@@ -19,16 +20,18 @@ module Relaton::Bsi
|
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
#
|
|
22
|
-
# @param
|
|
23
|
+
# @param ref [String, Pubid::Bsi::Identifier] the BSI standard reference
|
|
24
|
+
# to look up, or its parse from Relaton::Db (relaton#205)
|
|
23
25
|
# @param year [String] the year the standard was published (optional)
|
|
24
26
|
# @param opts [Hash] options
|
|
25
27
|
# @option opts [Boolean] :all_parts if all-parts reference is required
|
|
26
28
|
# @option opts [Boolean] :no_year if last published document is required
|
|
27
29
|
#
|
|
28
30
|
# @return [String] Relaton XML serialisation of reference
|
|
29
|
-
def get(
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
def get(ref, year = nil, opts = {})
|
|
32
|
+
query = ref.is_a?(String) ? parse(ref) : ref
|
|
33
|
+
year ||= year_of(query)
|
|
34
|
+
ret = bib_get(ref.to_s, year, opts, query)
|
|
32
35
|
return nil if ret.nil?
|
|
33
36
|
|
|
34
37
|
ret = ret.to_most_recent_reference unless year || opts[:keep_year]
|
|
@@ -40,18 +43,18 @@ module Relaton::Bsi
|
|
|
40
43
|
# Pubid::Errors::Error and renders "... is not a recognized standards
|
|
41
44
|
# identifier". A catalogue hit code is a different case: `Hit#pubid`
|
|
42
45
|
# rescues, because one odd Algolia row must not abort the search.
|
|
43
|
-
# @param [String]
|
|
46
|
+
# @param [String] ref the reference
|
|
44
47
|
# @return [Pubid::Bsi::Identifier]
|
|
45
48
|
# @raise [Pubid::Errors::ParseError]
|
|
46
|
-
def parse(
|
|
47
|
-
::Pubid::Bsi::Identifier.parse(
|
|
49
|
+
def parse(ref)
|
|
50
|
+
::Pubid::Bsi::Identifier.parse(ref)
|
|
48
51
|
end
|
|
49
52
|
|
|
50
53
|
# Publication year of a reference's base document.
|
|
51
|
-
# @param [String]
|
|
54
|
+
# @param [String] ref the reference
|
|
52
55
|
# @return [String, nil]
|
|
53
|
-
def publication_year(
|
|
54
|
-
year_of parse(
|
|
56
|
+
def publication_year(ref)
|
|
57
|
+
year_of parse(ref)
|
|
55
58
|
end
|
|
56
59
|
|
|
57
60
|
# Whether a query and a candidate hit denote the same BSI document, at the
|
|
@@ -154,14 +157,14 @@ module Relaton::Bsi
|
|
|
154
157
|
#
|
|
155
158
|
# Search for a BSI standard.
|
|
156
159
|
#
|
|
157
|
-
# @param [String]
|
|
160
|
+
# @param [String] ref the BSI standard reference to look up
|
|
161
|
+
# @param [Pubid::Bsi::Identifier] query the parsed reference
|
|
158
162
|
#
|
|
159
163
|
# @return [Relaton::Bsi::HitCollection] a collection of hits
|
|
160
164
|
#
|
|
161
|
-
def search_filter(
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
search(code).filter_hits!(query)
|
|
165
|
+
def search_filter(ref, query)
|
|
166
|
+
Util.info "Fetching from shop.bsigroup.com ...", key: ref
|
|
167
|
+
search(ref).filter_hits!(query)
|
|
165
168
|
end
|
|
166
169
|
|
|
167
170
|
# Sort through the results, and return the first result that matches the
|
|
@@ -180,15 +183,15 @@ module Relaton::Bsi
|
|
|
180
183
|
{ years: missed_years }
|
|
181
184
|
end
|
|
182
185
|
|
|
183
|
-
def bib_get(
|
|
184
|
-
result = search_filter(
|
|
186
|
+
def bib_get(ref, year, _opts, query)
|
|
187
|
+
result = search_filter(ref, query) || return
|
|
185
188
|
ret = results_filter(result, year)
|
|
186
189
|
if ret[:ret]
|
|
187
|
-
Util.info "Found: `#{ret[:ret].docidentifier.first&.content}`", key:
|
|
190
|
+
Util.info "Found: `#{ret[:ret].docidentifier.first&.content}`", key: ref
|
|
188
191
|
ret[:ret]
|
|
189
192
|
else
|
|
190
|
-
Util.info "Not found", key:
|
|
191
|
-
fetch_ref_err(
|
|
193
|
+
Util.info "Not found", key: ref
|
|
194
|
+
fetch_ref_err(ref, year, ret[:years])
|
|
192
195
|
end
|
|
193
196
|
end
|
|
194
197
|
end
|
|
@@ -19,6 +19,11 @@ module Relaton::Bsi
|
|
|
19
19
|
::Relaton::Bsi::Bibliography.get(code, date, opts)
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
+
# `Bibliography.get` applies a year to the document the reference names.
|
|
23
|
+
def fold_year(pubid, year)
|
|
24
|
+
fold_year_on_root pubid, year
|
|
25
|
+
end
|
|
26
|
+
|
|
22
27
|
# @param xml [String]
|
|
23
28
|
# @return [Relaton::Bsi::ItemData]
|
|
24
29
|
def from_xml(xml)
|
|
@@ -3,10 +3,10 @@ require "mechanize"
|
|
|
3
3
|
module Relaton::Calconnect
|
|
4
4
|
class Bibliography
|
|
5
5
|
class << self
|
|
6
|
-
# @param
|
|
6
|
+
# @param ref [String]
|
|
7
7
|
# @return [RelatonCalconnect::HitCollection]
|
|
8
|
-
def search(
|
|
9
|
-
HitCollection.new
|
|
8
|
+
def search(ref, year = nil, _opts = {})
|
|
9
|
+
HitCollection.new ref, year
|
|
10
10
|
rescue Mechanize::ResponseCodeError, SocketError, Errno::ECONNREFUSED
|
|
11
11
|
raise Relaton::RequestError, "Could not access https://standards.calconnect.org"
|
|
12
12
|
end
|
|
@@ -26,14 +26,14 @@ module Relaton::Calconnect
|
|
|
26
26
|
# afterwards, because an undated reference reaches every year of the
|
|
27
27
|
# document. That split is why `bib_results_filter` stays.
|
|
28
28
|
def get(ref, year = nil, opts = {})
|
|
29
|
-
Util.info "Fetching from Relaton repository ...", key: ref
|
|
29
|
+
Util.info "Fetching from Relaton repository ...", key: ref.to_s
|
|
30
30
|
result = search(ref, year, opts) || (return nil)
|
|
31
31
|
ret = bib_results_filter(result, year)
|
|
32
32
|
if ret[:ret]
|
|
33
|
-
Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: ref
|
|
33
|
+
Util.info "Found: `#{ret[:ret].docidentifier.first.content}`", key: ref.to_s
|
|
34
34
|
ret[:ret]
|
|
35
35
|
else
|
|
36
|
-
Util.info "Not found.", key: ref
|
|
36
|
+
Util.info "Not found.", key: ref.to_s
|
|
37
37
|
fetch_ref_err(ref, year, ret[:years])
|
|
38
38
|
end
|
|
39
39
|
end
|
|
@@ -55,7 +55,7 @@ module Relaton::Calconnect
|
|
|
55
55
|
# outside the transport rescue in `Bibliography.search`, so it never becomes
|
|
56
56
|
# a `Relaton::RequestError`.
|
|
57
57
|
#
|
|
58
|
-
# @param ref [String]
|
|
58
|
+
# @param ref [String, Pubid::Calconnect::Identifier]
|
|
59
59
|
# @return [Pubid::Calconnect::Identifier, nil]
|
|
60
60
|
#
|
|
61
61
|
# An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
|
|
@@ -66,6 +66,9 @@ module Relaton::Calconnect
|
|
|
66
66
|
# Rescuing here would collapse "this identifier is malformed" into "no
|
|
67
67
|
# such document", leaving a caller unable to tell them apart.
|
|
68
68
|
def parse_ref(ref)
|
|
69
|
+
# A parsed pubid comes from Relaton::Db (relaton#205); it is used as it is.
|
|
70
|
+
return ref unless ref.is_a?(String)
|
|
71
|
+
|
|
69
72
|
::Pubid::Calconnect::Identifier.parse ref.to_s.strip
|
|
70
73
|
end
|
|
71
74
|
|
|
@@ -20,32 +20,38 @@ module Relaton
|
|
|
20
20
|
# Get CCSDS standard by document reference.
|
|
21
21
|
# If format is not specified, then all format will be returned.
|
|
22
22
|
#
|
|
23
|
-
# @param
|
|
23
|
+
# @param ref [String, Pubid::Ccsds::Identifier] the reference, or
|
|
24
|
+
# the parse that Relaton::Db routed with (relaton#205)
|
|
24
25
|
# @param year [String, nil]
|
|
25
26
|
# @param opts [Hash]
|
|
26
27
|
# @option opts [String] :format format of fetched document (DOC, PDF)
|
|
27
28
|
#
|
|
28
29
|
# @return [RelatonCcsds::BibliographicItem]
|
|
29
30
|
#
|
|
30
|
-
def get(
|
|
31
|
-
|
|
32
|
-
Util.info "Fetching from Relaton repository ...", key:
|
|
33
|
-
item, hit = fetch_item(
|
|
31
|
+
def get(ref, _year = nil, opts = {})
|
|
32
|
+
query, opts = parse_format(ref, opts)
|
|
33
|
+
Util.info "Fetching from Relaton repository ...", key: ref.to_s
|
|
34
|
+
item, hit = fetch_item(query)
|
|
34
35
|
if item.nil? || filter_sources(item, opts[:format])
|
|
35
|
-
Util.info "Not found.", key:
|
|
36
|
+
Util.info "Not found.", key: ref.to_s
|
|
36
37
|
return nil
|
|
37
38
|
end
|
|
38
|
-
Util.info "Found: `#{hit[:code]}`.", key:
|
|
39
|
+
Util.info "Found: `#{hit[:code]}`.", key: ref.to_s
|
|
39
40
|
item
|
|
40
41
|
end
|
|
41
42
|
|
|
42
43
|
private
|
|
43
44
|
|
|
44
|
-
def parse_format(
|
|
45
|
-
|
|
45
|
+
def parse_format(ref, opts)
|
|
46
|
+
# A pubid carries no format suffix: Processor#query_pubid hands `get`
|
|
47
|
+
# the String whenever the reference has one.
|
|
48
|
+
return [ref, opts] unless ref.is_a?(String)
|
|
49
|
+
|
|
50
|
+
query = ref.sub(/\s\((DOC|PDF)\)$/, "")
|
|
46
51
|
opts[:format] ||= Regexp.last_match(1)
|
|
47
|
-
[
|
|
52
|
+
[query, opts]
|
|
48
53
|
end
|
|
54
|
+
public :parse_format
|
|
49
55
|
|
|
50
56
|
def fetch_item(ref)
|
|
51
57
|
hit = search(ref).first
|
|
@@ -8,6 +8,7 @@ module Relaton
|
|
|
8
8
|
def initialize # rubocop:disable Lint/MissingSuper
|
|
9
9
|
@short = :relaton_ccsds
|
|
10
10
|
@prefix = "CCSDS"
|
|
11
|
+
@pubid_identifier = :Ccsds # Db cache key
|
|
11
12
|
@defaultprefix = %r{^CCSDS(?!\w)}
|
|
12
13
|
@idtype = "CCSDS"
|
|
13
14
|
@datasets = %w[ccsds]
|
|
@@ -22,6 +23,18 @@ module Relaton
|
|
|
22
23
|
Bibliography.get(code, date, opts)
|
|
23
24
|
end
|
|
24
25
|
|
|
26
|
+
# A format (` (DOC)` or `opts[:format]`) keeps only the item's sources
|
|
27
|
+
# of that format, a filter the Db cache cannot apply to a cached item:
|
|
28
|
+
# such a query gets no pubid, so it is not cached, and `get` receives
|
|
29
|
+
# the String with its format.
|
|
30
|
+
def query_pubid(ref, opts = {}, parsed = nil)
|
|
31
|
+
require_relative "../ccsds"
|
|
32
|
+
_, format_opts = Bibliography.parse_format(ref, opts.dup)
|
|
33
|
+
return if format_opts[:format]
|
|
34
|
+
|
|
35
|
+
super
|
|
36
|
+
end
|
|
37
|
+
|
|
25
38
|
#
|
|
26
39
|
# Fetch all the documents from a source
|
|
27
40
|
#
|
|
@@ -5,10 +5,10 @@ module Relaton
|
|
|
5
5
|
# Class methods for search Cenelec standards.
|
|
6
6
|
class Bibliography
|
|
7
7
|
class << self
|
|
8
|
-
# @param
|
|
8
|
+
# @param ref [String] the printed reference
|
|
9
9
|
# @return [Relaton::Cen::HitCollection]
|
|
10
|
-
def search(
|
|
11
|
-
HitCollection.new(
|
|
10
|
+
def search(ref, year = nil)
|
|
11
|
+
HitCollection.new(ref, year).search
|
|
12
12
|
rescue Mechanize::ResponseCodeError, Net::ReadTimeout => e
|
|
13
13
|
raise Relaton::RequestError, e.message
|
|
14
14
|
end
|
|
@@ -32,23 +32,24 @@ module Relaton
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
#
|
|
35
|
-
# @param
|
|
35
|
+
# @param ref [String, Pubid::CenCenelec::Identifier] the CEN standard
|
|
36
|
+
# reference to look up, or its parse from Relaton::Db (relaton#205)
|
|
36
37
|
# @param year [String] the year the standard was published (optional)
|
|
37
38
|
# @param opts [Hash] options
|
|
38
39
|
# @option opts [Boolean] :keep_year don't upate reference
|
|
39
40
|
#
|
|
40
41
|
# @return [Relaton::Cen::ItemData, nil]
|
|
41
42
|
#
|
|
42
|
-
def get(
|
|
43
|
+
def get(ref, year = nil, opts = {})
|
|
43
44
|
# An empty string is no reference, not a malformed one, so it stays a
|
|
44
45
|
# plain nil rather than a parse failure. Mirrors the guard
|
|
45
46
|
# HitCollection#search already carries.
|
|
46
|
-
return if
|
|
47
|
+
return if ref.nil? || (ref.is_a?(String) && ref.strip.empty?)
|
|
47
48
|
|
|
48
|
-
query = parse
|
|
49
|
+
query = ref.is_a?(String) ? parse(ref) : ref
|
|
49
50
|
year ||= publication_year query
|
|
50
51
|
|
|
51
|
-
bib_get
|
|
52
|
+
bib_get ref.to_s, year, opts, query
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
private
|
|
@@ -77,10 +78,11 @@ module Relaton
|
|
|
77
78
|
id.exclude(*keys) == id
|
|
78
79
|
end
|
|
79
80
|
|
|
80
|
-
def fetch_ref_err(
|
|
81
|
+
def fetch_ref_err(ref, year, missed_years)
|
|
81
82
|
unless missed_years.empty?
|
|
82
83
|
Util.info "There was no match for `#{year}`, though there " \
|
|
83
|
-
"were matches found for `#{missed_years.join('`, `')}`."
|
|
84
|
+
"were matches found for `#{missed_years.join('`, `')}`.",
|
|
85
|
+
key: ref
|
|
84
86
|
end
|
|
85
87
|
nil
|
|
86
88
|
end
|
|
@@ -88,10 +90,11 @@ module Relaton
|
|
|
88
90
|
#
|
|
89
91
|
# Selects the portal hits that denote the same document as the query.
|
|
90
92
|
#
|
|
91
|
-
# The search text
|
|
92
|
-
#
|
|
93
|
-
#
|
|
94
|
-
# than aborting the
|
|
93
|
+
# The search text is the reference as printed (the caller's String, or
|
|
94
|
+
# the print of a pubid from Relaton::Db) — that is search-engine input,
|
|
95
|
+
# not identifier parsing — while the SELECTION is pubid's. A hit whose
|
|
96
|
+
# code the grammar cannot read is dropped rather than aborting the
|
|
97
|
+
# search.
|
|
95
98
|
#
|
|
96
99
|
# The selection is pubid's subset match `query === hit.pubid`. A part,
|
|
97
100
|
# a year or a supplement year that the query omits matches any value.
|
|
@@ -101,13 +104,13 @@ module Relaton
|
|
|
101
104
|
# amendment's record, and a supplement reference never answers with
|
|
102
105
|
# its base document.
|
|
103
106
|
#
|
|
104
|
-
# @param
|
|
107
|
+
# @param ref [String] the printed reference, as the portal form wants it
|
|
105
108
|
# @param query [Pubid::CenCenelec::Identifier]
|
|
106
109
|
#
|
|
107
110
|
# @return [Relaton::Cen::HitCollection]
|
|
108
111
|
#
|
|
109
|
-
def search_filter(
|
|
110
|
-
search(
|
|
112
|
+
def search_filter(ref, query)
|
|
113
|
+
search(ref).select! { |hit| hit.pubid && query === hit.pubid }
|
|
111
114
|
end
|
|
112
115
|
|
|
113
116
|
# Sort through the results from Isobib, fetching them three at a time,
|
|
@@ -129,18 +132,18 @@ module Relaton
|
|
|
129
132
|
{ years: missed_years }
|
|
130
133
|
end
|
|
131
134
|
|
|
132
|
-
def bib_get(
|
|
133
|
-
|
|
134
|
-
Util.info "Fetching from standards.cencenelec.eu ...", key:
|
|
135
|
-
result = search_filter(
|
|
135
|
+
def bib_get(ref, year, opts, query) # rubocop:disable Metrics/MethodLength
|
|
136
|
+
key = year && absent?(query, :year) ? "#{ref}:#{year}" : ref
|
|
137
|
+
Util.info "Fetching from standards.cencenelec.eu ...", key: key
|
|
138
|
+
result = search_filter(ref, query)
|
|
136
139
|
ret = isobib_results_filter(result, year)
|
|
137
140
|
if ret[:ret]
|
|
138
141
|
bib = year || opts[:keep_year] ? ret[:ret] : ret[:ret].to_most_recent_reference
|
|
139
|
-
Util.info "Found: `#{bib.docidentifier.first&.content}`", key:
|
|
142
|
+
Util.info "Found: `#{bib.docidentifier.first&.content}`", key: key
|
|
140
143
|
bib
|
|
141
144
|
else
|
|
142
|
-
Util.info "Not found.", key:
|
|
143
|
-
fetch_ref_err(
|
|
145
|
+
Util.info "Not found.", key: key
|
|
146
|
+
fetch_ref_err(ref, year, ret[:years])
|
|
144
147
|
end
|
|
145
148
|
end
|
|
146
149
|
end
|
|
@@ -105,7 +105,7 @@ module Relaton
|
|
|
105
105
|
# @return [Array<RelatonCen::Hit>]
|
|
106
106
|
def hits(resp)
|
|
107
107
|
resp.xpath("//table[@class='dashlist']/tbody/tr/td[2]").map do |h|
|
|
108
|
-
ref = h.
|
|
108
|
+
ref = h.at_xpath("strong/a")
|
|
109
109
|
code = ref.text.strip
|
|
110
110
|
url = ref[:href]
|
|
111
111
|
Hit.new({ code: code, url: url }, self)
|
|
@@ -20,6 +20,11 @@ module Relaton
|
|
|
20
20
|
::Relaton::Cen::Bibliography.get(code, date, opts)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
# `Bibliography.get` applies a year to the document the reference names.
|
|
24
|
+
def fold_year(pubid, year)
|
|
25
|
+
fold_year_on_root pubid, year
|
|
26
|
+
end
|
|
27
|
+
|
|
23
28
|
# @param xml [String]
|
|
24
29
|
# @return [Relaton::Cen::ItemData]
|
|
25
30
|
def from_xml(xml)
|
data/lib/relaton/cen/scraper.rb
CHANGED
|
@@ -53,7 +53,7 @@ module Relaton
|
|
|
53
53
|
# @param doc [Mechanize::Page]
|
|
54
54
|
# @return [Array<Relaton::Bib::LocalizedMarkedUpString>]
|
|
55
55
|
def fetch_abstract(doc)
|
|
56
|
-
content = doc.
|
|
56
|
+
content = doc.at_xpath("//tr[th[.='Abstract/Scope']]/td")
|
|
57
57
|
[Bib::Abstract.new(content: content.text, language: "en", script: "Latn")]
|
|
58
58
|
end
|
|
59
59
|
|
|
@@ -68,7 +68,7 @@ module Relaton
|
|
|
68
68
|
# @param doc [Mechanize::Page]
|
|
69
69
|
# @return [Relaton::Bib::DocumentStatus, nil]
|
|
70
70
|
def fetch_status(doc)
|
|
71
|
-
s = doc.
|
|
71
|
+
s = doc.at_xpath("//tr[th[.='Status']]/td")
|
|
72
72
|
return unless s
|
|
73
73
|
|
|
74
74
|
stage = Bib::Status::Stage.new(content: s.text.strip)
|
|
@@ -98,7 +98,7 @@ module Relaton
|
|
|
98
98
|
doc.xpath(
|
|
99
99
|
"//div[@id='DASHBOARD_LISTRELATIONS']/table/tr[th[.!='Sales Points']]",
|
|
100
100
|
).each_with_object([]) do |rt, acc|
|
|
101
|
-
type = relation_type rt.
|
|
101
|
+
type = relation_type rt.at_xpath("th").text.downcase
|
|
102
102
|
rt.xpath("td/a").each do |r|
|
|
103
103
|
acc << Bib::Relation.new(type: type, bibitem: create_relation(r))
|
|
104
104
|
end
|
|
@@ -124,7 +124,7 @@ module Relaton
|
|
|
124
124
|
# @param doc [Mechanize::Page]
|
|
125
125
|
# @return [Array<Relaton::Bib::Title>]
|
|
126
126
|
def fetch_titles(doc)
|
|
127
|
-
te = doc.
|
|
127
|
+
te = doc.at_xpath("//tr[th[.='Title']]/td").text.strip
|
|
128
128
|
Bib::Title.from_string te, "en", "Latn"
|
|
129
129
|
end
|
|
130
130
|
|
|
@@ -134,10 +134,10 @@ module Relaton
|
|
|
134
134
|
def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
|
|
135
135
|
doc.xpath("//div[@id='DASHBOARD_LISTIMPLEMENTATIONDATES']/table/tr")
|
|
136
136
|
.each_with_object([]) do |d, acc|
|
|
137
|
-
at = d.
|
|
137
|
+
at = d.at_xpath("td").text
|
|
138
138
|
next if at.empty?
|
|
139
139
|
|
|
140
|
-
t = d.
|
|
140
|
+
t = d.at_xpath("th").text
|
|
141
141
|
type = case t
|
|
142
142
|
when /DOR/ then "adapted"
|
|
143
143
|
when /DAV/ then "issued"
|
|
@@ -154,8 +154,8 @@ module Relaton
|
|
|
154
154
|
# @param doc [Mechanize::Page]
|
|
155
155
|
# @return [Array<Relaton::Bib::Contributor>]
|
|
156
156
|
def fetch_contributors(doc)
|
|
157
|
-
code = doc.
|
|
158
|
-
title = doc.
|
|
157
|
+
code = doc.at_xpath("//tr/td/h1/text()").text
|
|
158
|
+
title = doc.at_xpath("//tr/td[3]/h1").text
|
|
159
159
|
%r{/(?<type>\w+)(?:\s(?<num>[^/]+))?$} =~ code
|
|
160
160
|
org = owner_entity
|
|
161
161
|
|
|
@@ -190,7 +190,7 @@ module Relaton
|
|
|
190
190
|
# @param doc [Mechanize::Page]
|
|
191
191
|
# @return [Array<Bib::Copyright>]
|
|
192
192
|
def fetch_copyright(doc)
|
|
193
|
-
date = doc.
|
|
193
|
+
date = doc.at_xpath("//tr[th[.='date of Availability (DAV)']]/td").text
|
|
194
194
|
owner = Bib::ContributionInfo.new(organization: owner_entity)
|
|
195
195
|
from = date.match(/^\d{4}/).to_s
|
|
196
196
|
[Bib::Copyright.new(owner: [owner], from: from)]
|