relaton 3.0.0.pre.alpha.4 → 3.0.0.pre.alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/relaton/3gpp/bibliography.rb +19 -6
- data/lib/relaton/3gpp/processor.rb +6 -0
- data/lib/relaton/adobe/processor.rb +9 -0
- data/lib/relaton/bib/converter/csl.rb +110 -0
- data/lib/relaton/bib/converter/ris.rb +104 -0
- data/lib/relaton/bib/converter/titles.rb +24 -0
- data/lib/relaton/bib/item_data.rb +23 -0
- data/lib/relaton/bib/model/item.rb +6 -6
- data/lib/relaton/bib/sanitizer.rb +71 -30
- data/lib/relaton/bib.rb +10 -0
- data/lib/relaton/bipm/bibliography.rb +24 -21
- data/lib/relaton/bipm/processor.rb +1 -0
- data/lib/relaton/bipm/rawdata_bipm_metrologia/affiliations.rb +6 -6
- data/lib/relaton/bipm/si_brochure_parser.rb +2 -2
- data/lib/relaton/bsi/bibliography.rb +26 -23
- data/lib/relaton/bsi/processor.rb +5 -0
- data/lib/relaton/calconnect/bibliography.rb +6 -6
- data/lib/relaton/calconnect/hit_collection.rb +4 -1
- data/lib/relaton/ccsds/bibliography.rb +16 -10
- data/lib/relaton/ccsds/hit_collection.rb +1 -1
- data/lib/relaton/ccsds/processor.rb +13 -0
- data/lib/relaton/cen/bibliography.rb +27 -24
- data/lib/relaton/cen/hit_collection.rb +1 -1
- data/lib/relaton/cen/processor.rb +5 -0
- data/lib/relaton/cen/scraper.rb +9 -9
- data/lib/relaton/cie/bibliography.rb +11 -9
- data/lib/relaton/cie/data_fetcher.rb +18 -18
- data/lib/relaton/cie/processor.rb +1 -0
- data/lib/relaton/cie/scrapper.rb +15 -8
- data/lib/relaton/cie.rb +1 -1
- data/lib/relaton/cloud.rb +127 -0
- data/lib/relaton/core/hit_collection.rb +6 -10
- data/lib/relaton/core/processor.rb +121 -0
- data/lib/relaton/core/request_error.rb +21 -3
- data/lib/relaton/db/cache.rb +474 -148
- data/lib/relaton/db/cache_entry.rb +42 -0
- data/lib/relaton/db/registry.rb +152 -0
- data/lib/relaton/db.rb +275 -102
- data/lib/relaton/doi/crossref.rb +23 -6
- data/lib/relaton/doi/processor.rb +1 -0
- data/lib/relaton/easc/processor.rb +1 -0
- data/lib/relaton/ecma/bibliography.rb +18 -13
- data/lib/relaton/ecma/data_fetcher.rb +1 -1
- data/lib/relaton/ecma/data_parser.rb +1 -1
- data/lib/relaton/ecma/edition_parser.rb +2 -2
- data/lib/relaton/ecma/memento_parser.rb +4 -4
- data/lib/relaton/ecma/standard_parser.rb +6 -6
- data/lib/relaton/etsi/bibliography.rb +8 -7
- data/lib/relaton/etsi/processor.rb +1 -0
- data/lib/relaton/gb/bibliography.rb +20 -12
- data/lib/relaton/gb/gb_scraper.rb +5 -5
- data/lib/relaton/gb/scraper.rb +16 -16
- data/lib/relaton/gb/sec_scraper.rb +8 -8
- data/lib/relaton/gb/t_scraper.rb +5 -5
- data/lib/relaton/gost/processor.rb +1 -0
- data/lib/relaton/iala/bibliography.rb +15 -12
- data/lib/relaton/iala/processor.rb +1 -0
- data/lib/relaton/iana/bibliography.rb +12 -12
- data/lib/relaton/iana/data_fetcher.rb +3 -3
- data/lib/relaton/iana/parser.rb +12 -5
- data/lib/relaton/iana/processor.rb +9 -0
- data/lib/relaton/iec/bibliography.rb +18 -7
- data/lib/relaton/iec/data_parser.rb +24 -9
- data/lib/relaton/iec/processor.rb +11 -0
- data/lib/relaton/iec.rb +1 -1
- data/lib/relaton/ieee/bibliography.rb +19 -13
- data/lib/relaton/ieee/data_fetcher.rb +1 -1
- data/lib/relaton/ieee/processor.rb +24 -0
- data/lib/relaton/ieee/rawbib_id_parser.rb +2 -2
- data/lib/relaton/ietf/bibliography.rb +11 -9
- data/lib/relaton/ietf/data_fetcher.rb +1 -1
- data/lib/relaton/ietf/processor.rb +1 -0
- data/lib/relaton/ietf/rfc/entry.rb +15 -19
- data/lib/relaton/ietf/scraper.rb +5 -4
- data/lib/relaton/iho/processor.rb +1 -0
- data/lib/relaton/index/file_io.rb +2 -2
- data/lib/relaton/index/pool.rb +4 -3
- data/lib/relaton/index/shard_source.rb +1 -1
- data/lib/relaton/index/type.rb +2 -5
- data/lib/relaton/isbn/open_library.rb +11 -7
- data/lib/relaton/isbn/processor.rb +10 -0
- data/lib/relaton/iso/bibliography.rb +11 -9
- data/lib/relaton/iso/data_parser.rb +2 -2
- data/lib/relaton/iso/processor.rb +5 -0
- data/lib/relaton/iso/scraper.rb +20 -20
- data/lib/relaton/itu/bibliography.rb +107 -16
- data/lib/relaton/itu/data_crawler_r.rb +2 -2
- data/lib/relaton/itu/hit_collection.rb +64 -52
- data/lib/relaton/itu/processor.rb +1 -0
- data/lib/relaton/itu/scraper.rb +13 -3
- data/lib/relaton/itu.rb +0 -2
- data/lib/relaton/jis/bibliography.rb +22 -11
- data/lib/relaton/jis/data_fetcher.rb +4 -4
- data/lib/relaton/jis/processor.rb +1 -0
- data/lib/relaton/jis/scraper.rb +8 -8
- data/lib/relaton/nist/bibliography.rb +25 -21
- data/lib/relaton/oasis/bibliography.rb +16 -13
- data/lib/relaton/oasis/browser_agent.rb +2 -2
- data/lib/relaton/oasis/data_parser.rb +5 -5
- data/lib/relaton/oasis/data_parser_utils.rb +2 -2
- data/lib/relaton/oasis/data_part_parser.rb +7 -7
- data/lib/relaton/ogc/bibliography.rb +15 -14
- data/lib/relaton/ogc/hit_collection.rb +9 -7
- data/lib/relaton/ogc/processor.rb +13 -0
- data/lib/relaton/oiml/processor.rb +1 -0
- data/lib/relaton/omg/bibliography.rb +9 -9
- data/lib/relaton/omg/scraper.rb +14 -13
- data/lib/relaton/omg.rb +1 -1
- data/lib/relaton/plateau/bibliography.rb +9 -7
- data/lib/relaton/plateau/hit_collection.rb +2 -1
- data/lib/relaton/plateau/processor.rb +1 -0
- data/lib/relaton/un/bibliography.rb +21 -10
- data/lib/relaton/un/processor.rb +1 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/processor.rb +1 -0
- data/lib/relaton/xsf/bibliography.rb +11 -6
- data/lib/relaton.rb +13 -0
- metadata +60 -14
- data/lib/relaton/itu/pubid.rb +0 -199
|
@@ -15,10 +15,10 @@ module Relaton
|
|
|
15
15
|
module Bibliography
|
|
16
16
|
SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
|
|
17
17
|
|
|
18
|
-
# @param
|
|
18
|
+
# @param ref [String, Pubid::Iana::Identifier]
|
|
19
19
|
# @return [Relaton::Iana::Item, nil]
|
|
20
|
-
def search(
|
|
21
|
-
pubid = parse_ref
|
|
20
|
+
def search(ref)
|
|
21
|
+
pubid = parse_ref ref
|
|
22
22
|
return unless pubid
|
|
23
23
|
|
|
24
24
|
# Pass the pubid so Relaton::Index narrows candidates by number via
|
|
@@ -34,19 +34,19 @@ module Relaton
|
|
|
34
34
|
raise Relaton::RequestError, e.message
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
# @param ref [String] the IANA registry slug to look up
|
|
37
|
+
# @param ref [String, Pubid::Iana::Identifier] the IANA registry slug to look up
|
|
38
38
|
# @param year [String, NilClass] not used
|
|
39
39
|
# @param opts [Hash] options
|
|
40
40
|
# @return [Relaton::Iana::Item, nil]
|
|
41
41
|
def get(ref, _year = nil, _opts = {})
|
|
42
|
-
Util.info "Fetching from Relaton repository ...", key: ref
|
|
42
|
+
Util.info "Fetching from Relaton repository ...", key: ref.to_s
|
|
43
43
|
result = search(ref)
|
|
44
44
|
unless result
|
|
45
|
-
Util.info "Not found.", key: ref
|
|
45
|
+
Util.info "Not found.", key: ref.to_s
|
|
46
46
|
return
|
|
47
47
|
end
|
|
48
48
|
|
|
49
|
-
Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
|
|
49
|
+
Util.info "Found: `#{result.docidentifier[0].content}`", key: ref.to_s
|
|
50
50
|
result
|
|
51
51
|
end
|
|
52
52
|
|
|
@@ -63,14 +63,14 @@ module Relaton
|
|
|
63
63
|
# RuntimeError on anything else, and a citation relaton cannot parse is a
|
|
64
64
|
# miss, not an error — `#get` reports it as "Not found.".
|
|
65
65
|
#
|
|
66
|
-
# @param
|
|
66
|
+
# @param ref [String, Pubid::Iana::Identifier]
|
|
67
67
|
# @return [Pubid::Iana::Identifier, nil]
|
|
68
|
-
def parse_ref(
|
|
69
|
-
return
|
|
68
|
+
def parse_ref(ref)
|
|
69
|
+
return ref unless ref.is_a? String
|
|
70
70
|
|
|
71
|
-
::Pubid::Iana::Identifier.parse
|
|
71
|
+
::Pubid::Iana::Identifier.parse ref
|
|
72
72
|
rescue StandardError => e
|
|
73
|
-
Util.warn "Failed to parse pubid `#{
|
|
73
|
+
Util.warn "Failed to parse pubid `#{ref}`: #{e.message}"
|
|
74
74
|
nil
|
|
75
75
|
end
|
|
76
76
|
|
|
@@ -51,11 +51,11 @@ module Relaton
|
|
|
51
51
|
end
|
|
52
52
|
|
|
53
53
|
def parse(content)
|
|
54
|
-
xml =
|
|
55
|
-
registry = xml.
|
|
54
|
+
xml = Moxml.parse(content)
|
|
55
|
+
registry = xml.at_xpath("/xmlns:registry", NS)
|
|
56
56
|
doc = Parser.parse registry, nil, @errors
|
|
57
57
|
save_doc doc
|
|
58
|
-
registry.xpath("./xmlns:registry").each { |r| save_doc Parser.parse(r, doc, @errors) }
|
|
58
|
+
registry.xpath("./xmlns:registry", NS).each { |r| save_doc Parser.parse(r, doc, @errors) }
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
#
|
data/lib/relaton/iana/parser.rb
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
module Relaton
|
|
2
2
|
module Iana
|
|
3
|
+
# The IANA registry XML uses a default namespace, and moxml (unlike raw
|
|
4
|
+
# nokogiri) does not bind the literal `xmlns:` prefix to it — every
|
|
5
|
+
# namespaced query passes this binding explicitly.
|
|
6
|
+
NS = { "xmlns" => "http://www.iana.org/assignments" }.freeze
|
|
3
7
|
class Parser
|
|
4
8
|
#
|
|
5
9
|
# Document parser initalization
|
|
6
10
|
#
|
|
7
|
-
# @param [
|
|
11
|
+
# @param [Moxml::Element] xml
|
|
8
12
|
#
|
|
9
13
|
def initialize(xml, rootdoc, errors = {})
|
|
10
14
|
@xml = xml
|
|
@@ -15,7 +19,7 @@ module Relaton
|
|
|
15
19
|
#
|
|
16
20
|
# Initialize document parser and run it
|
|
17
21
|
#
|
|
18
|
-
# @param [
|
|
22
|
+
# @param [Moxml::Element] xml
|
|
19
23
|
#
|
|
20
24
|
# @return [Relaton::Iana::ItemData, nil] bibliographic item
|
|
21
25
|
#
|
|
@@ -50,7 +54,7 @@ module Relaton
|
|
|
50
54
|
# @return [Array<Relaton::Bib::Title>] title
|
|
51
55
|
#
|
|
52
56
|
def parse_title
|
|
53
|
-
content = @xml.
|
|
57
|
+
content = @xml.at_xpath("./xmlns:title", NS)&.text || @xml[:id]
|
|
54
58
|
result = [Bib::Title.new(content: content, language: "en", script: "Latn")]
|
|
55
59
|
@errors[:title] &&= result.empty?
|
|
56
60
|
result
|
|
@@ -121,8 +125,11 @@ module Relaton
|
|
|
121
125
|
# @return [Array<Relaton::Bib::Date>] date
|
|
122
126
|
#
|
|
123
127
|
def parse_date
|
|
124
|
-
|
|
125
|
-
|
|
128
|
+
# moxml 0.5.98 drops every branch after the second in an N>=3 union
|
|
129
|
+
# under the reserved `xmlns` prefix (lutaml/moxml#315), which silently
|
|
130
|
+
# stripped `updated` dates from every crawled registry.
|
|
131
|
+
d = %w[created published updated].flat_map do |type|
|
|
132
|
+
@xml.xpath("./xmlns:#{type}", NS).map { |dt| Bib::Date.new(type: dt.name, at: dt.text) }
|
|
126
133
|
end
|
|
127
134
|
result = d.none? && @rootdoc ? @rootdoc.date : d
|
|
128
135
|
@errors[:date] &&= result.empty?
|
|
@@ -8,6 +8,7 @@ module Relaton
|
|
|
8
8
|
def initialize # rubocop:disable Lint/MissingSuper
|
|
9
9
|
@short = :relaton_iana
|
|
10
10
|
@prefix = "IANA"
|
|
11
|
+
@pubid_identifier = :Iana # Db cache key
|
|
11
12
|
@defaultprefix = %r{^IANA\s}
|
|
12
13
|
@idtype = "IANA"
|
|
13
14
|
@datasets = %w[iana-registries]
|
|
@@ -35,6 +36,14 @@ module Relaton
|
|
|
35
36
|
DataFetcher.fetch(source, **opts)
|
|
36
37
|
end
|
|
37
38
|
|
|
39
|
+
# `get` reads a reference pubid cannot parse as a miss, so it gets no key
|
|
40
|
+
# here either: it is not cached.
|
|
41
|
+
def cache_pubid(ref)
|
|
42
|
+
super
|
|
43
|
+
rescue StandardError
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
38
47
|
# @param xml [String]
|
|
39
48
|
# @return [Relaton::Iana::ItemData]
|
|
40
49
|
def from_xml(xml)
|
|
@@ -24,18 +24,29 @@ module Relaton
|
|
|
24
24
|
raise Relaton::RequestError, e.message
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
-
# @param
|
|
27
|
+
# @param ref [String, Pubid::Iec::Identifier] the IEC standard code to
|
|
28
|
+
# look up (e.g. "IEC 8000"), or the parse that Relaton::Db routed
|
|
29
|
+
# with (relaton#205); a pubid is never mutated
|
|
28
30
|
# @param year [String] the year the standard was published (optional)
|
|
29
31
|
# @param opts [Hash] options; restricted to :all_parts if all-parts
|
|
30
32
|
# reference is required
|
|
31
33
|
# @return [Relaton::Iec::ItemData, nil]
|
|
32
|
-
def get(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
34
|
+
def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
|
|
35
|
+
if ref.is_a?(String)
|
|
36
|
+
opts[:all_parts] ||= ref.match?(/\s\(all parts\)/)
|
|
37
|
+
text = ref.sub(/\s\(all parts\)/, "")
|
|
38
|
+
return iev if text.casecmp("IEV").zero?
|
|
39
|
+
|
|
40
|
+
pubid = ::Pubid::Iec::Identifier.parse text.upcase
|
|
41
|
+
elsif ref.all_parts?
|
|
42
|
+
opts[:all_parts] = true
|
|
43
|
+
pubid = ref.identifiers.first
|
|
44
|
+
else
|
|
45
|
+
pubid = ref
|
|
46
|
+
end
|
|
38
47
|
if year
|
|
48
|
+
# Copy first: a caller's pubid is also its Db cache key.
|
|
49
|
+
pubid = pubid.class.from_hash(pubid.to_hash)
|
|
39
50
|
pubid.date = ::Pubid::Components::Date.new(year: year.to_s)
|
|
40
51
|
end
|
|
41
52
|
|
|
@@ -289,38 +289,53 @@ module Relaton
|
|
|
289
289
|
#
|
|
290
290
|
# @return [Array<Relaton::Bib::Relation>] relation
|
|
291
291
|
#
|
|
292
|
-
def relation
|
|
292
|
+
def relation
|
|
293
|
+
result = fetch_relations
|
|
294
|
+
@errors[:relation] &&= result.empty?
|
|
295
|
+
result
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
#
|
|
299
|
+
# Fetch relations from the webstore. Retry on network errors only.
|
|
300
|
+
# A non-success response (the legacy endpoint now redirects to a 404
|
|
301
|
+
# HTML page) or a malformed body gives no relations.
|
|
302
|
+
#
|
|
303
|
+
# @return [Array<Relaton::Bib::Relation>] relations
|
|
304
|
+
#
|
|
305
|
+
def fetch_relations # rubocop:disable Metrics/MethodLength
|
|
293
306
|
try = 0
|
|
294
|
-
|
|
307
|
+
begin
|
|
295
308
|
uri = URI "#{DOMAIN}/webstore/webstore.nsf/AjaxRequestXML?Openagent&url=#{urn_id}"
|
|
296
309
|
resp = Net::HTTP.get_response uri
|
|
297
|
-
doc = Nokogiri::XML resp.body
|
|
298
|
-
create_relations doc
|
|
299
310
|
rescue StandardError => e
|
|
300
311
|
try += 1
|
|
301
312
|
try < 3 ? retry : raise(e)
|
|
302
313
|
end
|
|
303
|
-
|
|
304
|
-
|
|
314
|
+
return [] if resp.is_a?(Net::HTTPResponse) && !resp.is_a?(Net::HTTPSuccess)
|
|
315
|
+
|
|
316
|
+
create_relations Moxml.parse(resp.body)
|
|
317
|
+
rescue Moxml::ParseError => e
|
|
318
|
+
Util.warn "Failed to parse relations for `#{urn_id}`: #{e.message.lines.first&.strip}"
|
|
319
|
+
[]
|
|
305
320
|
end
|
|
306
321
|
|
|
307
322
|
#
|
|
308
323
|
# Create relations.
|
|
309
324
|
#
|
|
310
|
-
# @param [
|
|
325
|
+
# @param [Moxml::Document] doc XML document
|
|
311
326
|
#
|
|
312
327
|
# @return [Array<Relaton::Bib::Relation>] relations
|
|
313
328
|
#
|
|
314
329
|
def create_relations(doc) # rubocop:disable Metrics/MethodLength
|
|
315
330
|
doc.xpath('//ROW[STATUS[.!="PREPARING" and .!="PUBLISHED"]]')
|
|
316
331
|
.map do |r|
|
|
317
|
-
r_type = r.
|
|
332
|
+
r_type = r.at_xpath("STATUS").text.downcase
|
|
318
333
|
type = case r_type
|
|
319
334
|
when "revised", "replaced" then "updates"
|
|
320
335
|
when "withdrawn" then "obsoletes"
|
|
321
336
|
else r_type
|
|
322
337
|
end
|
|
323
|
-
ref = r.
|
|
338
|
+
ref = r.at_xpath("FULL_NAME").text
|
|
324
339
|
docid = Docidentifier.new(content: ref, type: "IEC", primary: true)
|
|
325
340
|
bibitem = ItemData.new(formattedref: Bib::Formattedref.new(content: ref), docidentifier: [docid])
|
|
326
341
|
Relation.new type: type, bibitem: bibitem
|
|
@@ -34,6 +34,12 @@ module Relaton
|
|
|
34
34
|
DataFetcher.fetch(source, **opts)
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
+
# `IEV` is not a document identifier (`get` answers it with the IEV
|
|
38
|
+
# vocabulary), so it gets no key: it is not cached.
|
|
39
|
+
def cache_pubid(ref)
|
|
40
|
+
ref.strip.casecmp?("IEV") ? nil : super
|
|
41
|
+
end
|
|
42
|
+
|
|
37
43
|
# @param xml [String]
|
|
38
44
|
# @return [Relaton::Iec::ItemData]
|
|
39
45
|
def from_xml(xml)
|
|
@@ -57,7 +63,12 @@ module Relaton
|
|
|
57
63
|
|
|
58
64
|
# @param code [String]
|
|
59
65
|
# @return [String, nil]
|
|
66
|
+
# Only a URN: `Relaton::Iec.urn_to_code` splits on `:`, so it would
|
|
67
|
+
# rewrite any reference with enough colons
|
|
68
|
+
# (`IEC 60034-1:1969+AMD1:1977+AMD2:1979+AMD3:1980 CSV`).
|
|
60
69
|
def urn_to_code(code)
|
|
70
|
+
return unless code.match?(/\Aurn:/i)
|
|
71
|
+
|
|
61
72
|
require_relative "../iec"
|
|
62
73
|
Relaton::Iec.urn_to_code code
|
|
63
74
|
end
|
data/lib/relaton/iec.rb
CHANGED
|
@@ -7,11 +7,12 @@ module Relaton
|
|
|
7
7
|
#
|
|
8
8
|
# Search IEEE bibliography item by reference.
|
|
9
9
|
#
|
|
10
|
-
# @param
|
|
10
|
+
# @param ref [String, Pubid::Ieee::Identifier] the reference, or its
|
|
11
|
+
# parse from Relaton::Db (relaton#205)
|
|
11
12
|
#
|
|
12
13
|
# @return [Relaton::Ieee::ItemData, nil]
|
|
13
14
|
#
|
|
14
|
-
def search(
|
|
15
|
+
def search(ref) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
15
16
|
index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
|
|
16
17
|
pubid_class: ::Pubid::Ieee::Identifier
|
|
17
18
|
# Pass the parsed pubid (not the raw String) so index-v2 narrows
|
|
@@ -20,7 +21,7 @@ module Relaton
|
|
|
20
21
|
# unparseable/partial ref falls back to the full-scan String search.
|
|
21
22
|
# Rows are Pubid::Ieee::Identifier objects (not Comparable), so pick by
|
|
22
23
|
# the string form.
|
|
23
|
-
pubid = parse_pubid
|
|
24
|
+
pubid = parse_pubid ref
|
|
24
25
|
needle = pubid.to_s
|
|
25
26
|
row = index.search(pubid) { |r| r[:id].to_s.include?(needle) }.min_by { |r| r[:id].to_s }
|
|
26
27
|
return unless row
|
|
@@ -36,20 +37,22 @@ module Relaton
|
|
|
36
37
|
#
|
|
37
38
|
# Get IEEE bibliography item by reference.
|
|
38
39
|
#
|
|
39
|
-
# @param
|
|
40
|
+
# @param ref [String, Pubid::Ieee::Identifier] the IEEE standard
|
|
41
|
+
# reference to look up (e.g. "IEEE 528-2019"), or its parse from
|
|
42
|
+
# Relaton::Db (relaton#205)
|
|
40
43
|
# @param year [String] the year the standard was published (optional)
|
|
41
44
|
# @param opts [Hash] options
|
|
42
45
|
#
|
|
43
46
|
# @return [Relaton::Ieee::ItemData, nil]
|
|
44
47
|
#
|
|
45
|
-
def get(
|
|
46
|
-
Util.info "Fetching from Relaton repository ...", key:
|
|
47
|
-
item = search(
|
|
48
|
+
def get(ref, _year = nil, _opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
49
|
+
Util.info "Fetching from Relaton repository ...", key: ref.to_s
|
|
50
|
+
item = search(ref)
|
|
48
51
|
if item
|
|
49
|
-
Util.info "Found: `#{item.docidentifier.first.content}`", key:
|
|
52
|
+
Util.info "Found: `#{item.docidentifier.first.content}`", key: ref.to_s
|
|
50
53
|
item
|
|
51
54
|
else
|
|
52
|
-
Util.info "Not found.", key:
|
|
55
|
+
Util.info "Not found.", key: ref.to_s
|
|
53
56
|
nil
|
|
54
57
|
end
|
|
55
58
|
end
|
|
@@ -60,12 +63,15 @@ module Relaton
|
|
|
60
63
|
# return the raw String when pubid can't parse it (e.g. a partial ref) so
|
|
61
64
|
# the search falls back to the substring scan.
|
|
62
65
|
#
|
|
63
|
-
# @param
|
|
66
|
+
# @param ref [String, ::Pubid::Ieee::Identifier]
|
|
64
67
|
# @return [::Pubid::Ieee::Identifier, String]
|
|
65
|
-
def parse_pubid(
|
|
66
|
-
::
|
|
68
|
+
def parse_pubid(ref)
|
|
69
|
+
# A pubid from Relaton::Db (relaton#205) is used as it is.
|
|
70
|
+
return ref unless ref.is_a?(String)
|
|
71
|
+
|
|
72
|
+
::Pubid::Ieee::Identifier.parse ref
|
|
67
73
|
rescue StandardError
|
|
68
|
-
|
|
74
|
+
ref
|
|
69
75
|
end
|
|
70
76
|
end
|
|
71
77
|
end
|
|
@@ -147,7 +147,7 @@ module Relaton
|
|
|
147
147
|
# IdamsParser#parse_relation, so mutates crossrefs under a mutex.
|
|
148
148
|
#
|
|
149
149
|
# @param [String] docnumber of main document
|
|
150
|
-
# @param [
|
|
150
|
+
# @param [Moxml::Element] amsid relation data
|
|
151
151
|
#
|
|
152
152
|
def add_crossref(docnumber, amsid)
|
|
153
153
|
return if RELATION_TYPES[amsid.type] == false
|
|
@@ -36,6 +36,30 @@ module Relaton
|
|
|
36
36
|
DataFetcher.fetch(source, **opts)
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
+
# `get` reads a reference pubid cannot parse as a miss, so it gets no key
|
|
40
|
+
# here either: it is not cached.
|
|
41
|
+
def cache_pubid(ref)
|
|
42
|
+
super
|
|
43
|
+
rescue StandardError
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# An IEEE pubid keeps the other publishers in `publishers` (a joint
|
|
48
|
+
# development) or `copublisher` (a standard), not in the shared
|
|
49
|
+
# `copublishers`.
|
|
50
|
+
#
|
|
51
|
+
# @param pubid [Pubid::Ieee::Identifier]
|
|
52
|
+
# @return [Array<String>]
|
|
53
|
+
def copublishers(pubid)
|
|
54
|
+
pubid = pubid.identifiers.first if pubid.all_parts?
|
|
55
|
+
others = if pubid.respond_to?(:publishers) && pubid.publishers
|
|
56
|
+
pubid.publishers
|
|
57
|
+
elsif pubid.respond_to?(:copublisher)
|
|
58
|
+
pubid.copublisher
|
|
59
|
+
end
|
|
60
|
+
Array(others).map(&:to_s) - %w[IEEE AIEE]
|
|
61
|
+
end
|
|
62
|
+
|
|
39
63
|
# @param xml [String]
|
|
40
64
|
# @return [Relaton::Ieee::ItemData]
|
|
41
65
|
def from_xml(xml)
|
|
@@ -138,8 +138,8 @@ module Relaton
|
|
|
138
138
|
Renderer.new(publisher: "IEEE", number: "P802", part: "11aj", draft: "8.0", year: "2017")
|
|
139
139
|
when "IEEE P802.11ajD9.0, November 2017" # "IEEE P802-11aj/D9.0.2017"
|
|
140
140
|
Renderer.new(publisher: "IEEE", number: "P802", part: "11aj", draft: "9.0", year: "2017")
|
|
141
|
-
when "ISO/IEC/IEEE P29119-4-DISMay2013" # "ISO/IEC/IEEE DIS P29119-4.2013"
|
|
142
|
-
Renderer.new(publisher: "ISO/IEC/IEEE", stage: "DIS", number: "P29119", part: "4", year: "2013")
|
|
141
|
+
when "ISO/IEC/IEEE P29119-4-DISMay2013" # "ISO/IEC/IEEE DIS P29119-4.2013-05"
|
|
142
|
+
Renderer.new(publisher: "ISO/IEC/IEEE", stage: "DIS", number: "P29119", part: "4", year: "2013", month: "05")
|
|
143
143
|
when "IEEE-P15026-3-DIS-January 2015" # "IEEE DIS P15026-3.2015"
|
|
144
144
|
Renderer.new(publisher: "IEEE", stage: "DIS", number: "P15026", year: "2015")
|
|
145
145
|
when "ANSI/IEEE PC63.7/D rev17, December 2014" # "ANSI/IEEE PC63-7/D/REV-17.2014"
|
|
@@ -7,25 +7,27 @@ module Relaton
|
|
|
7
7
|
# IETF bibliography module
|
|
8
8
|
module Bibliography
|
|
9
9
|
class << self
|
|
10
|
-
# @param
|
|
10
|
+
# @param ref [String, Pubid::Ietf::Identifier] the IETF reference
|
|
11
|
+
# (e.g. "RFC 8341"), or its parse from Relaton::Db (relaton#205)
|
|
11
12
|
# @return [RelatonIetf::IetfBibliographicItem]
|
|
12
|
-
def search(
|
|
13
|
-
Scraper.scrape_page
|
|
13
|
+
def search(ref)
|
|
14
|
+
Scraper.scrape_page ref
|
|
14
15
|
end
|
|
15
16
|
|
|
16
|
-
# @param
|
|
17
|
+
# @param ref [String, Pubid::Ietf::Identifier] the IETF reference
|
|
18
|
+
# (e.g. "RFC 8341"), or its parse from Relaton::Db (relaton#205)
|
|
17
19
|
# @param year [String] the year the standard was published (optional)
|
|
18
20
|
# @param opts [Hash] options; restricted to :all_parts if all-parts
|
|
19
21
|
# reference is required
|
|
20
22
|
# @return [RelatonIetf::IetfBibliographicItem] Relaton of reference
|
|
21
|
-
def get(
|
|
22
|
-
Util.info "Fetching from Relaton repository ...", key:
|
|
23
|
-
result = search
|
|
23
|
+
def get(ref, _year = nil, _opts = {})
|
|
24
|
+
Util.info "Fetching from Relaton repository ...", key: ref.to_s
|
|
25
|
+
result = search ref
|
|
24
26
|
if result
|
|
25
27
|
docid = result.docidentifier.detect(&:primary) || result.docidentifier.first
|
|
26
|
-
Util.info "Found: `#{docid.content}`", key:
|
|
28
|
+
Util.info "Found: `#{docid.content}`", key: ref.to_s
|
|
27
29
|
else
|
|
28
|
-
Util.info "Not found.", key:
|
|
30
|
+
Util.info "Not found.", key: ref.to_s
|
|
29
31
|
end
|
|
30
32
|
result
|
|
31
33
|
end
|
|
@@ -6,6 +6,7 @@ module Relaton
|
|
|
6
6
|
def initialize # rubocop:disable Lint/MissingSuper
|
|
7
7
|
@short = :relaton_ietf
|
|
8
8
|
@prefix = "IETF"
|
|
9
|
+
@pubid_identifier = :Ietf # Db cache key
|
|
9
10
|
@defaultprefix = /^((IETF|RFC|BCP|FYI|STD)\s|I-D[.\s])/
|
|
10
11
|
@idtype = "IETF"
|
|
11
12
|
@datasets = %w[ietf-rfcsubseries ietf-internet-drafts ietf-rfc-entries]
|
|
@@ -195,8 +195,7 @@ module Relaton
|
|
|
195
195
|
formattedref: build_formattedref,
|
|
196
196
|
date: build_subseries_date(rfc_index),
|
|
197
197
|
relation: build_relations(rfc_index, wg_names: wg_names),
|
|
198
|
-
|
|
199
|
-
ext: Ext.new(doctype: Doctype.new(content: "rfc"), stream: stream, flavor: "ietf"),
|
|
198
|
+
ext: Ext.new(doctype: Doctype.new(content: "rfc"), stream: canonical_stream, flavor: "ietf"),
|
|
200
199
|
)
|
|
201
200
|
end
|
|
202
201
|
|
|
@@ -259,13 +258,6 @@ module Relaton
|
|
|
259
258
|
ItemData.new(formattedref: Bib::Formattedref.new(content: ref), docidentifier: [docid])
|
|
260
259
|
end
|
|
261
260
|
|
|
262
|
-
def build_series
|
|
263
|
-
return [] unless stream
|
|
264
|
-
|
|
265
|
-
t = Bib::Title.new(content: stream)
|
|
266
|
-
[Bib::Series.new(type: "stream", title: [t])]
|
|
267
|
-
end
|
|
268
|
-
|
|
269
261
|
# --- RFC entry builders ---
|
|
270
262
|
|
|
271
263
|
def build_rfc_docid
|
|
@@ -388,7 +380,7 @@ module Relaton
|
|
|
388
380
|
def build_rfc_series
|
|
389
381
|
series = build_rfc_is_also_series
|
|
390
382
|
series << Bib::Series.new(title: [Bib::Title.new(content: "RFC")], number: shortnum)
|
|
391
|
-
series
|
|
383
|
+
series
|
|
392
384
|
end
|
|
393
385
|
|
|
394
386
|
def build_rfc_is_also_series
|
|
@@ -401,27 +393,31 @@ module Relaton
|
|
|
401
393
|
end
|
|
402
394
|
end
|
|
403
395
|
|
|
404
|
-
def build_rfc_stream_series
|
|
405
|
-
return [] unless stream
|
|
406
|
-
|
|
407
|
-
t = Bib::Title.new(content: stream)
|
|
408
|
-
[Bib::Series.new(type: "stream", title: [t])]
|
|
409
|
-
end
|
|
410
|
-
|
|
411
396
|
STREAM_ORGS = {
|
|
412
397
|
"IETF" => ["IETF", "Internet Engineering Task Force"],
|
|
413
398
|
"IRTF" => ["IRTF", "Internet Research Task Force"],
|
|
414
399
|
"IAB" => ["IAB", "Internet Architecture Board"],
|
|
415
400
|
}.freeze
|
|
416
401
|
|
|
402
|
+
# rfc-index.xml spells the independent stream "INDEPENDENT";
|
|
403
|
+
# Ext.stream's declared values use "Independent". Match case-insensitively
|
|
404
|
+
# against the declared values so both Ext.stream and STREAM_ORGS see the
|
|
405
|
+
# canonical spelling.
|
|
406
|
+
def canonical_stream
|
|
407
|
+
return unless stream
|
|
408
|
+
|
|
409
|
+
values = Ext.attributes[:stream].options[:values]
|
|
410
|
+
values.find { |v| v.casecmp?(stream) } || stream
|
|
411
|
+
end
|
|
412
|
+
|
|
417
413
|
def build_rfc_ext
|
|
418
|
-
Ext.new(doctype: Doctype.new(content: "rfc"), stream:
|
|
414
|
+
Ext.new(doctype: Doctype.new(content: "rfc"), stream: canonical_stream, flavor: "ietf")
|
|
419
415
|
end
|
|
420
416
|
|
|
421
417
|
def build_committee_contributor(wg_names = {})
|
|
422
418
|
return if wg_acronym.nil? || wg_acronym == "NON WORKING GROUP"
|
|
423
419
|
|
|
424
|
-
abbr, name = STREAM_ORGS[
|
|
420
|
+
abbr, name = STREAM_ORGS[canonical_stream]
|
|
425
421
|
org = if abbr
|
|
426
422
|
Ietf::BibXMLParser.build_org(abbr, name)
|
|
427
423
|
else
|
data/lib/relaton/ietf/scraper.rb
CHANGED
|
@@ -15,17 +15,18 @@ module Relaton
|
|
|
15
15
|
# publishing their `index-v1` for released relatons, untouched.
|
|
16
16
|
IETF = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/main/"
|
|
17
17
|
|
|
18
|
-
# @param
|
|
18
|
+
# @param ref [String, Pubid::Ietf::Identifier] a reference, or its parse
|
|
19
|
+
# from Relaton::Db (relaton#205), which is used as it is
|
|
19
20
|
# @return [Relaton::Ietf::ItemData, nil]
|
|
20
|
-
def scrape_page(
|
|
21
|
-
id = parse_id
|
|
21
|
+
def scrape_page(ref)
|
|
22
|
+
id = ref.is_a?(String) ? parse_id(ref.sub(/\AIETF\s+/, "")) : ref
|
|
22
23
|
return unless id
|
|
23
24
|
|
|
24
25
|
fetch_doc id
|
|
25
26
|
rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
|
|
26
27
|
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
|
27
28
|
Net::ProtocolError, SocketError
|
|
28
|
-
raise Relaton::RequestError, "No document found for #{
|
|
29
|
+
raise Relaton::RequestError, "No document found for #{ref} reference"
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
private
|
|
@@ -28,11 +28,11 @@ module Relaton
|
|
|
28
28
|
# if nil then the fiename is used to read and write file (used to create indes in GH actions)
|
|
29
29
|
# @param [Pubid::Identifier] pubid class for deserialization
|
|
30
30
|
#
|
|
31
|
-
#
|
|
31
|
+
# The index format check round-trips each id through `pubid_class`;
|
|
32
32
|
# index format is now validated by round-tripping a sample of ids through
|
|
33
33
|
# the pubid class (see #check_serialization), which understands the pubid
|
|
34
34
|
# v2 (lutaml) `_type` serialization that the old key-allowlist could not.
|
|
35
|
-
def initialize(dir, url, filename
|
|
35
|
+
def initialize(dir, url: nil, filename: nil, pubid_class: nil)
|
|
36
36
|
@dir = dir
|
|
37
37
|
@url = url
|
|
38
38
|
@filename = filename
|
data/lib/relaton/index/pool.rb
CHANGED
|
@@ -14,7 +14,6 @@ module Relaton
|
|
|
14
14
|
# @param [String] type <description>
|
|
15
15
|
# @param [String, nil] url external URL to index, used to fetch index for searching files
|
|
16
16
|
# @param [String, nil] file output file name
|
|
17
|
-
# @param [Array<Symbol>, nil] id_keys keys to check if index is correct
|
|
18
17
|
# @param [String, nil] pages_url base URL of the Pages site that serves
|
|
19
18
|
# the machine index (manifest + shards), see Relaton::Index::ShardSource
|
|
20
19
|
#
|
|
@@ -24,9 +23,11 @@ module Relaton
|
|
|
24
23
|
if @pool[type.upcase.to_sym]&.actual?(**args)
|
|
25
24
|
@pool[type.upcase.to_sym]
|
|
26
25
|
else
|
|
26
|
+
if args.key?(:id_keys)
|
|
27
|
+
Util.warn "id_keys is deprecated and ignored by Relaton::Index"
|
|
28
|
+
end
|
|
27
29
|
@pool[type.upcase.to_sym] = Type.new(
|
|
28
|
-
type, args
|
|
29
|
-
pages_url: args[:pages_url]
|
|
30
|
+
type, **args.slice(:url, :file, :pubid_class, :pages_url)
|
|
30
31
|
)
|
|
31
32
|
end
|
|
32
33
|
end
|
|
@@ -47,7 +47,7 @@ module Relaton
|
|
|
47
47
|
@pages_url = pages_url.end_with?("/") ? pages_url : "#{pages_url}/"
|
|
48
48
|
# Only the deserialization helpers are used: this FileIO reads and
|
|
49
49
|
# writes no file.
|
|
50
|
-
@file_io = FileIO.new(dir,
|
|
50
|
+
@file_io = FileIO.new(dir, pubid_class: pubid_class)
|
|
51
51
|
@mutex = Mutex.new
|
|
52
52
|
@state = nil
|
|
53
53
|
end
|
data/lib/relaton/index/type.rb
CHANGED
|
@@ -10,21 +10,18 @@ module Relaton
|
|
|
10
10
|
# @param [String, Symbol] type type of index (ISO, IEC, etc.)
|
|
11
11
|
# @param [String, nil] url external URL to index, used to fetch index for searching files
|
|
12
12
|
# @param [String, nil] file output file name
|
|
13
|
-
# @param [Array<Symbol>] id_keys keys of identifier to be used for sorting index
|
|
14
|
-
# format of index file is checked if id_keys all is provided at least in one of the IDs
|
|
15
13
|
# @param [Pubid::Identifier, nil] pubid class for deserialization
|
|
16
14
|
# @param [String, nil] pages_url base URL of the Pages site that serves
|
|
17
15
|
# the machine index. With it the type reads the index from there, in
|
|
18
16
|
# memory (see ShardSource), and a parsed query fetches one shard only.
|
|
19
17
|
#
|
|
20
|
-
def initialize(type, url
|
|
21
|
-
pages_url: nil)
|
|
18
|
+
def initialize(type, url: nil, file: nil, pubid_class: nil, pages_url: nil)
|
|
22
19
|
@file = file
|
|
23
20
|
@dir = type.to_s.downcase
|
|
24
21
|
@pubid_class = pubid_class
|
|
25
22
|
@pages_url = pages_url
|
|
26
23
|
filename = file || Index.config.filename
|
|
27
|
-
@file_io = FileIO.new @dir, url, filename
|
|
24
|
+
@file_io = FileIO.new @dir, url: url, filename: filename, pubid_class: pubid_class
|
|
28
25
|
@source = new_source
|
|
29
26
|
end
|
|
30
27
|
|