relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.adoc +57 -1
- data/bin/console +0 -1
- data/lib/relaton/3gpp/bibliography.rb +82 -7
- data/lib/relaton/3gpp/data_fetcher.rb +51 -3
- data/lib/relaton/3gpp/docidentifier.rb +114 -0
- data/lib/relaton/3gpp/item.rb +6 -0
- data/lib/relaton/3gpp/parser.rb +1 -1
- data/lib/relaton/3gpp/processor.rb +4 -1
- data/lib/relaton/3gpp.rb +5 -1
- data/lib/relaton/adobe/bibdata.rb +8 -0
- data/lib/relaton/adobe/bibitem.rb +8 -0
- data/lib/relaton/adobe/bibliography.rb +92 -0
- data/lib/relaton/adobe/docidentifier.rb +49 -0
- data/lib/relaton/adobe/doctype.rb +14 -0
- data/lib/relaton/adobe/ext.rb +32 -0
- data/lib/relaton/adobe/item.rb +15 -0
- data/lib/relaton/adobe/item_base.rb +18 -0
- data/lib/relaton/adobe/item_data.rb +6 -0
- data/lib/relaton/adobe/processor.rb +45 -0
- data/lib/relaton/adobe/util.rb +8 -0
- data/lib/relaton/adobe.rb +37 -0
- data/lib/relaton/bib/model/address.rb +2 -2
- data/lib/relaton/bib/model/docidentifier.rb +24 -9
- data/lib/relaton/bib/model/localized_string.rb +1 -1
- data/lib/relaton/bib/model/structured_identifier.rb +10 -9
- data/lib/relaton/bib/sanitizer.rb +202 -6
- data/lib/relaton/bib.rb +0 -2
- data/lib/relaton/bipm/bibliography.rb +159 -10
- data/lib/relaton/bipm/data_fetcher.rb +26 -2
- data/lib/relaton/bipm/data_outcomes_parser.rb +3 -4
- data/lib/relaton/bipm/id_parser.rb +5 -4
- data/lib/relaton/bipm/model/structured_identifier.rb +21 -0
- data/lib/relaton/bipm/processor.rb +2 -2
- data/lib/relaton/bipm/rawdata_bipm_metrologia/fetcher.rb +2 -4
- data/lib/relaton/bipm/si_brochure_parser.rb +5 -3
- data/lib/relaton/bipm.rb +7 -1
- data/lib/relaton/bsi/bibliography.rb +115 -43
- data/lib/relaton/bsi/hit.rb +14 -0
- data/lib/relaton/bsi/hit_collection.rb +15 -16
- data/lib/relaton/bsi/model/docidentifier.rb +99 -1
- data/lib/relaton/bsi/processor.rb +1 -0
- data/lib/relaton/calconnect/bibliography.rb +12 -14
- data/lib/relaton/calconnect/data_fetcher.rb +77 -9
- data/lib/relaton/calconnect/docidentifier.rb +80 -0
- data/lib/relaton/calconnect/hit_collection.rb +65 -57
- data/lib/relaton/calconnect/model/item.rb +7 -0
- data/lib/relaton/calconnect/processor.rb +7 -1
- data/lib/relaton/calconnect.rb +11 -1
- data/lib/relaton/ccsds/data/fetcher.rb +17 -12
- data/lib/relaton/ccsds/data/parser.rb +1 -1
- data/lib/relaton/ccsds/hit_collection.rb +6 -1
- data/lib/relaton/ccsds/model/docidentifier.rb +121 -0
- data/lib/relaton/ccsds/model/item.rb +2 -0
- data/lib/relaton/cen/bibliography.rb +75 -47
- data/lib/relaton/cen/hit.rb +16 -1
- data/lib/relaton/cen/hit_collection.rb +59 -11
- data/lib/relaton/cen/model/docidentifier.rb +92 -1
- data/lib/relaton/cen/processor.rb +13 -8
- data/lib/relaton/cen/scraper.rb +13 -5
- data/lib/relaton/cen.rb +1 -0
- data/lib/relaton/cie/data_fetcher.rb +215 -30
- data/lib/relaton/cie/processor.rb +3 -1
- data/lib/relaton/cie/scrapper.rb +15 -2
- data/lib/relaton/cie.rb +2 -1
- data/lib/relaton/core/data_fetcher.rb +150 -3
- data/lib/relaton/core/governor.rb +320 -0
- data/lib/relaton/core/pacer.rb +134 -0
- data/lib/relaton/core/processor.rb +19 -0
- data/lib/relaton/core/request_error.rb +14 -0
- data/lib/relaton/core.rb +3 -0
- data/lib/relaton/db/registry.rb +41 -1
- data/lib/relaton/doi/crossref.rb +19 -2
- data/lib/relaton/doi/parser.rb +109 -15
- data/lib/relaton/easc/bibdata.rb +8 -0
- data/lib/relaton/easc/bibitem.rb +8 -0
- data/lib/relaton/easc/bibliography.rb +95 -0
- data/lib/relaton/easc/docidentifier.rb +100 -0
- data/lib/relaton/easc/doctype.rb +14 -0
- data/lib/relaton/easc/ext.rb +44 -0
- data/lib/relaton/easc/item.rb +13 -0
- data/lib/relaton/easc/item_base.rb +18 -0
- data/lib/relaton/easc/item_data.rb +6 -0
- data/lib/relaton/easc/processor.rb +46 -0
- data/lib/relaton/easc/util.rb +8 -0
- data/lib/relaton/easc.rb +35 -0
- data/lib/relaton/ecma/bibliography.rb +93 -25
- data/lib/relaton/ecma/data_fetcher.rb +71 -12
- data/lib/relaton/ecma/docidentifier.rb +124 -0
- data/lib/relaton/ecma/item.rb +2 -0
- data/lib/relaton/ecma/memento_parser.rb +1 -1
- data/lib/relaton/ecma/page_fetcher.rb +15 -3
- data/lib/relaton/ecma/parser_common.rb +2 -2
- data/lib/relaton/ecma/processor.rb +4 -1
- data/lib/relaton/ecma/standard_parser.rb +2 -2
- data/lib/relaton/ecma.rb +10 -1
- data/lib/relaton/etsi/bibliography.rb +67 -2
- data/lib/relaton/etsi/data_fetcher.rb +43 -4
- data/lib/relaton/etsi/processor.rb +3 -1
- data/lib/relaton/etsi.rb +2 -1
- data/lib/relaton/gb/bibliography.rb +55 -29
- data/lib/relaton/gb/docidentifier.rb +58 -9
- data/lib/relaton/gb/processor.rb +3 -0
- data/lib/relaton/gb/scraper.rb +27 -10
- data/lib/relaton/gost/bibdata.rb +8 -0
- data/lib/relaton/gost/bibitem.rb +8 -0
- data/lib/relaton/gost/bibliography.rb +107 -0
- data/lib/relaton/gost/docidentifier.rb +80 -0
- data/lib/relaton/gost/doctype.rb +16 -0
- data/lib/relaton/gost/ext.rb +46 -0
- data/lib/relaton/gost/item.rb +15 -0
- data/lib/relaton/gost/item_base.rb +18 -0
- data/lib/relaton/gost/item_data.rb +6 -0
- data/lib/relaton/gost/processor.rb +49 -0
- data/lib/relaton/gost/util.rb +8 -0
- data/lib/relaton/gost.rb +36 -0
- data/lib/relaton/iala/bibdata.rb +8 -0
- data/lib/relaton/iala/bibitem.rb +8 -0
- data/lib/relaton/iala/bibliography.rb +146 -0
- data/lib/relaton/iala/docidentifier.rb +89 -0
- data/lib/relaton/iala/doctype.rb +18 -0
- data/lib/relaton/iala/ext.rb +32 -0
- data/lib/relaton/iala/item.rb +21 -0
- data/lib/relaton/iala/item_base.rb +18 -0
- data/lib/relaton/iala/item_data.rb +6 -0
- data/lib/relaton/iala/processor.rb +43 -0
- data/lib/relaton/iala/relation.rb +7 -0
- data/lib/relaton/iala/util.rb +8 -0
- data/lib/relaton/iala.rb +35 -0
- data/lib/relaton/iana/bibliography.rb +67 -14
- data/lib/relaton/iana/data_fetcher.rb +35 -5
- data/lib/relaton/iana/processor.rb +3 -1
- data/lib/relaton/iana.rb +12 -1
- data/lib/relaton/iec/data_fetcher.rb +7 -1
- data/lib/relaton/iec/hit_collection.rb +1 -1
- data/lib/relaton/iec/model/docidentifier.rb +9 -5
- data/lib/relaton/iec/model/ext.rb +2 -2
- data/lib/relaton/iec/processor.rb +1 -0
- data/lib/relaton/ieee/bibliography.rb +25 -3
- data/lib/relaton/ieee/data_fetcher.rb +158 -17
- data/lib/relaton/ieee/idams_parser.rb +18 -11
- data/lib/relaton/ieee/processor.rb +4 -1
- data/lib/relaton/ieee/rawbib_id_parser.rb +291 -86
- data/lib/relaton/ieee.rb +2 -1
- data/lib/relaton/ietf/data_fetcher.rb +295 -12
- data/lib/relaton/ietf/processor.rb +7 -3
- data/lib/relaton/ietf/rfc/entry.rb +39 -3
- data/lib/relaton/ietf/scraper.rb +69 -36
- data/lib/relaton/ietf.rb +4 -1
- data/lib/relaton/iho/bibliography.rb +1 -1
- data/lib/relaton/iho/docidentifier.rb +1 -1
- data/lib/relaton/index/file_io.rb +11 -11
- data/lib/relaton/index/file_storage.rb +6 -1
- data/lib/relaton/index/pool.rb +6 -1
- data/lib/relaton/index/shard_source.rb +201 -0
- data/lib/relaton/index/type.rb +63 -12
- data/lib/relaton/index.rb +2 -1
- data/lib/relaton/iso/bibliography.rb +20 -15
- data/lib/relaton/iso/data_fetcher.rb +3 -3
- data/lib/relaton/iso/data_parser.rb +17 -3
- data/lib/relaton/iso/hit_collection.rb +27 -15
- data/lib/relaton/iso/item_data.rb +22 -0
- data/lib/relaton/iso/model/docidentifier.rb +24 -12
- data/lib/relaton/iso/processor.rb +1 -0
- data/lib/relaton/iso/scraper.rb +19 -3
- data/lib/relaton/itu/bibliography.rb +9 -4
- data/lib/relaton/itu/data_crawler_r.rb +664 -0
- data/lib/relaton/itu/data_fetcher.rb +496 -50
- data/lib/relaton/itu/data_merge_r.rb +149 -0
- data/lib/relaton/itu/data_parser_r.rb +163 -89
- data/lib/relaton/itu/data_parser_t.rb +228 -0
- data/lib/relaton/itu/family_cache.rb +177 -0
- data/lib/relaton/itu/governor.rb +56 -0
- data/lib/relaton/itu/hit.rb +9 -3
- data/lib/relaton/itu/hit_collection.rb +258 -86
- data/lib/relaton/itu/model/docidentifier.rb +67 -1
- data/lib/relaton/itu/model/structured_identifier.rb +19 -0
- data/lib/relaton/itu/processor.rb +10 -4
- data/lib/relaton/itu/pubid.rb +27 -5
- data/lib/relaton/itu/recommendation_fields.rb +334 -0
- data/lib/relaton/itu/recommendation_parser.rb +18 -149
- data/lib/relaton/itu/scraper.rb +13 -3
- data/lib/relaton/itu.rb +2 -1
- data/lib/relaton/jcgm/bibdata.rb +8 -0
- data/lib/relaton/jcgm/bibitem.rb +8 -0
- data/lib/relaton/jcgm/bibliography.rb +97 -0
- data/lib/relaton/jcgm/data_fetcher.rb +81 -0
- data/lib/relaton/jcgm/docidentifier.rb +102 -0
- data/lib/relaton/jcgm/doctype.rb +12 -0
- data/lib/relaton/jcgm/ext.rb +23 -0
- data/lib/relaton/jcgm/item.rb +20 -0
- data/lib/relaton/jcgm/item_base.rb +18 -0
- data/lib/relaton/jcgm/item_data.rb +6 -0
- data/lib/relaton/jcgm/meetings_parser.rb +175 -0
- data/lib/relaton/jcgm/processor.rb +71 -0
- data/lib/relaton/jcgm/relation.rb +9 -0
- data/lib/relaton/jcgm/structured_identifier.rb +40 -0
- data/lib/relaton/jcgm/util.rb +8 -0
- data/lib/relaton/jcgm.rb +24 -0
- data/lib/relaton/jis/bibliography.rb +8 -10
- data/lib/relaton/jis/data_fetcher.rb +21 -19
- data/lib/relaton/jis/docidentifier.rb +104 -5
- data/lib/relaton/jis/hit.rb +18 -23
- data/lib/relaton/jis/hit_collection.rb +19 -18
- data/lib/relaton/jis/processor.rb +1 -1
- data/lib/relaton/jis.rb +2 -3
- data/lib/relaton/logger/channels/gh_issue.rb +78 -13
- data/lib/relaton/nist/data_fetcher.rb +63 -13
- data/lib/relaton/nist/docidentifier.rb +165 -0
- data/lib/relaton/nist/item.rb +2 -0
- data/lib/relaton/nist/item_base.rb +16 -0
- data/lib/relaton/nist/mods_parser.rb +38 -12
- data/lib/relaton/nist/processor.rb +2 -1
- data/lib/relaton/nist/relation.rb +3 -0
- data/lib/relaton/nist/scraper.rb +6 -3
- data/lib/relaton/oasis/bibliography.rb +147 -6
- data/lib/relaton/oasis/data_fetcher.rb +41 -5
- data/lib/relaton/oasis/data_parser_utils.rb +37 -3
- data/lib/relaton/oasis/docidentifier.rb +54 -0
- data/lib/relaton/oasis/item.rb +3 -0
- data/lib/relaton/oasis/processor.rb +7 -1
- data/lib/relaton/oasis.rb +14 -1
- data/lib/relaton/ogc/data_fetcher.rb +23 -2
- data/lib/relaton/ogc/docidentifier.rb +105 -0
- data/lib/relaton/ogc/hit_collection.rb +78 -3
- data/lib/relaton/ogc/processor.rb +2 -1
- data/lib/relaton/ogc.rb +5 -1
- data/lib/relaton/oiml/bibliography.rb +90 -15
- data/lib/relaton/oiml/docidentifier.rb +18 -3
- data/lib/relaton/omg/docidentifier.rb +67 -0
- data/lib/relaton/omg/item.rb +1 -0
- data/lib/relaton/omg/processor.rb +1 -0
- data/lib/relaton/omg/scraper.rb +61 -16
- data/lib/relaton/omg.rb +1 -0
- data/lib/relaton/plateau/bibliography.rb +10 -3
- data/lib/relaton/plateau/data_fetcher.rb +25 -2
- data/lib/relaton/plateau/handbook_parser.rb +8 -1
- data/lib/relaton/plateau/hit.rb +10 -2
- data/lib/relaton/plateau/hit_collection.rb +31 -11
- data/lib/relaton/plateau/processor.rb +3 -1
- data/lib/relaton/plateau/technical_report_parser.rb +8 -1
- data/lib/relaton/plateau.rb +2 -1
- data/lib/relaton/sdo/config.rb +34 -0
- data/lib/relaton/sdo/fetcher.rb +52 -0
- data/lib/relaton/sdo/logo.rb +95 -0
- data/lib/relaton/sdo/name.rb +26 -0
- data/lib/relaton/sdo/organization.rb +71 -0
- data/lib/relaton/sdo/store.rb +49 -0
- data/lib/relaton/sdo.rb +29 -0
- data/lib/relaton/version.rb +1 -1
- data/lib/relaton/w3c/bibliography.rb +132 -12
- data/lib/relaton/w3c/data_fetcher.rb +194 -16
- data/lib/relaton/w3c/data_parser.rb +3 -3
- data/lib/relaton/w3c/docidentifier.rb +48 -0
- data/lib/relaton/w3c/governor.rb +32 -0
- data/lib/relaton/w3c/item.rb +3 -0
- data/lib/relaton/w3c/pubid.rb +12 -0
- data/lib/relaton/w3c/safe_realize.rb +110 -21
- data/lib/relaton/w3c.rb +12 -1
- data/lib/relaton/xsf/bibliography.rb +61 -1
- data/lib/relaton/xsf/data_fetcher.rb +55 -5
- data/lib/relaton/xsf/docidentifier.rb +46 -0
- data/lib/relaton/xsf/hit_collection.rb +31 -3
- data/lib/relaton/xsf/item.rb +6 -0
- data/lib/relaton/xsf/processor.rb +1 -0
- data/lib/relaton/xsf.rb +5 -1
- data/lib/relaton.rb +42 -0
- metadata +135 -24
- data/lib/relaton/ieee/pub_id.rb +0 -161
- data/lib/relaton/index/id_number.rb +0 -30
|
@@ -2,33 +2,42 @@
|
|
|
2
2
|
|
|
3
3
|
module Relaton
|
|
4
4
|
module Iana
|
|
5
|
-
#
|
|
5
|
+
# Retrieval front-end for the IANA flavor. Parses the citation with
|
|
6
|
+
# Pubid::Iana, looks it up in the pubid-structured `relaton-data-iana`
|
|
7
|
+
# index-v2, and fetches the matching per-document YAML over HTTP.
|
|
8
|
+
#
|
|
9
|
+
# An IANA entry is a protocol registry, not a numbered standard: the
|
|
10
|
+
# identifier is a registry slug with at most one "/"
|
|
11
|
+
# (`IANA _6lowpan-parameters`, `IANA _6lowpan-parameters/lowpan_nhc`). The
|
|
12
|
+
# index rows key on the TOP-LEVEL slug (`Identifiers::Registry#number`), so
|
|
13
|
+
# a registry and all of its sub-registries share one bsearch bucket and the
|
|
14
|
+
# block below distinguishes them.
|
|
6
15
|
module Bibliography
|
|
7
16
|
SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
|
|
8
17
|
|
|
9
|
-
# @param text [String]
|
|
10
|
-
# @return [Relaton::Iana::
|
|
11
|
-
def search(text)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
row = index.search(id).min_by { |i| i[:id] }
|
|
15
|
-
return unless row
|
|
18
|
+
# @param text [String, Pubid::Iana::Identifier]
|
|
19
|
+
# @return [Relaton::Iana::Item, nil]
|
|
20
|
+
def search(text)
|
|
21
|
+
pubid = parse_ref text
|
|
22
|
+
return unless pubid
|
|
16
23
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
24
|
+
# Pass the pubid so Relaton::Index narrows candidates by number via
|
|
25
|
+
# binary search before applying the block. The match is exact, so there
|
|
26
|
+
# is at most one row.
|
|
27
|
+
row = index.search(pubid) { |r| pubid_match? r[:id], pubid }.first
|
|
28
|
+
return unless row
|
|
20
29
|
|
|
21
|
-
|
|
30
|
+
fetch_row row
|
|
22
31
|
rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
|
|
23
32
|
EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
|
24
33
|
Net::ProtocolError, Errno::ETIMEDOUT => e
|
|
25
34
|
raise Relaton::RequestError, e.message
|
|
26
35
|
end
|
|
27
36
|
|
|
28
|
-
# @param ref [String] the
|
|
37
|
+
# @param ref [String] the IANA registry slug to look up
|
|
29
38
|
# @param year [String, NilClass] not used
|
|
30
39
|
# @param opts [Hash] options
|
|
31
|
-
# @return [Relaton::Iana::
|
|
40
|
+
# @return [Relaton::Iana::Item, nil]
|
|
32
41
|
def get(ref, _year = nil, _opts = {})
|
|
33
42
|
Util.info "Fetching from Relaton repository ...", key: ref
|
|
34
43
|
result = search(ref)
|
|
@@ -41,6 +50,50 @@ module Relaton
|
|
|
41
50
|
result
|
|
42
51
|
end
|
|
43
52
|
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def index
|
|
56
|
+
Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip",
|
|
57
|
+
file: "#{INDEXFILE}.yaml",
|
|
58
|
+
pubid_class: ::Pubid::Iana::Identifier
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# `Pubid::Iana::Identifier.parse` accepts both the printed `IANA <slug>`
|
|
62
|
+
# form and the bare slug the index rows render back to. It raises a plain
|
|
63
|
+
# RuntimeError on anything else, and a citation relaton cannot parse is a
|
|
64
|
+
# miss, not an error — `#get` reports it as "Not found.".
|
|
65
|
+
#
|
|
66
|
+
# @param text [String, Pubid::Iana::Identifier]
|
|
67
|
+
# @return [Pubid::Iana::Identifier, nil]
|
|
68
|
+
def parse_ref(text)
|
|
69
|
+
return text unless text.is_a? String
|
|
70
|
+
|
|
71
|
+
::Pubid::Iana::Identifier.parse text
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
Util.warn "Failed to parse pubid `#{text}`: #{e.message}"
|
|
74
|
+
nil
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Both arguments are Pubid::Iana identifiers. Compared field by field
|
|
78
|
+
# rather than with `==` so the two identity fields are explicit: a bare
|
|
79
|
+
# registry must NOT match its own sub-registries, which share its bsearch
|
|
80
|
+
# bucket. (index-v1 matched by substring, so `IANA rpki` also hit
|
|
81
|
+
# `rpki/signed-objects` and `min_by` picked the shortest id.)
|
|
82
|
+
def pubid_match?(row_id, query)
|
|
83
|
+
row_id.class == query.class &&
|
|
84
|
+
row_id.number == query.number &&
|
|
85
|
+
row_id.sub_registry.to_s == query.sub_registry.to_s
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# @param row [Hash] an index row
|
|
89
|
+
# @return [Relaton::Iana::Item, nil]
|
|
90
|
+
def fetch_row(row)
|
|
91
|
+
resp = Net::HTTP.get_response URI("#{SOURCE}#{row[:file]}")
|
|
92
|
+
return unless resp.code == "200"
|
|
93
|
+
|
|
94
|
+
Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
|
|
95
|
+
end
|
|
96
|
+
|
|
44
97
|
extend self
|
|
45
98
|
end
|
|
46
99
|
end
|
|
@@ -26,7 +26,28 @@ module Relaton
|
|
|
26
26
|
private
|
|
27
27
|
|
|
28
28
|
def index
|
|
29
|
-
@index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml"
|
|
29
|
+
@index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml",
|
|
30
|
+
pubid_class: ::Pubid::Iana::Identifier
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
#
|
|
34
|
+
# Add a document to the pubid-keyed index-v2.
|
|
35
|
+
#
|
|
36
|
+
# Stores the Pubid::Iana identifier object (not its hash): with
|
|
37
|
+
# pubid_class set, Relaton::Index sorts the index by the id's number on
|
|
38
|
+
# save and serializes each id to its `_type: pubid:iana:registry` hash.
|
|
39
|
+
#
|
|
40
|
+
# The rescue is load-bearing. Relaton::Index rejects the WHOLE index if a
|
|
41
|
+
# single row fails to deserialize, so a slug pubid cannot parse has to be
|
|
42
|
+
# skipped here rather than raise and abort the crawl.
|
|
43
|
+
#
|
|
44
|
+
# @param [String] docnumber bare registry slug, `registry[/sub-registry]`
|
|
45
|
+
# @param [String] file path of the document file
|
|
46
|
+
#
|
|
47
|
+
def add_to_index(docnumber, file)
|
|
48
|
+
index.add_or_update ::Pubid::Iana::Identifier.parse(docnumber), file
|
|
49
|
+
rescue StandardError => e
|
|
50
|
+
Util.warn "Skipping index entry for `#{docnumber}` (#{file}): #{e.message}"
|
|
30
51
|
end
|
|
31
52
|
|
|
32
53
|
def parse(content)
|
|
@@ -45,13 +66,22 @@ module Relaton
|
|
|
45
66
|
def save_doc(bib) # rubocop:disable Metrics/MethodLength
|
|
46
67
|
return unless bib
|
|
47
68
|
|
|
48
|
-
|
|
69
|
+
# Two distinct registries can sanitize to one filename (`/` and `-` both
|
|
70
|
+
# collapse to `-`), e.g. `rpki/signed-objects` and `rpki-signed-objects`.
|
|
71
|
+
# Take a path of our own rather than overwriting the other document, but
|
|
72
|
+
# keep the clash visible in the crawl log.
|
|
73
|
+
file = unique_output_file bib.docnumber
|
|
49
74
|
if @files.include? file
|
|
75
|
+
# A reserved path only ever belongs to one docnumber, so this is the
|
|
76
|
+
# same document again: a genuine duplicate, not a collision. Checked
|
|
77
|
+
# FIRST, because a disambiguated path stays != output_file forever.
|
|
50
78
|
Util.warn "File #{file} already exists. Document: #{bib.docnumber}"
|
|
51
|
-
|
|
52
|
-
|
|
79
|
+
elsif file != output_file(bib.docnumber)
|
|
80
|
+
Util.warn "File #{output_file bib.docnumber} already exists. " \
|
|
81
|
+
"Document: #{bib.docnumber}. Writing #{file} instead."
|
|
53
82
|
end
|
|
54
|
-
|
|
83
|
+
@files << file
|
|
84
|
+
add_to_index bib.docnumber, file
|
|
55
85
|
File.write file, serialize(bib), encoding: "UTF-8"
|
|
56
86
|
end
|
|
57
87
|
|
|
@@ -61,7 +61,9 @@ module Relaton
|
|
|
61
61
|
#
|
|
62
62
|
def remove_index_file
|
|
63
63
|
require_relative "../iana"
|
|
64
|
-
Relaton::Index.find_or_create(
|
|
64
|
+
Relaton::Index.find_or_create(
|
|
65
|
+
:iana, url: true, file: "#{INDEXFILE}.yaml"
|
|
66
|
+
).remove_file
|
|
65
67
|
end
|
|
66
68
|
end
|
|
67
69
|
end
|
data/lib/relaton/iana.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "faraday"
|
|
4
4
|
require "relaton/index"
|
|
5
|
+
require "pubid"
|
|
5
6
|
require "relaton/bib"
|
|
6
7
|
require_relative "version"
|
|
7
8
|
require_relative "iana/util"
|
|
@@ -15,7 +16,17 @@ require_relative "iana/bibliography"
|
|
|
15
16
|
|
|
16
17
|
module Relaton
|
|
17
18
|
module Iana
|
|
18
|
-
|
|
19
|
+
# The runtime index is the pubid-backed `index-v2`, whose rows are
|
|
20
|
+
# `Pubid::Iana::Identifiers::Registry` identifiers serialized as
|
|
21
|
+
# `_type: pubid:iana:registry` with `number` (the TOP-LEVEL registry slug)
|
|
22
|
+
# and an optional `sub_registry` — built and read via
|
|
23
|
+
# `pubid_class: ::Pubid::Iana::Identifier`.
|
|
24
|
+
#
|
|
25
|
+
# The legacy string-keyed `index-v1` (whose `:id` was the bare
|
|
26
|
+
# `registry[/sub-registry]` slug) is no longer produced or read here; it is
|
|
27
|
+
# emitted by `relaton-data-iana`'s own `crawler.rb`, derived from `index-v2`,
|
|
28
|
+
# so released gem lines keep resolving. See lib/relaton/iana/CLAUDE.md.
|
|
29
|
+
INDEXFILE = "index-v2".freeze
|
|
19
30
|
|
|
20
31
|
# Returns hash of XML reammar
|
|
21
32
|
# @return [String]
|
|
@@ -187,9 +187,15 @@ module Relaton
|
|
|
187
187
|
def fetch_pub(pub) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
188
188
|
bib = DataParser.new(pub, @errors).parse
|
|
189
189
|
did = bib.docidentifier.detect(&:primary)
|
|
190
|
-
|
|
190
|
+
# Distinct docids can sanitize to one filename; take a path of our own
|
|
191
|
+
# rather than overwriting the other document. A reserved path only ever
|
|
192
|
+
# belongs to one docid, so an @files hit is the same document again.
|
|
193
|
+
file = unique_output_file(did.to_s)
|
|
191
194
|
if @files.include? file then Util.warn "File #{file} exists."
|
|
192
195
|
else
|
|
196
|
+
if file != output_file(did.to_s)
|
|
197
|
+
Util.warn "File #{output_file did.to_s} exists. Docid: #{did}. Writing #{file} instead."
|
|
198
|
+
end
|
|
193
199
|
@files << file
|
|
194
200
|
pubid = parse_pubid(did.to_s)
|
|
195
201
|
index.add_or_update pubid, file if pubid
|
|
@@ -91,7 +91,7 @@ module Relaton
|
|
|
91
91
|
# removed at every nesting level. More robust than comparing attribute
|
|
92
92
|
# objects, whose derived `type`/`stage` differ between a parsed identifier
|
|
93
93
|
# (component object) and a deserialized one (nil/symbol) — at the top level
|
|
94
|
-
# and inside `
|
|
94
|
+
# and inside `base`.
|
|
95
95
|
def canonical_id(pubid, exclude)
|
|
96
96
|
drop = exclude.filter_map { |e| EXCLUDE_KEYS[e] }
|
|
97
97
|
prune(stringify(pubid.to_hash), drop)
|
|
@@ -50,13 +50,17 @@ module Relaton
|
|
|
50
50
|
content.to_s
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
+
# `remove_part!`/`remove_date!` are unnecessary here: `to_all_parts`'s
|
|
54
|
+
# own identity computation already excludes part/date (down the whole
|
|
55
|
+
# base chain, so a supplement's own year is dropped too — a disclosed
|
|
56
|
+
# pubid behavior change, not a bug; see spec/iec/…/docidentifier_spec.rb
|
|
57
|
+
# "#to_all_parts!"). `remove_stage!` stays: stage isn't one of the
|
|
58
|
+
# attributes pubid's `all_parts_edition_keys` strips.
|
|
53
59
|
def to_all_parts!
|
|
54
60
|
return unless @pubid
|
|
55
61
|
|
|
56
|
-
remove_part!
|
|
57
|
-
remove_date!
|
|
58
62
|
remove_stage!
|
|
59
|
-
@pubid
|
|
63
|
+
@pubid = @pubid.to_all_parts
|
|
60
64
|
refresh_content!
|
|
61
65
|
end
|
|
62
66
|
|
|
@@ -97,7 +101,7 @@ module Relaton
|
|
|
97
101
|
clear_attr_on(@pubid, attr)
|
|
98
102
|
end
|
|
99
103
|
|
|
100
|
-
node = @pubid.
|
|
104
|
+
node = @pubid.base
|
|
101
105
|
while node
|
|
102
106
|
clear_attr_on(node, attr)
|
|
103
107
|
# ConsolidatedIdentifier carries a sibling collection of bundled
|
|
@@ -109,7 +113,7 @@ module Relaton
|
|
|
109
113
|
clear_attr_on(id, attr)
|
|
110
114
|
end
|
|
111
115
|
end
|
|
112
|
-
node = node.
|
|
116
|
+
node = node.base
|
|
113
117
|
end
|
|
114
118
|
refresh_content!
|
|
115
119
|
end
|
|
@@ -17,7 +17,7 @@ module Relaton
|
|
|
17
17
|
attribute :cen_processing, :boolean
|
|
18
18
|
attribute :secretary, :string
|
|
19
19
|
attribute :interest_to_committees, :string
|
|
20
|
-
attribute :tc_sc_officers_note, :string
|
|
20
|
+
attribute :tc_sc_officers_note, :string
|
|
21
21
|
|
|
22
22
|
xml do
|
|
23
23
|
map_element "horizontal", to: :horizontal
|
|
@@ -30,7 +30,7 @@ module Relaton
|
|
|
30
30
|
map_element "cen-processing", to: :cen_processing
|
|
31
31
|
map_element "secretary", to: :secretary
|
|
32
32
|
map_element "interest-to-committees", to: :interest_to_committees
|
|
33
|
-
map_element "tc-sc-officers-note", to: :tc_sc_officers_note
|
|
33
|
+
map_element "tc-sc-officers-note", to: :tc_sc_officers_note, raw: :content
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
key_value do
|
|
@@ -6,6 +6,7 @@ module Relaton
|
|
|
6
6
|
def initialize
|
|
7
7
|
@short = :relaton_iec
|
|
8
8
|
@prefix = "IEC"
|
|
9
|
+
@pubid_flavor = :Iec # global prefixes sourced from Pubid::Iec.prefixes
|
|
9
10
|
@defaultprefix = %r{^(IEC\s|CISPR\s|IEV($|\s))}
|
|
10
11
|
@idtype = "IEC"
|
|
11
12
|
@datasets = %w[iec-harmonized-all iec-harmonized-latest]
|
|
@@ -12,9 +12,17 @@ module Relaton
|
|
|
12
12
|
# @return [Relaton::Ieee::ItemData, nil]
|
|
13
13
|
#
|
|
14
14
|
def search(code) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
index = Relaton::Index.find_or_create :ieee, url: "#{GH_URL}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
|
|
16
|
+
pubid_class: ::Pubid::Ieee::Identifier
|
|
17
|
+
# Pass the parsed pubid (not the raw String) so index-v2 narrows
|
|
18
|
+
# candidates by number via binary search before the block runs; the
|
|
19
|
+
# block keeps the broad substring match the string index gave, and an
|
|
20
|
+
# unparseable/partial ref falls back to the full-scan String search.
|
|
21
|
+
# Rows are Pubid::Ieee::Identifier objects (not Comparable), so pick by
|
|
22
|
+
# the string form.
|
|
23
|
+
pubid = parse_pubid code
|
|
24
|
+
needle = pubid.to_s
|
|
25
|
+
row = index.search(pubid) { |r| r[:id].to_s.include?(needle) }.min_by { |r| r[:id].to_s }
|
|
18
26
|
return unless row
|
|
19
27
|
|
|
20
28
|
resp = Faraday.get "#{GH_URL}#{row[:file]}"
|
|
@@ -45,6 +53,20 @@ module Relaton
|
|
|
45
53
|
nil
|
|
46
54
|
end
|
|
47
55
|
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Parse a reference into a Pubid::Ieee::Identifier for index narrowing, or
|
|
60
|
+
# return the raw String when pubid can't parse it (e.g. a partial ref) so
|
|
61
|
+
# the search falls back to the substring scan.
|
|
62
|
+
#
|
|
63
|
+
# @param code [String]
|
|
64
|
+
# @return [::Pubid::Ieee::Identifier, String]
|
|
65
|
+
def parse_pubid(code)
|
|
66
|
+
::Pubid::Ieee::Identifier.parse code
|
|
67
|
+
rescue StandardError
|
|
68
|
+
code
|
|
69
|
+
end
|
|
48
70
|
end
|
|
49
71
|
end
|
|
50
72
|
end
|
|
@@ -20,6 +20,90 @@ module Relaton
|
|
|
20
20
|
"E" => false, "B" => false, "W" => false
|
|
21
21
|
}.freeze
|
|
22
22
|
|
|
23
|
+
# Publication subtypes that IEEE does not give a designation to. Such a
|
|
24
|
+
# record carries a category in `stdnumber` ("White Paper", "Smart Grid
|
|
25
|
+
# Research: …") and a title in `normtitle`, and IEEE itself cites it by
|
|
26
|
+
# title. See {#no_standard_number?}.
|
|
27
|
+
NON_STANDARD_SUBTYPES = ["Whitepapers", "Research Documents"].freeze
|
|
28
|
+
|
|
29
|
+
class << self
|
|
30
|
+
#
|
|
31
|
+
# Build the pubid-structured `index-v2` from a directory of already
|
|
32
|
+
# generated per-document BibYAML files (the output of {.fetch}). This is
|
|
33
|
+
# a **separate step** from doc fetching: `relaton-data-ieee`'s crawler
|
|
34
|
+
# runs it after obtaining `data/` (a fresh crawl, or a clone of the
|
|
35
|
+
# published data repo). It parses each document's primary IEEE
|
|
36
|
+
# docidentifier into a `Pubid::Ieee::Identifier` and writes the index
|
|
37
|
+
# keyed by `pubid_class: ::Pubid::Ieee::Identifier`, so rows serialize to
|
|
38
|
+
# the `_type: pubid:ieee:*` structured form.
|
|
39
|
+
#
|
|
40
|
+
# Ids that pubid can't parse/round-trip are skipped (they'd otherwise
|
|
41
|
+
# break the all-pubid index that FileIO deserializes and number-sorts);
|
|
42
|
+
# the per-file skip and the final coverage summary are logged so the loss
|
|
43
|
+
# is never silent.
|
|
44
|
+
#
|
|
45
|
+
# @param dir [String] directory holding the per-document `*.yaml` files
|
|
46
|
+
# @param index_file [String] output index filename
|
|
47
|
+
# @return [Hash] `{ total:, indexed:, skipped: }` counts
|
|
48
|
+
#
|
|
49
|
+
def build_index(dir: "data", index_file: "#{INDEXFILE}.yaml")
|
|
50
|
+
index = Relaton::Index.find_or_create(
|
|
51
|
+
:ieee, file: index_file, pubid_class: ::Pubid::Ieee::Identifier
|
|
52
|
+
)
|
|
53
|
+
total = indexed = 0
|
|
54
|
+
Dir["#{dir}/*.yaml"].sort.each do |file|
|
|
55
|
+
total += 1
|
|
56
|
+
docid = primary_docid(file)
|
|
57
|
+
pid = docid && pubid(docid)
|
|
58
|
+
if pid
|
|
59
|
+
index.add_or_update pid, file
|
|
60
|
+
indexed += 1
|
|
61
|
+
else
|
|
62
|
+
Util.warn "Skipped (unparseable id): `#{docid || '<none>'}` (#{file})"
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
index.save
|
|
66
|
+
Util.info "IEEE #{index_file}: #{indexed}/#{total} indexed, " \
|
|
67
|
+
"#{total - indexed} skipped " \
|
|
68
|
+
"(#{total.zero? ? 0 : (100.0 * indexed / total).round(1)}% coverage)"
|
|
69
|
+
{ total: total, indexed: indexed, skipped: total - indexed }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
# Extract the primary, non-trademark IEEE docidentifier string from a
|
|
75
|
+
# BibYAML file, or nil when absent/unreadable.
|
|
76
|
+
#
|
|
77
|
+
# @param file [String]
|
|
78
|
+
# @return [String, nil]
|
|
79
|
+
def primary_docid(file)
|
|
80
|
+
yaml = YAML.safe_load(File.read(file, encoding: "UTF-8"),
|
|
81
|
+
permitted_classes: [Symbol, Date, Time])
|
|
82
|
+
id = yaml && yaml["docidentifier"]&.find do |i|
|
|
83
|
+
i["type"] == "IEEE" && i["primary"] == true && i["trademark"].nil?
|
|
84
|
+
end
|
|
85
|
+
id && id["content"]
|
|
86
|
+
rescue StandardError
|
|
87
|
+
nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Parse a docid string into a Pubid::Ieee::Identifier, or nil if pubid
|
|
91
|
+
# can't parse it or the structured id won't round-trip through the index
|
|
92
|
+
# (matching FileIO's `id_supported?` acceptance test).
|
|
93
|
+
#
|
|
94
|
+
# @param id [String]
|
|
95
|
+
# @return [::Pubid::Ieee::Identifier, nil]
|
|
96
|
+
def pubid(id)
|
|
97
|
+
pid = ::Pubid::Ieee::Identifier.parse id
|
|
98
|
+
hash = pid.to_hash
|
|
99
|
+
return nil unless ::Pubid::Ieee::Identifier.from_hash(hash).to_hash == hash
|
|
100
|
+
|
|
101
|
+
pid
|
|
102
|
+
rescue StandardError
|
|
103
|
+
nil
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
23
107
|
#
|
|
24
108
|
# Convert documents from `ieee-rawbib` dir (IEEE dataset) to BibYAML/BibXML
|
|
25
109
|
#
|
|
@@ -229,10 +313,22 @@ module Relaton
|
|
|
229
313
|
|
|
230
314
|
normtitle = cheap_extract_field(xml, "normtitle")
|
|
231
315
|
stdnumber = cheap_extract_field(xml, "stdnumber")
|
|
316
|
+
# Drop it here as well as in #parse_entry: the winner selection groups
|
|
317
|
+
# files by docnumber, so a white paper that reads as `IEEE Std 802`
|
|
318
|
+
# would join the group of the real standard and push its file out.
|
|
319
|
+
subtype = cheap_extract_field(xml, "publicationsubtype")
|
|
320
|
+
return nil if no_standard_number?(subtype, stdnumber)
|
|
321
|
+
|
|
232
322
|
docnumber = nil
|
|
233
323
|
if normtitle && stdnumber
|
|
234
324
|
pubid = RawbibIdParser.parse(normtitle, stdnumber)
|
|
235
|
-
|
|
325
|
+
# The article title (what `IdamsParser#pubid` compares against) is
|
|
326
|
+
# the *last* `<title>` in the file: the top-level one (first) is the
|
|
327
|
+
# designation, present even for a real standard.
|
|
328
|
+
title = cheap_extract_field(xml, "title", last: true)
|
|
329
|
+
isbn = xml.include?("<isbn")
|
|
330
|
+
fabricated = RawbibIdParser.fabricated_title_id?(pubid, title, isbn)
|
|
331
|
+
docnumber = pubid&.to_s unless fabricated
|
|
236
332
|
end
|
|
237
333
|
[idx, file, docnumber, file.include?("/updates.")]
|
|
238
334
|
rescue StandardError
|
|
@@ -241,9 +337,28 @@ module Relaton
|
|
|
241
337
|
[idx, file, nil, file.include?("/updates.")]
|
|
242
338
|
end
|
|
243
339
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
340
|
+
#
|
|
341
|
+
# True when the record has no standard number. IEEE gives a white paper
|
|
342
|
+
# or a research document no designation, so `stdnumber` holds a category
|
|
343
|
+
# and `normtitle` holds a title. An id built from either is a title
|
|
344
|
+
# fragment, and records that share a fragment overwrite each other's
|
|
345
|
+
# output file. Both conditions are required: a record of such a subtype
|
|
346
|
+
# that does carry a number keeps it.
|
|
347
|
+
#
|
|
348
|
+
# @param [String, nil] subtype document element "publicationsubtype"
|
|
349
|
+
# @param [String, nil] stdnumber document element "stdnumber"
|
|
350
|
+
#
|
|
351
|
+
# @return [Boolean]
|
|
352
|
+
#
|
|
353
|
+
def no_standard_number?(subtype, stdnumber)
|
|
354
|
+
NON_STANDARD_SUBTYPES.include?(subtype) && !stdnumber.to_s.match?(/\d/)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def cheap_extract_field(xml, tag, last: false)
|
|
358
|
+
re = %r{<#{tag}[^>]*?>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</#{tag}>}m
|
|
359
|
+
matches = xml.scan(re)
|
|
360
|
+
m = last ? matches.last : matches.first
|
|
361
|
+
m && m[0].strip
|
|
247
362
|
end
|
|
248
363
|
|
|
249
364
|
def select_prefilter_winners(index, total)
|
|
@@ -326,12 +441,16 @@ module Relaton
|
|
|
326
441
|
|
|
327
442
|
# Stragglers: any remaining staged files (losing duplicates,
|
|
328
443
|
# or bib filenames that didn't end up in saved_writes due to a
|
|
329
|
-
# crash) get cleaned up so they don't pollute `data/`.
|
|
330
|
-
|
|
444
|
+
# crash) get cleaned up so they don't pollute `data/`. Log the
|
|
445
|
+
# count so an unexpectedly large sweep (e.g. a worker that lost
|
|
446
|
+
# its state) is visible instead of silent.
|
|
447
|
+
stragglers = Dir.glob(File.join(@output, "*.#{@ext}.*"))
|
|
448
|
+
stragglers.each do |f|
|
|
331
449
|
File.unlink(f)
|
|
332
450
|
rescue StandardError
|
|
333
451
|
# ignore — best-effort cleanup
|
|
334
452
|
end
|
|
453
|
+
Util.info "Reconcile: cleaned #{stragglers.size} staged straggler(s)" unless stragglers.empty?
|
|
335
454
|
end
|
|
336
455
|
|
|
337
456
|
#
|
|
@@ -407,12 +526,7 @@ module Relaton
|
|
|
407
526
|
pid = Process.fork do
|
|
408
527
|
batch_files.each_with_index do |file, i|
|
|
409
528
|
glob_idx = base_idx + i
|
|
410
|
-
|
|
411
|
-
next unless result
|
|
412
|
-
|
|
413
|
-
_, _, doc, bib, local_errors = result
|
|
414
|
-
merge_errors(local_errors)
|
|
415
|
-
commit_doc(doc, bib, file, glob_idx)
|
|
529
|
+
commit_entry(glob_idx, file, glob_idx)
|
|
416
530
|
end
|
|
417
531
|
File.binwrite(state_path, Marshal.dump(
|
|
418
532
|
backrefs: backrefs,
|
|
@@ -469,12 +583,32 @@ module Relaton
|
|
|
469
583
|
def run_shard(shard, _shard_idx)
|
|
470
584
|
shard.each_with_index do |entry, i|
|
|
471
585
|
idx, file = entry.is_a?(Array) ? entry : [i, entry]
|
|
472
|
-
|
|
473
|
-
|
|
586
|
+
commit_entry(idx, file)
|
|
587
|
+
end
|
|
588
|
+
end
|
|
474
589
|
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
590
|
+
#
|
|
591
|
+
# Parse one file and commit it, guarding the commit so a single bad
|
|
592
|
+
# document (e.g. a pathological docnumber that would raise
|
|
593
|
+
# Errno::ENAMETOOLONG) logs and is skipped rather than aborting the
|
|
594
|
+
# serial crawl or killing a parallel worker before it persists its
|
|
595
|
+
# state. Shared by the serial (`run_shard`) and parallel
|
|
596
|
+
# (`spawn_batch`) paths.
|
|
597
|
+
#
|
|
598
|
+
# @param [Integer] idx original glob index (preserves dedup order)
|
|
599
|
+
# @param [String] file path to rawbib file
|
|
600
|
+
# @param [Integer, nil] glob_idx staging index for parallel mode
|
|
601
|
+
#
|
|
602
|
+
def commit_entry(idx, file, glob_idx = nil)
|
|
603
|
+
result = parse_entry(idx, file)
|
|
604
|
+
return unless result
|
|
605
|
+
|
|
606
|
+
_, _, doc, bib, local_errors = result
|
|
607
|
+
merge_errors(local_errors)
|
|
608
|
+
begin
|
|
609
|
+
commit_doc(doc, bib, file, glob_idx)
|
|
610
|
+
rescue StandardError => e
|
|
611
|
+
Util.error "commit failed for `#{file}`: #{e.class}: #{e.message}"
|
|
478
612
|
end
|
|
479
613
|
end
|
|
480
614
|
|
|
@@ -501,6 +635,13 @@ module Relaton
|
|
|
501
635
|
end
|
|
502
636
|
return nil if doc.publicationinfo&.standard_id == "0"
|
|
503
637
|
|
|
638
|
+
info = doc.publicationinfo
|
|
639
|
+
if no_standard_number?(info&.publicationsubtype, info&.stdnumber)
|
|
640
|
+
Util.warn "No standard number. Normtitle: `#{doc.normtitle}`, " \
|
|
641
|
+
"file: `#{file}`"
|
|
642
|
+
return nil
|
|
643
|
+
end
|
|
644
|
+
|
|
504
645
|
local_errors = Hash.new(true)
|
|
505
646
|
bib = IdamsParser.new(doc, self, local_errors).parse
|
|
506
647
|
if bib.docnumber.nil?
|
|
@@ -43,23 +43,28 @@ module Relaton
|
|
|
43
43
|
#
|
|
44
44
|
# Parse docnumber
|
|
45
45
|
#
|
|
46
|
-
# @return [String]
|
|
46
|
+
# @return [String, nil] canonical pubid string, or nil when unparseable
|
|
47
47
|
#
|
|
48
48
|
def docnumber
|
|
49
|
-
@docnumber ||= pubid&.
|
|
49
|
+
@docnumber ||= pubid&.to_s
|
|
50
50
|
end
|
|
51
51
|
|
|
52
52
|
#
|
|
53
|
-
#
|
|
53
|
+
# Resolve the document's IEEE identifier (pubid-first, RawbibIdParser
|
|
54
|
+
# fallback).
|
|
54
55
|
#
|
|
55
|
-
# @return [
|
|
56
|
+
# @return [::Pubid::Ieee::Identifier, nil]
|
|
56
57
|
#
|
|
57
58
|
def pubid
|
|
58
|
-
@pubid
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
59
|
+
return @pubid if defined?(@pubid)
|
|
60
|
+
|
|
61
|
+
normtitle = @doc.normtitle
|
|
62
|
+
stdnumber = @doc.publicationinfo.stdnumber
|
|
63
|
+
pid = RawbibIdParser.parse(normtitle, stdnumber)
|
|
64
|
+
title = @doc.btitle.find { |t| t[:type] == "main" }&.dig(:content)
|
|
65
|
+
isbn = @doc.isbn_doi.any? { |id| id[:type] == "ISBN" }
|
|
66
|
+
fabricated = RawbibIdParser.fabricated_title_id?(pid, title, isbn)
|
|
67
|
+
@pubid = fabricated ? nil : pid
|
|
63
68
|
end
|
|
64
69
|
|
|
65
70
|
#
|
|
@@ -92,8 +97,10 @@ module Relaton
|
|
|
92
97
|
def parse_docidentifier # rubocop:disable Metrics/MethodLength
|
|
93
98
|
ids = @doc.isbn_doi.map { |id| id[:content] = id.delete(:id); id }
|
|
94
99
|
|
|
95
|
-
|
|
96
|
-
|
|
100
|
+
if pubid
|
|
101
|
+
ids.unshift(content: pubid.to_s(trademark: true), scope: "trademark", type: "IEEE", primary: true)
|
|
102
|
+
ids.unshift(content: pubid.to_s, type: "IEEE", primary: true)
|
|
103
|
+
end
|
|
97
104
|
|
|
98
105
|
result = ids.map { |dcid| Bib::Docidentifier.new(**dcid) }
|
|
99
106
|
@errors[:docidentifier] &&= result.empty?
|
|
@@ -8,6 +8,7 @@ module Relaton
|
|
|
8
8
|
def initialize # rubocop:disable Lint/MissingSuper
|
|
9
9
|
@short = :relaton_ieee
|
|
10
10
|
@prefix = "IEEE"
|
|
11
|
+
@pubid_flavor = :Ieee # global prefixes sourced from Pubid::Ieee.prefixes
|
|
11
12
|
@defaultprefix = %r{^(?:(?:(?:ANSI|NACE)/)?IEEE|ANSI|AIEE|ASA|NACE|IRE)\s}
|
|
12
13
|
@idtype = "IEEE"
|
|
13
14
|
@datasets = %w[ieee-rawbib]
|
|
@@ -61,7 +62,9 @@ module Relaton
|
|
|
61
62
|
#
|
|
62
63
|
def remove_index_file
|
|
63
64
|
require_relative "../ieee"
|
|
64
|
-
Relaton::Index.find_or_create(
|
|
65
|
+
Relaton::Index.find_or_create(
|
|
66
|
+
:ieee, url: true, file: "#{INDEXFILE}.yaml"
|
|
67
|
+
).remove_file
|
|
65
68
|
end
|
|
66
69
|
end
|
|
67
70
|
end
|