relaton-ietf 1.14.3 → 1.14.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/rake.yml +1 -0
- data/.github/workflows/release.yml +3 -2
- data/.gitignore +1 -0
- data/.rubocop.yml +1 -1
- data/Gemfile +7 -1
- data/grammars/basicdoc.rng +0 -1
- data/grammars/biblio.rng +12 -2
- data/lib/relaton_ietf/bibxml_parser.rb +10 -1
- data/lib/relaton_ietf/data_fetcher.rb +10 -0
- data/lib/relaton_ietf/processor.rb +9 -0
- data/lib/relaton_ietf/rfc_entry.rb +3 -3
- data/lib/relaton_ietf/scrapper.rb +34 -23
- data/lib/relaton_ietf/version.rb +1 -1
- data/lib/relaton_ietf.rb +1 -0
- data/relaton_ietf.gemspec +5 -9
- metadata +18 -71
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6dd0e3485bf64d666190d66b48f5ade9d4e23ede1537ccf01f388b09fbec2a42
|
4
|
+
data.tar.gz: 01c1334f4730bbb0d636b15ebead6e28d54b524cb834c1d12d6d1453c66e7e3e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 933282984bd0231477efa96ec6711bcd732b8978942f441239ec005e202c9e0aeb85cfc26da480cda68796ae42b31d8e7aba2899f0583a448dfb0833f02cc1d3
|
7
|
+
data.tar.gz: 75475472dcd00bf9821daadd236313648f6a38573d6fed82afd1bd39c4dcaac70439f978f0f864b924a9a0226ab71ed26da09e419c16b60a65709707b7e5e2c1
|
data/.github/workflows/rake.yml
CHANGED
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/Gemfile
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
source "https://rubygems.org"
|
2
2
|
|
3
|
-
git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
|
3
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in relaton_ietf.gemspec
|
6
6
|
gemspec
|
7
|
+
|
8
|
+
gem "pry-byebug"
|
9
|
+
gem "ruby-jing"
|
10
|
+
gem "simplecov"
|
11
|
+
gem "vcr"
|
12
|
+
gem "webmock"
|
data/grammars/basicdoc.rng
CHANGED
data/grammars/biblio.rng
CHANGED
@@ -216,6 +216,9 @@
|
|
216
216
|
<optional>
|
217
217
|
<ref name="fullname"/>
|
218
218
|
</optional>
|
219
|
+
<zeroOrMore>
|
220
|
+
<ref name="credential"/>
|
221
|
+
</zeroOrMore>
|
219
222
|
<zeroOrMore>
|
220
223
|
<ref name="affiliation"/>
|
221
224
|
</zeroOrMore>
|
@@ -232,6 +235,11 @@
|
|
232
235
|
<ref name="FullNameType"/>
|
233
236
|
</element>
|
234
237
|
</define>
|
238
|
+
<define name="credential">
|
239
|
+
<element name="credential">
|
240
|
+
<text/>
|
241
|
+
</element>
|
242
|
+
</define>
|
235
243
|
<define name="FullNameType">
|
236
244
|
<choice>
|
237
245
|
<group>
|
@@ -305,7 +313,9 @@
|
|
305
313
|
<zeroOrMore>
|
306
314
|
<ref name="affiliationdescription"/>
|
307
315
|
</zeroOrMore>
|
308
|
-
<
|
316
|
+
<optional>
|
317
|
+
<ref name="organization"/>
|
318
|
+
</optional>
|
309
319
|
</element>
|
310
320
|
</define>
|
311
321
|
<define name="affiliationname">
|
@@ -1316,7 +1326,7 @@
|
|
1316
1326
|
<value>commentaryOf</value>
|
1317
1327
|
<value>hasCommentary</value>
|
1318
1328
|
<value>related</value>
|
1319
|
-
<value>
|
1329
|
+
<value>hasComplement</value>
|
1320
1330
|
<value>complementOf</value>
|
1321
1331
|
<value>obsoletes</value>
|
1322
1332
|
<value>obsoletedBy</value>
|
@@ -16,6 +16,16 @@ module RelatonIetf
|
|
16
16
|
RelatonIetf::IetfBibliographicItem.new(**attrs)
|
17
17
|
end
|
18
18
|
|
19
|
+
# def docids(reference, ver)
|
20
|
+
# ids = super
|
21
|
+
# si = reference.at("./seriesInfo[@name='Internet-Draft']",
|
22
|
+
# "./front/seriesInfo[@name='Internet-Draft']")
|
23
|
+
# if si
|
24
|
+
# id = si[:value]
|
25
|
+
# end
|
26
|
+
# ids
|
27
|
+
# end
|
28
|
+
|
19
29
|
#
|
20
30
|
# Extract document identifier type form identifier
|
21
31
|
#
|
@@ -27,7 +37,6 @@ module RelatonIetf
|
|
27
37
|
type = super
|
28
38
|
case type
|
29
39
|
when "BCP", "FYI", "STD", "RFC" then "RFC"
|
30
|
-
# when "Internet-Draft" then type
|
31
40
|
when "I-D" then "Internet-Draft"
|
32
41
|
else "IETF"
|
33
42
|
end
|
@@ -6,6 +6,7 @@ require "relaton_ietf/rfc_entry"
|
|
6
6
|
|
7
7
|
module RelatonIetf
|
8
8
|
class DataFetcher
|
9
|
+
INDEX1 = "index-v1".freeze
|
9
10
|
#
|
10
11
|
# Data fetcher initializer
|
11
12
|
#
|
@@ -20,6 +21,7 @@ module RelatonIetf
|
|
20
21
|
@format = format
|
21
22
|
@ext = @format.sub(/^bib|^rfc/, "")
|
22
23
|
@files = []
|
24
|
+
@index = Relaton::Index.find_or_create :IETF, file: "#{INDEX1}.yaml"
|
23
25
|
end
|
24
26
|
|
25
27
|
#
|
@@ -49,6 +51,7 @@ module RelatonIetf
|
|
49
51
|
when "ietf-internet-drafts" then fetch_ieft_internet_drafts
|
50
52
|
when "ietf-rfc-entries" then fetch_ieft_rfcs
|
51
53
|
end
|
54
|
+
@index.save
|
52
55
|
end
|
53
56
|
|
54
57
|
#
|
@@ -201,6 +204,13 @@ module RelatonIetf
|
|
201
204
|
@files << file
|
202
205
|
end
|
203
206
|
File.write file, c, encoding: "UTF-8"
|
207
|
+
add_to_index entry, file
|
208
|
+
end
|
209
|
+
|
210
|
+
def add_to_index(entry, file)
|
211
|
+
docid = entry.docidentifier.detect(&:primary)
|
212
|
+
docid ||= entry.docidentifier.first
|
213
|
+
@index.add_or_update docid.id, file
|
204
214
|
end
|
205
215
|
|
206
216
|
#
|
@@ -48,5 +48,14 @@ module RelatonIetf
|
|
48
48
|
def grammar_hash
|
49
49
|
@grammar_hash ||= ::RelatonIetf.grammar_hash
|
50
50
|
end
|
51
|
+
|
52
|
+
#
|
53
|
+
# Remove index file
|
54
|
+
#
|
55
|
+
def remove_index_file
|
56
|
+
Relaton::Index.find_or_create(:RFC, url: true, file: Scrapper::INDEX_FILE).remove_file
|
57
|
+
Relaton::Index.find_or_create(:RSS, url: true, file: Scrapper::INDEX_FILE).remove_file
|
58
|
+
Relaton::Index.find_or_create(:IDS, url: true, file: Scrapper::INDEX_FILE).remove_file
|
59
|
+
end
|
51
60
|
end
|
52
61
|
end
|
@@ -5,16 +5,17 @@ module RelatonIetf
|
|
5
5
|
module Scrapper
|
6
6
|
extend Scrapper
|
7
7
|
|
8
|
-
IDS = "https://raw.githubusercontent.com/relaton/relaton-data-ids/main/
|
9
|
-
RFC = "https://raw.githubusercontent.com/relaton/relaton-data-rfcs/main/
|
10
|
-
RSS = "https://raw.githubusercontent.com/relaton/relaton-data-rfcsubseries/main/
|
8
|
+
IDS = "https://raw.githubusercontent.com/relaton/relaton-data-ids/main/"
|
9
|
+
RFC = "https://raw.githubusercontent.com/relaton/relaton-data-rfcs/main/"
|
10
|
+
RSS = "https://raw.githubusercontent.com/relaton/relaton-data-rfcsubseries/main/"
|
11
|
+
INDEX_FILE = "index-v1.yaml"
|
11
12
|
|
12
13
|
# @param text [String]
|
13
14
|
# @return [RelatonIetf::IetfBibliographicItem]
|
14
15
|
def scrape_page(text)
|
15
16
|
# Remove initial "IETF " string if specified
|
16
17
|
ref = text.gsub(/^IETF /, "")
|
17
|
-
ref.sub!(/(?<=^(?:RFC|BCP|FYI|STD))\s(\d+)/) { $1.rjust 4, "0" }
|
18
|
+
# ref.sub!(/(?<=^(?:RFC|BCP|FYI|STD))\s(\d+)/) { $1.rjust 4, "0" }
|
18
19
|
rfc_item ref
|
19
20
|
rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
|
20
21
|
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
@@ -27,32 +28,42 @@ module RelatonIetf
|
|
27
28
|
# @param ref [String]
|
28
29
|
# @return [RelatonIetf::IetfBibliographicItem]
|
29
30
|
def rfc_item(ref) # rubocop:disable Metrics/MethodLength
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
31
|
+
case ref
|
32
|
+
when /^RFC/ then get_rfcs ref
|
33
|
+
when /^(?:BCP|FYI|STD)/ then get_rfcsubseries ref
|
34
|
+
when /^I-D/
|
35
|
+
ref.sub!(/^I-D[.\s]/, "")
|
36
|
+
get_ids ref
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def get_rfcs(ref)
|
41
|
+
index = Relaton::Index.find_or_create :RFC, url: "#{RFC}index-v1.zip", file: INDEX_FILE
|
42
|
+
row = index.search(ref).first
|
43
|
+
get_page "#{RFC}#{row[:file]}" if row
|
44
|
+
end
|
45
|
+
|
46
|
+
def get_rfcsubseries(ref)
|
47
|
+
index = Relaton::Index.find_or_create :RSS, url: "#{RSS}index-v1.zip", file: INDEX_FILE
|
48
|
+
row = index.search(ref).first
|
49
|
+
get_page "#{RSS}#{row[:file]}" if row
|
50
|
+
end
|
51
|
+
|
52
|
+
def get_ids(ref)
|
53
|
+
index = Relaton::Index.find_or_create :IDS, url: "#{IDS}index-v1.zip", file: INDEX_FILE
|
54
|
+
row = index.search(ref).first
|
55
|
+
get_page "#{IDS}#{row[:file]}" if row
|
47
56
|
end
|
48
57
|
|
49
58
|
# @param uri [String]
|
50
|
-
# @return [
|
59
|
+
# @return [RelatonIetf::IetfBibliographicItem, nil] HTTP response body
|
51
60
|
def get_page(uri)
|
52
61
|
res = Net::HTTP.get_response(URI(uri))
|
53
62
|
return unless res.code == "200"
|
54
63
|
|
55
|
-
res.body
|
64
|
+
hash = YAML.safe_load res.body
|
65
|
+
hash["fetched"] = Date.today.to_s
|
66
|
+
IetfBibliographicItem.from_hash hash
|
56
67
|
end
|
57
68
|
end
|
58
69
|
end
|
data/lib/relaton_ietf/version.rb
CHANGED
data/lib/relaton_ietf.rb
CHANGED
data/relaton_ietf.gemspec
CHANGED
@@ -10,10 +10,10 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.authors = ["Ribose Inc."]
|
11
11
|
spec.email = ["open.source@ribose.com"]
|
12
12
|
|
13
|
-
spec.summary = "RelatonIetf: retrieve IETF Standards for
|
14
|
-
"using the BibliographicItem model"
|
13
|
+
spec.summary = "RelatonIetf: retrieve IETF Standards for " \
|
14
|
+
"bibliographic use using the BibliographicItem model"
|
15
15
|
spec.description = <<~DESCRIPTION
|
16
|
-
RelatonIetf: retrieve IETF Standards for bibliographic use
|
16
|
+
RelatonIetf: retrieve IETF Standards for bibliographic use
|
17
17
|
using the BibliographicItem model.
|
18
18
|
|
19
19
|
Formerly known as rfcbib.
|
@@ -27,16 +27,12 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.bindir = "exe"
|
28
28
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
29
29
|
spec.require_paths = ["lib"]
|
30
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 2.
|
30
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
|
31
31
|
|
32
32
|
spec.add_development_dependency "equivalent-xml", "~> 0.6"
|
33
|
-
spec.add_development_dependency "pry-byebug"
|
34
33
|
spec.add_development_dependency "rake", "~> 13.0"
|
35
34
|
spec.add_development_dependency "rspec", "~> 3.0"
|
36
|
-
spec.add_development_dependency "ruby-jing"
|
37
|
-
spec.add_development_dependency "simplecov"
|
38
|
-
spec.add_development_dependency "vcr"
|
39
|
-
spec.add_development_dependency "webmock"
|
40
35
|
|
41
36
|
spec.add_dependency "relaton-bib", "~> 1.14.0"
|
37
|
+
spec.add_dependency "relaton-index", "~> 0.2.0"
|
42
38
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: relaton-ietf
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.14.
|
4
|
+
version: 1.14.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-05-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: equivalent-xml
|
@@ -24,20 +24,6 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0.6'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: pry-byebug
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: rake
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,77 +53,38 @@ dependencies:
|
|
67
53
|
- !ruby/object:Gem::Version
|
68
54
|
version: '3.0'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: simplecov
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
|
-
- !ruby/object:Gem::Dependency
|
98
|
-
name: vcr
|
99
|
-
requirement: !ruby/object:Gem::Requirement
|
100
|
-
requirements:
|
101
|
-
- - ">="
|
102
|
-
- !ruby/object:Gem::Version
|
103
|
-
version: '0'
|
104
|
-
type: :development
|
105
|
-
prerelease: false
|
106
|
-
version_requirements: !ruby/object:Gem::Requirement
|
107
|
-
requirements:
|
108
|
-
- - ">="
|
109
|
-
- !ruby/object:Gem::Version
|
110
|
-
version: '0'
|
111
|
-
- !ruby/object:Gem::Dependency
|
112
|
-
name: webmock
|
56
|
+
name: relaton-bib
|
113
57
|
requirement: !ruby/object:Gem::Requirement
|
114
58
|
requirements:
|
115
|
-
- - "
|
59
|
+
- - "~>"
|
116
60
|
- !ruby/object:Gem::Version
|
117
|
-
version:
|
118
|
-
type: :
|
61
|
+
version: 1.14.0
|
62
|
+
type: :runtime
|
119
63
|
prerelease: false
|
120
64
|
version_requirements: !ruby/object:Gem::Requirement
|
121
65
|
requirements:
|
122
|
-
- - "
|
66
|
+
- - "~>"
|
123
67
|
- !ruby/object:Gem::Version
|
124
|
-
version:
|
68
|
+
version: 1.14.0
|
125
69
|
- !ruby/object:Gem::Dependency
|
126
|
-
name: relaton-
|
70
|
+
name: relaton-index
|
127
71
|
requirement: !ruby/object:Gem::Requirement
|
128
72
|
requirements:
|
129
73
|
- - "~>"
|
130
74
|
- !ruby/object:Gem::Version
|
131
|
-
version:
|
75
|
+
version: 0.2.0
|
132
76
|
type: :runtime
|
133
77
|
prerelease: false
|
134
78
|
version_requirements: !ruby/object:Gem::Requirement
|
135
79
|
requirements:
|
136
80
|
- - "~>"
|
137
81
|
- !ruby/object:Gem::Version
|
138
|
-
version:
|
139
|
-
description:
|
140
|
-
|
82
|
+
version: 0.2.0
|
83
|
+
description: |
|
84
|
+
RelatonIetf: retrieve IETF Standards for bibliographic use
|
85
|
+
using the BibliographicItem model.
|
86
|
+
|
87
|
+
Formerly known as rfcbib.
|
141
88
|
email:
|
142
89
|
- open.source@ribose.com
|
143
90
|
executables: []
|
@@ -190,14 +137,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
190
137
|
requirements:
|
191
138
|
- - ">="
|
192
139
|
- !ruby/object:Gem::Version
|
193
|
-
version: 2.
|
140
|
+
version: 2.7.0
|
194
141
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
195
142
|
requirements:
|
196
143
|
- - ">="
|
197
144
|
- !ruby/object:Gem::Version
|
198
145
|
version: '0'
|
199
146
|
requirements: []
|
200
|
-
rubygems_version: 3.
|
147
|
+
rubygems_version: 3.3.26
|
201
148
|
signing_key:
|
202
149
|
specification_version: 4
|
203
150
|
summary: 'RelatonIetf: retrieve IETF Standards for bibliographic use using the BibliographicItem
|