relaton-itu 1.6.0 → 1.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/rake.yml +46 -0
- data/README.adoc +33 -0
- data/grammars/isodoc.rng +14 -43
- data/grammars/itu.rng +14 -0
- data/lib/relaton_itu/editorial_group.rb +1 -1
- data/lib/relaton_itu/hash_converter.rb +2 -2
- data/lib/relaton_itu/hit.rb +2 -0
- data/lib/relaton_itu/hit_collection.rb +25 -14
- data/lib/relaton_itu/itu_bibliographic_item.rb +1 -1
- data/lib/relaton_itu/itu_bibliography.rb +39 -21
- data/lib/relaton_itu/itu_group.rb +1 -1
- data/lib/relaton_itu/scrapper.rb +52 -27
- data/lib/relaton_itu/version.rb +1 -1
- data/lib/relaton_itu/xml_parser.rb +1 -1
- data/relaton-itu.gemspec +3 -3
- metadata +5 -35
- data/.github/workflows/macos.yml +0 -34
- data/.github/workflows/ubuntu.yml +0 -33
- data/.github/workflows/windows.yml +0 -35
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cd478e1103e026b9ff40937a19dccc0166da6f839fc2fb216a59ac306b4958f1
|
|
4
|
+
data.tar.gz: 12bc7a7304aadb7ea91df2bc618b49ef1333ea58d577c760e9273c955fd5fd01
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a4ee1c7e0c5ff26729f598b744035e8ebbf285703f04da910a978d38f3e3a9dc44f9e76d4bc17e6257e443b953c5dd218526305bd95106af410ddbcc667db83f
|
|
7
|
+
data.tar.gz: b10b5dc734cfed86b6653e8d0f56257ac52a4e06718c30a9256897eecd11a03642ded7820cf7d9797d2c3c7731ccc093035b6e7a7ab4477b6787f120449ca494
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
+
# See https://github.com/metanorma/cimas
|
|
3
|
+
name: rake
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [ master, main ]
|
|
8
|
+
tags: [ v* ]
|
|
9
|
+
pull_request:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
rake:
|
|
13
|
+
name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
|
|
14
|
+
runs-on: ${{ matrix.os }}
|
|
15
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
16
|
+
strategy:
|
|
17
|
+
fail-fast: false
|
|
18
|
+
matrix:
|
|
19
|
+
ruby: [ '2.7', '2.6', '2.5', '2.4' ]
|
|
20
|
+
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
|
21
|
+
experimental: [ false ]
|
|
22
|
+
include:
|
|
23
|
+
- ruby: '3.0'
|
|
24
|
+
os: 'ubuntu-latest'
|
|
25
|
+
experimental: true
|
|
26
|
+
- ruby: '3.0'
|
|
27
|
+
os: 'windows-latest'
|
|
28
|
+
experimental: true
|
|
29
|
+
- ruby: '3.0'
|
|
30
|
+
os: 'macos-latest'
|
|
31
|
+
experimental: true
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v2
|
|
34
|
+
with:
|
|
35
|
+
submodules: true
|
|
36
|
+
|
|
37
|
+
# https://github.com/ruby-debug/debase/issues/89#issuecomment-686827382
|
|
38
|
+
- if: matrix.os == 'macos-latest' && matrix.ruby == '2.5'
|
|
39
|
+
run: echo BUNDLE_BUILD__DEBASE="--with-cflags=\"-Wno-error=implicit-function-declaration\"" >> $GITHUB_ENV
|
|
40
|
+
|
|
41
|
+
- uses: ruby/setup-ruby@v1
|
|
42
|
+
with:
|
|
43
|
+
ruby-version: ${{ matrix.ruby }}
|
|
44
|
+
bundler-cache: true
|
|
45
|
+
|
|
46
|
+
- run: bundle exec rake
|
data/README.adoc
CHANGED
|
@@ -87,6 +87,39 @@ RelatonItu::ItuBibliography.get("ITU-T L.163", "2018", {})
|
|
|
87
87
|
...
|
|
88
88
|
----
|
|
89
89
|
|
|
90
|
+
=== Get amendment
|
|
91
|
+
[source,ruby]
|
|
92
|
+
----
|
|
93
|
+
RelatonItu::ItuBibliography.get "ITU-T G.989.2/Amd 1"
|
|
94
|
+
[relaton-itu] ("ITU-T G.989.2/Amd 1") fetching...
|
|
95
|
+
[relaton-itu] ("ITU-T G.989.2/Amd 1") found ITU-T G.989.2/Amd 1
|
|
96
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fe5409e5840
|
|
97
|
+
...
|
|
98
|
+
----
|
|
99
|
+
|
|
100
|
+
=== Get ITU-R documents
|
|
101
|
+
|
|
102
|
+
[source,ruby]
|
|
103
|
+
----
|
|
104
|
+
RelatonItu::ItuBibliography.get "ITU-R REC-BO.600-1"
|
|
105
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") fetching...
|
|
106
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") found R-REC-BO.600-1
|
|
107
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007f84e9930498
|
|
108
|
+
...
|
|
109
|
+
----
|
|
110
|
+
|
|
111
|
+
=== Get ITU-R Radio Regulations (RR)
|
|
112
|
+
|
|
113
|
+
[source,ruby]
|
|
114
|
+
----
|
|
115
|
+
RelatonItu::ItuBibliography.get 'ITU-R RR (2020)'
|
|
116
|
+
[relaton-itu] ("ITU-R RR") fetching...
|
|
117
|
+
[relaton-itu] WARNING: invalid doctype: publication
|
|
118
|
+
[relaton-itu] ("ITU-R RR") found ITU-R RR
|
|
119
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fb4b55eb660
|
|
120
|
+
...
|
|
121
|
+
----
|
|
122
|
+
|
|
90
123
|
=== Create bibliographic item form YAML
|
|
91
124
|
[source,ruby]
|
|
92
125
|
----
|
data/grammars/isodoc.rng
CHANGED
|
@@ -24,6 +24,14 @@
|
|
|
24
24
|
<start>
|
|
25
25
|
<ref name="standard-document"/>
|
|
26
26
|
</start>
|
|
27
|
+
<define name="doctype">
|
|
28
|
+
<element name="doctype">
|
|
29
|
+
<optional>
|
|
30
|
+
<attribute name="abbreviation"/>
|
|
31
|
+
</optional>
|
|
32
|
+
<ref name="DocumentType"/>
|
|
33
|
+
</element>
|
|
34
|
+
</define>
|
|
27
35
|
<define name="hyperlink">
|
|
28
36
|
<element name="link">
|
|
29
37
|
<attribute name="target">
|
|
@@ -141,6 +149,11 @@
|
|
|
141
149
|
<data type="boolean"/>
|
|
142
150
|
</attribute>
|
|
143
151
|
</optional>
|
|
152
|
+
<optional>
|
|
153
|
+
<attribute name="key">
|
|
154
|
+
<data type="boolean"/>
|
|
155
|
+
</attribute>
|
|
156
|
+
</optional>
|
|
144
157
|
<oneOrMore>
|
|
145
158
|
<ref name="dt"/>
|
|
146
159
|
<ref name="dd"/>
|
|
@@ -1164,49 +1177,7 @@
|
|
|
1164
1177
|
</define>
|
|
1165
1178
|
<define name="annex">
|
|
1166
1179
|
<element name="annex">
|
|
1167
|
-
<
|
|
1168
|
-
<attribute name="id">
|
|
1169
|
-
<data type="ID"/>
|
|
1170
|
-
</attribute>
|
|
1171
|
-
</optional>
|
|
1172
|
-
<optional>
|
|
1173
|
-
<attribute name="language"/>
|
|
1174
|
-
</optional>
|
|
1175
|
-
<optional>
|
|
1176
|
-
<attribute name="script"/>
|
|
1177
|
-
</optional>
|
|
1178
|
-
<optional>
|
|
1179
|
-
<attribute name="inline-header">
|
|
1180
|
-
<data type="boolean"/>
|
|
1181
|
-
</attribute>
|
|
1182
|
-
</optional>
|
|
1183
|
-
<attribute name="obligation">
|
|
1184
|
-
<choice>
|
|
1185
|
-
<value>normative</value>
|
|
1186
|
-
<value>informative</value>
|
|
1187
|
-
</choice>
|
|
1188
|
-
</attribute>
|
|
1189
|
-
<optional>
|
|
1190
|
-
<ref name="section-title"/>
|
|
1191
|
-
</optional>
|
|
1192
|
-
<group>
|
|
1193
|
-
<group>
|
|
1194
|
-
<zeroOrMore>
|
|
1195
|
-
<ref name="BasicBlock"/>
|
|
1196
|
-
</zeroOrMore>
|
|
1197
|
-
<zeroOrMore>
|
|
1198
|
-
<ref name="note"/>
|
|
1199
|
-
</zeroOrMore>
|
|
1200
|
-
</group>
|
|
1201
|
-
<zeroOrMore>
|
|
1202
|
-
<choice>
|
|
1203
|
-
<ref name="annex-subsection"/>
|
|
1204
|
-
<ref name="terms"/>
|
|
1205
|
-
<ref name="definitions"/>
|
|
1206
|
-
<ref name="references"/>
|
|
1207
|
-
</choice>
|
|
1208
|
-
</zeroOrMore>
|
|
1209
|
-
</group>
|
|
1180
|
+
<ref name="Annex-Section"/>
|
|
1210
1181
|
</element>
|
|
1211
1182
|
</define>
|
|
1212
1183
|
<define name="terms">
|
data/grammars/itu.rng
CHANGED
|
@@ -41,6 +41,9 @@
|
|
|
41
41
|
<value>technical-report</value>
|
|
42
42
|
<value>joint-itu-iso-iec</value>
|
|
43
43
|
<value>resolution</value>
|
|
44
|
+
<value>service-publication</value>
|
|
45
|
+
<value>handbook</value>
|
|
46
|
+
<value>question</value>
|
|
44
47
|
</choice>
|
|
45
48
|
</define>
|
|
46
49
|
<define name="editorialgroup">
|
|
@@ -97,6 +100,9 @@
|
|
|
97
100
|
<optional>
|
|
98
101
|
<ref name="meeting"/>
|
|
99
102
|
</optional>
|
|
103
|
+
<optional>
|
|
104
|
+
<ref name="meeting-place"/>
|
|
105
|
+
</optional>
|
|
100
106
|
<optional>
|
|
101
107
|
<ref name="meeting-date"/>
|
|
102
108
|
</optional>
|
|
@@ -127,6 +133,14 @@
|
|
|
127
133
|
</include>
|
|
128
134
|
<define name="meeting">
|
|
129
135
|
<element name="meeting">
|
|
136
|
+
<optional>
|
|
137
|
+
<attribute name="acronym"/>
|
|
138
|
+
</optional>
|
|
139
|
+
<text/>
|
|
140
|
+
</element>
|
|
141
|
+
</define>
|
|
142
|
+
<define name="meeting-place">
|
|
143
|
+
<element name="meeting-place">
|
|
130
144
|
<text/>
|
|
131
145
|
</element>
|
|
132
146
|
</define>
|
|
@@ -20,7 +20,7 @@ module RelatonItu
|
|
|
20
20
|
warn "[relaton-itu] WARNING: invalid bureau: #{bureau}"
|
|
21
21
|
end
|
|
22
22
|
@bureau = bureau
|
|
23
|
-
@group = group.is_a?(Hash) ? ItuGroup.new(group) : group
|
|
23
|
+
@group = group.is_a?(Hash) ? ItuGroup.new(**group) : group
|
|
24
24
|
@subgroup = subgroup.is_a?(Hash) ? ItuGroup.new(subgroup) : subgroup
|
|
25
25
|
@workgroup = workgroup.is_a?(Hash) ? ItuGroup.new(workgroup) : workgroup
|
|
26
26
|
end
|
|
@@ -7,7 +7,7 @@ module RelatonItu
|
|
|
7
7
|
eg = ret[:editorialgroup]
|
|
8
8
|
return unless eg
|
|
9
9
|
|
|
10
|
-
ret[:editorialgroup] = EditorialGroup.new eg
|
|
10
|
+
ret[:editorialgroup] = EditorialGroup.new **eg
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
# @param ret [Hash]
|
|
@@ -15,7 +15,7 @@ module RelatonItu
|
|
|
15
15
|
return unless ret[:structuredidentifier]
|
|
16
16
|
|
|
17
17
|
ret[:structuredidentifier] = StructuredIdentifier.new(
|
|
18
|
-
ret[:structuredidentifier]
|
|
18
|
+
**ret[:structuredidentifier]
|
|
19
19
|
)
|
|
20
20
|
end
|
|
21
21
|
end
|
data/lib/relaton_itu/hit.rb
CHANGED
|
@@ -14,23 +14,35 @@ module RelatonItu
|
|
|
14
14
|
|
|
15
15
|
# @param ref [String]
|
|
16
16
|
# @param year [String]
|
|
17
|
-
def initialize(ref, year = nil)
|
|
17
|
+
def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
18
18
|
text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
|
|
19
19
|
super text, year
|
|
20
20
|
@gi_imp = /\.Imp\d/.match?(ref)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
if ref.match? /^(ITU-T|ITU-R\sRR)/
|
|
22
|
+
uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
|
|
23
|
+
data = { json: params.to_json }
|
|
24
|
+
resp = Net::HTTP.post(uri, data.to_json,
|
|
25
|
+
"Content-Type" => "application/json")
|
|
26
|
+
@array = hits JSON.parse(resp.body)
|
|
27
|
+
elsif ref.match? /^ITU-R/
|
|
28
|
+
rf = ref.sub(/^ITU-R\s/, "").upcase
|
|
29
|
+
url = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/data/#{rf}.yaml"
|
|
30
|
+
hash = YAML.safe_load Net::HTTP.get(URI(url))
|
|
31
|
+
item_hash = HashConverter.hash_to_bib(hash)
|
|
32
|
+
item = ItuBibliographicItem.new **item_hash
|
|
33
|
+
hit = Hit.new({ url: url }, self)
|
|
34
|
+
hit.fetch = item
|
|
35
|
+
@array = [hit]
|
|
36
|
+
end
|
|
26
37
|
end
|
|
27
38
|
|
|
28
39
|
private
|
|
29
40
|
|
|
30
41
|
# @return [String]
|
|
31
42
|
def group
|
|
32
|
-
@group ||=
|
|
33
|
-
|
|
43
|
+
@group ||= case text
|
|
44
|
+
when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
|
|
45
|
+
when %r{^ITU-T} then "Recommendations"
|
|
34
46
|
end
|
|
35
47
|
end
|
|
36
48
|
|
|
@@ -46,7 +58,7 @@ module RelatonItu
|
|
|
46
58
|
"ExactPhrase" => false,
|
|
47
59
|
"CollectionName" => "General",
|
|
48
60
|
"CollectionGroup" => group,
|
|
49
|
-
"Sector" =>
|
|
61
|
+
"Sector" => text.match(/(?<=^ITU-)\w/).to_s.downcase,
|
|
50
62
|
"Criterias" => [{
|
|
51
63
|
"Name" => "Search in",
|
|
52
64
|
"Criterias" => [
|
|
@@ -54,7 +66,7 @@ module RelatonItu
|
|
|
54
66
|
"Selected" => false,
|
|
55
67
|
"Value" => "",
|
|
56
68
|
"Label" => "Name",
|
|
57
|
-
"Target" => "
|
|
69
|
+
"Target" => "\\/name_s",
|
|
58
70
|
"TypeName" => "CHECKBOX",
|
|
59
71
|
"GetCriteriaType" => 0,
|
|
60
72
|
},
|
|
@@ -62,7 +74,7 @@ module RelatonItu
|
|
|
62
74
|
"Selected" => false,
|
|
63
75
|
"Value" => "",
|
|
64
76
|
"Label" => "Short description",
|
|
65
|
-
"Target" => "
|
|
77
|
+
"Target" => "\\/short_description_s",
|
|
66
78
|
"TypeName" => "CHECKBOX",
|
|
67
79
|
"GetCriteriaType" => 0,
|
|
68
80
|
},
|
|
@@ -70,7 +82,7 @@ module RelatonItu
|
|
|
70
82
|
"Selected" => false,
|
|
71
83
|
"Value" => "",
|
|
72
84
|
"Label" => "File content",
|
|
73
|
-
"Target" => "
|
|
85
|
+
"Target" => "\\/file",
|
|
74
86
|
"TypeName" => "CHECKBOX",
|
|
75
87
|
"GetCriteriaType" => 0,
|
|
76
88
|
},
|
|
@@ -79,9 +91,8 @@ module RelatonItu
|
|
|
79
91
|
"Selected" => false,
|
|
80
92
|
}],
|
|
81
93
|
"Topics" => "",
|
|
82
|
-
"ClientData" => {
|
|
94
|
+
"ClientData" => {},
|
|
83
95
|
"Language" => "en",
|
|
84
|
-
"IP" => "",
|
|
85
96
|
"SearchType" => "All",
|
|
86
97
|
}
|
|
87
98
|
end
|
|
@@ -4,7 +4,7 @@ module RelatonItu
|
|
|
4
4
|
recommendation recommendation-supplement recommendation-amendment
|
|
5
5
|
recommendation-corrigendum recommendation-errata recommendation-annex
|
|
6
6
|
focus-group implementers-guide technical-paper technical-report
|
|
7
|
-
joint-itu-iso-iec
|
|
7
|
+
joint-itu-iso-iec resolution service-publication handbook question
|
|
8
8
|
].freeze
|
|
9
9
|
|
|
10
10
|
# @params structuredidentifier [RelatonItu::StructuredIdentifier]
|
|
@@ -30,9 +30,9 @@ module RelatonItu
|
|
|
30
30
|
# @param year [String] the year the standard was published (optional)
|
|
31
31
|
# @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
|
|
32
32
|
# @return [String] Relaton XML serialisation of reference
|
|
33
|
-
def get(code, year = nil, opts = {})
|
|
33
|
+
def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
34
34
|
if year.nil?
|
|
35
|
-
/^(?<code1>[^\s]+\s[^\s]+)\s\(\d{2}\/(?<year1>\d+)\)$/ =~ code
|
|
35
|
+
/^(?<code1>[^\s]+\s[^\s]+)\s\((\d{2}\/)?(?<year1>\d+)\)$/ =~ code
|
|
36
36
|
unless code1.nil?
|
|
37
37
|
code = code1
|
|
38
38
|
year = year1
|
|
@@ -50,13 +50,15 @@ module RelatonItu
|
|
|
50
50
|
|
|
51
51
|
private
|
|
52
52
|
|
|
53
|
-
def fetch_ref_err(code, year, missed_years)
|
|
53
|
+
def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
|
|
54
54
|
id = year ? "#{code}:#{year}" : code
|
|
55
55
|
warn "[relaton-itu] WARNING: no match found online for #{id}. "\
|
|
56
56
|
"The code must be exactly like it is on the standards website."
|
|
57
|
-
|
|
58
|
-
"
|
|
59
|
-
|
|
57
|
+
unless missed_years.empty?
|
|
58
|
+
warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
|
|
59
|
+
"found for #{missed_years.join(', ')}.)"
|
|
60
|
+
end
|
|
61
|
+
if /\d-\d/.match? code
|
|
60
62
|
warn "[relaton-itu] The provided document part may not exist, or the document "\
|
|
61
63
|
"may no longer be published in parts."
|
|
62
64
|
else
|
|
@@ -67,14 +69,35 @@ module RelatonItu
|
|
|
67
69
|
nil
|
|
68
70
|
end
|
|
69
71
|
|
|
70
|
-
def search_filter(code)
|
|
71
|
-
|
|
72
|
-
|
|
72
|
+
def search_filter(code, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
73
|
+
%r{
|
|
74
|
+
^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+)
|
|
75
|
+
(\s\(((?<month1>\d{2})\/)?(?<year1>\d{4})\))?
|
|
76
|
+
(\s-\s(?<buldate1>\d{2}\.\w{1,4}\.\d{4}))?
|
|
77
|
+
(\/(?<corr1>(Amd|Cor)\s?\d+))?
|
|
78
|
+
(\s\(((?<cormonth1>\d{2})\/)?(?<coryear1>\d{4})\))?
|
|
79
|
+
}x =~ code
|
|
80
|
+
year ||= year1
|
|
81
|
+
# docidrx = %r{\w+\.\d+|\w\sSuppl\.\s\d+} # %r{^ITU-T\s[^\s]+}
|
|
82
|
+
# c = code.sub(/Imp\s?/, "").match(docidrx).to_s
|
|
73
83
|
warn "[relaton-itu] (\"#{code}\") fetching..."
|
|
74
84
|
result = search(code)
|
|
85
|
+
code1.sub! /(?<=\.)Imp(?=\d)/, "" if result.gi_imp
|
|
75
86
|
result.select do |i|
|
|
76
|
-
i.hit[:code]
|
|
77
|
-
|
|
87
|
+
next true unless i.hit[:code]
|
|
88
|
+
|
|
89
|
+
%r{
|
|
90
|
+
^(?<pref2>ITU)?(-(?<type2>\w))?\s?(?<code2>[\S]+)
|
|
91
|
+
(\s\(((?<month2>\d{2})\/)?(?<year2>\d{4})\))?
|
|
92
|
+
(\s(?<corr2>(Amd|Cor)\.\s?\d+))?
|
|
93
|
+
(\s\(((?<cormonth2>\d{2})\/)?(?<coryear2>\d{4})\))?
|
|
94
|
+
}x =~ i.hit[:code]
|
|
95
|
+
/:[^\(]+\((?<buldate2>\d{2}\.\w{1,4}\.\d{4})\)/ =~ i.hit[:title]
|
|
96
|
+
corr2&.sub! /\.\s?/, " "
|
|
97
|
+
pref1 == pref2 && (!type1 || type1 == type2) && code1 == code2 &&
|
|
98
|
+
(!year || year == year2) && (!month1 || month1 == month2) &&
|
|
99
|
+
corr1 == corr2 && (!coryear1 || coryear1 == coryear2) &&
|
|
100
|
+
buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2)
|
|
78
101
|
end
|
|
79
102
|
end
|
|
80
103
|
|
|
@@ -86,24 +109,19 @@ module RelatonItu
|
|
|
86
109
|
# If no match, returns any years which caused mismatch, for error reporting
|
|
87
110
|
def isobib_results_filter(result, year)
|
|
88
111
|
missed_years = []
|
|
89
|
-
# result.each_slice(3) do |s| # ISO website only allows 3 connections
|
|
90
|
-
# fetch_pages(s, 3).each do |r|
|
|
91
112
|
result.each do |r|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
return { ret: r.fetch } if year == pyear
|
|
113
|
+
if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
|
|
114
|
+
ret = r.fetch
|
|
115
|
+
return { ret: ret } if ret
|
|
116
|
+
end
|
|
97
117
|
|
|
98
118
|
missed_years << pyear
|
|
99
|
-
# end
|
|
100
|
-
# end
|
|
101
119
|
end
|
|
102
120
|
{ years: missed_years }
|
|
103
121
|
end
|
|
104
122
|
|
|
105
123
|
def itubib_get1(code, year, _opts)
|
|
106
|
-
result = search_filter(code) || return
|
|
124
|
+
result = search_filter(code, year) || return
|
|
107
125
|
ret = isobib_results_filter(result, year)
|
|
108
126
|
if ret[:ret]
|
|
109
127
|
warn "[relaton-itu] (\"#{code}\") found #{ret[:ret].docidentifier.first&.id}"
|
data/lib/relaton_itu/scrapper.rb
CHANGED
|
@@ -31,6 +31,8 @@ module RelatonItu
|
|
|
31
31
|
# @return [Hash]
|
|
32
32
|
def parse_page(hit_data, imp = false)
|
|
33
33
|
url, doc = get_page hit_data[:url]
|
|
34
|
+
return unless doc
|
|
35
|
+
|
|
34
36
|
if imp
|
|
35
37
|
a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
|
|
36
38
|
return unless a
|
|
@@ -44,7 +46,7 @@ module RelatonItu
|
|
|
44
46
|
ItuBibliographicItem.new(
|
|
45
47
|
fetched: Date.today.to_s,
|
|
46
48
|
type: "standard",
|
|
47
|
-
docid: fetch_docid(doc),
|
|
49
|
+
docid: fetch_docid(doc, hit_data[:title]),
|
|
48
50
|
edition: edition,
|
|
49
51
|
language: ["en"],
|
|
50
52
|
script: ["Latn"],
|
|
@@ -59,7 +61,7 @@ module RelatonItu
|
|
|
59
61
|
copyright: fetch_copyright(hit_data[:code], doc),
|
|
60
62
|
link: fetch_link(doc, url),
|
|
61
63
|
relation: fetch_relations(doc),
|
|
62
|
-
place: ["Geneva"]
|
|
64
|
+
place: ["Geneva"]
|
|
63
65
|
)
|
|
64
66
|
end
|
|
65
67
|
# rubocop:enable Metrics/AbcSize
|
|
@@ -69,17 +71,19 @@ module RelatonItu
|
|
|
69
71
|
# Fetch abstracts.
|
|
70
72
|
# @param doc [Nokigiri::HTML::Document]
|
|
71
73
|
# @return [Array<Array>]
|
|
72
|
-
def fetch_abstract(doc)
|
|
74
|
+
def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
|
|
73
75
|
abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
content = if abstract_url
|
|
77
|
+
url = abstract_url[:onclick].match(/https?[^']+/).to_s
|
|
78
|
+
d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
|
|
79
|
+
d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
|
|
80
|
+
elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
|
|
81
|
+
a.text.strip
|
|
82
|
+
end
|
|
83
|
+
return [] unless content
|
|
80
84
|
|
|
81
85
|
[{
|
|
82
|
-
content:
|
|
86
|
+
content: content,
|
|
83
87
|
language: "en",
|
|
84
88
|
script: "Latn",
|
|
85
89
|
}]
|
|
@@ -92,7 +96,9 @@ module RelatonItu
|
|
|
92
96
|
uri = URI url
|
|
93
97
|
resp = Net::HTTP.get_response(uri)
|
|
94
98
|
until resp.code == "200"
|
|
95
|
-
|
|
99
|
+
return if resp["location"] == "/en/publications/pages/notfound.aspx"
|
|
100
|
+
|
|
101
|
+
uri = URI resp["location"] if resp.code.match? /^30/
|
|
96
102
|
resp = Net::HTTP.get_response(uri)
|
|
97
103
|
end
|
|
98
104
|
[uri.to_s, Nokogiri::HTML(resp.body)]
|
|
@@ -105,18 +111,32 @@ module RelatonItu
|
|
|
105
111
|
|
|
106
112
|
# Fetch docid.
|
|
107
113
|
# @param doc [Nokogiri::HTML::Document]
|
|
114
|
+
# @param title [String]
|
|
108
115
|
# @return [Hash]
|
|
109
|
-
def fetch_docid(doc)
|
|
110
|
-
doc.xpath(
|
|
116
|
+
def fetch_docid(doc, title)
|
|
117
|
+
docids = doc.xpath(
|
|
111
118
|
"//span[@id='ctl00_content_main_uc_rec_main_info1_rpt_main_ctl00_lbl_rec']",
|
|
112
119
|
"//td[.='Identical standard:']/following-sibling::td",
|
|
113
|
-
"//div/table[1]/tr[4]/td/strong"
|
|
114
|
-
).map
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
+
"//div/table[1]/tr[4]/td/strong"
|
|
121
|
+
).map { |c| createdocid c.text }
|
|
122
|
+
docids << createdocid(title) unless docids.any?
|
|
123
|
+
docids
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def createdocid(text) # rubocop:disable Metrics/MethodLength
|
|
127
|
+
%r{
|
|
128
|
+
^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
|
|
129
|
+
(\(((?<_month>\d{2})\/)?(?<_year>\d{4})\))?
|
|
130
|
+
(:[^\(]+\((?<buldate>\d{2}\.\w{1,4}\.\d{4})\))?
|
|
131
|
+
(\s(?<corr>(Amd|Cor)\.\s?\d+))?
|
|
132
|
+
# (\s\(((?<_cormonth>\d{2})\/)?(?<_coryear>\d{4})\))?
|
|
133
|
+
}x =~ text.squeeze(" ")
|
|
134
|
+
corr&.sub! /\.\s?/, " "
|
|
135
|
+
id = [code.sub(/[[:space:]]$/, ""), corr].compact.join "/"
|
|
136
|
+
id += " - #{buldate}" if buldate
|
|
137
|
+
type = id.match(%r{^\w+}).to_s
|
|
138
|
+
type = "ITU" if type == "G"
|
|
139
|
+
RelatonBib::DocumentIdentifier.new(type: type, id: id)
|
|
120
140
|
end
|
|
121
141
|
|
|
122
142
|
# Fetch status.
|
|
@@ -142,13 +162,13 @@ module RelatonItu
|
|
|
142
162
|
group = wg && itugroup(wg.text)
|
|
143
163
|
EditorialGroup.new(
|
|
144
164
|
bureau: code.match(/(?<=-)./).to_s,
|
|
145
|
-
group: group
|
|
165
|
+
group: group
|
|
146
166
|
)
|
|
147
167
|
end
|
|
148
168
|
|
|
149
169
|
# @param name [String]
|
|
150
170
|
# @return [RelatonItu::ItuGroup]
|
|
151
|
-
def itugroup(name)
|
|
171
|
+
def itugroup(name) # rubocop:disable Metrics/MethodLength
|
|
152
172
|
if name.include? "Study Group"
|
|
153
173
|
type = "study-group"
|
|
154
174
|
acronym = "SG"
|
|
@@ -190,13 +210,15 @@ module RelatonItu
|
|
|
190
210
|
# Fetch dates
|
|
191
211
|
# @param doc [Nokogiri::HTML::Document]
|
|
192
212
|
# @return [Array<Hash>]
|
|
193
|
-
def fetch_dates(doc)
|
|
213
|
+
def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
|
194
214
|
dates = []
|
|
195
215
|
date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]",
|
|
196
216
|
"//p[contains(.,'Approved in')]")
|
|
197
217
|
pdate = date&.text&.match(/\d{4}-\d{2}-\d{2}/).to_s || ob_date(doc)
|
|
198
218
|
if pdate && !pdate&.empty?
|
|
199
219
|
dates << { type: "published", on: pdate }
|
|
220
|
+
elsif pdate = ob_date(doc)
|
|
221
|
+
dates << { type: "published", on: pdate }
|
|
200
222
|
end
|
|
201
223
|
dates
|
|
202
224
|
end
|
|
@@ -208,7 +230,7 @@ module RelatonItu
|
|
|
208
230
|
pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
|
|
209
231
|
return unless pdate
|
|
210
232
|
|
|
211
|
-
roman_to_arabic pdate.text.match(%r{(?<=Year: )\d{2}.\w
|
|
233
|
+
roman_to_arabic pdate.text.match(%r{(?<=Year: )(\d{2}.\w+.)?\d{4}}).to_s
|
|
212
234
|
end
|
|
213
235
|
|
|
214
236
|
# Convert roman month number in string date to arabic number
|
|
@@ -216,8 +238,11 @@ module RelatonItu
|
|
|
216
238
|
# @return [String]
|
|
217
239
|
def roman_to_arabic(date)
|
|
218
240
|
%r{(?<rmonth>[IVX]+)} =~ date
|
|
219
|
-
|
|
220
|
-
|
|
241
|
+
if ROMAN_MONTHS.index(rmonth)
|
|
242
|
+
month = ROMAN_MONTHS.index(rmonth) + 1
|
|
243
|
+
Date.parse(date.sub(%r{[IVX]+}, month.to_s)).to_s
|
|
244
|
+
else date
|
|
245
|
+
end
|
|
221
246
|
end
|
|
222
247
|
|
|
223
248
|
# Fetch contributors
|
|
@@ -244,7 +269,7 @@ module RelatonItu
|
|
|
244
269
|
links = [{ type: "src", content: url }]
|
|
245
270
|
obp_elm = doc.at(
|
|
246
271
|
'//a[@title="Persistent link to download the PDF file"]',
|
|
247
|
-
"//font[contains(.,'PDF')]/../.."
|
|
272
|
+
"//font[contains(.,'PDF')]/../.."
|
|
248
273
|
)
|
|
249
274
|
links << typed_link("obp", obp_elm) if obp_elm
|
|
250
275
|
wrd_elm = doc.at("//font[contains(.,'Word')]/../..")
|
data/lib/relaton_itu/version.rb
CHANGED
data/relaton-itu.gemspec
CHANGED
|
@@ -26,16 +26,16 @@ Gem::Specification.new do |spec|
|
|
|
26
26
|
spec.require_paths = ["lib"]
|
|
27
27
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
|
28
28
|
|
|
29
|
-
spec.add_development_dependency "debase"
|
|
29
|
+
# spec.add_development_dependency "debase"
|
|
30
30
|
spec.add_development_dependency "equivalent-xml", "~> 0.6"
|
|
31
31
|
spec.add_development_dependency "pry-byebug"
|
|
32
32
|
spec.add_development_dependency "rake", "~> 10.0"
|
|
33
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
|
34
|
-
spec.add_development_dependency "ruby-debug-ide"
|
|
34
|
+
# spec.add_development_dependency "ruby-debug-ide"
|
|
35
35
|
spec.add_development_dependency "ruby-jing"
|
|
36
36
|
spec.add_development_dependency "simplecov"
|
|
37
37
|
spec.add_development_dependency "vcr", "~> 5.0.0"
|
|
38
38
|
spec.add_development_dependency "webmock"
|
|
39
39
|
|
|
40
|
-
spec.add_dependency "relaton-bib", "~> 1.
|
|
40
|
+
spec.add_dependency "relaton-bib", "~> 1.7.0"
|
|
41
41
|
end
|
metadata
CHANGED
|
@@ -1,29 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton-itu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.7.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-03-18 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
|
-
- !ruby/object:Gem::Dependency
|
|
14
|
-
name: debase
|
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ">="
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '0'
|
|
20
|
-
type: :development
|
|
21
|
-
prerelease: false
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
-
requirements:
|
|
24
|
-
- - ">="
|
|
25
|
-
- !ruby/object:Gem::Version
|
|
26
|
-
version: '0'
|
|
27
13
|
- !ruby/object:Gem::Dependency
|
|
28
14
|
name: equivalent-xml
|
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -80,20 +66,6 @@ dependencies:
|
|
|
80
66
|
- - "~>"
|
|
81
67
|
- !ruby/object:Gem::Version
|
|
82
68
|
version: '3.0'
|
|
83
|
-
- !ruby/object:Gem::Dependency
|
|
84
|
-
name: ruby-debug-ide
|
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
|
86
|
-
requirements:
|
|
87
|
-
- - ">="
|
|
88
|
-
- !ruby/object:Gem::Version
|
|
89
|
-
version: '0'
|
|
90
|
-
type: :development
|
|
91
|
-
prerelease: false
|
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
-
requirements:
|
|
94
|
-
- - ">="
|
|
95
|
-
- !ruby/object:Gem::Version
|
|
96
|
-
version: '0'
|
|
97
69
|
- !ruby/object:Gem::Dependency
|
|
98
70
|
name: ruby-jing
|
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -156,14 +128,14 @@ dependencies:
|
|
|
156
128
|
requirements:
|
|
157
129
|
- - "~>"
|
|
158
130
|
- !ruby/object:Gem::Version
|
|
159
|
-
version: 1.
|
|
131
|
+
version: 1.7.0
|
|
160
132
|
type: :runtime
|
|
161
133
|
prerelease: false
|
|
162
134
|
version_requirements: !ruby/object:Gem::Requirement
|
|
163
135
|
requirements:
|
|
164
136
|
- - "~>"
|
|
165
137
|
- !ruby/object:Gem::Version
|
|
166
|
-
version: 1.
|
|
138
|
+
version: 1.7.0
|
|
167
139
|
description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
|
|
168
140
|
model'
|
|
169
141
|
email:
|
|
@@ -172,9 +144,7 @@ executables: []
|
|
|
172
144
|
extensions: []
|
|
173
145
|
extra_rdoc_files: []
|
|
174
146
|
files:
|
|
175
|
-
- ".github/workflows/
|
|
176
|
-
- ".github/workflows/ubuntu.yml"
|
|
177
|
-
- ".github/workflows/windows.yml"
|
|
147
|
+
- ".github/workflows/rake.yml"
|
|
178
148
|
- ".gitignore"
|
|
179
149
|
- ".rspec"
|
|
180
150
|
- ".rubocop.yml"
|
data/.github/workflows/macos.yml
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: macos
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-macos:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} macOS
|
|
14
|
-
runs-on: macos-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
run: |
|
|
28
|
-
sudo gem install bundler --force
|
|
29
|
-
ruby -v | grep 2.5 && bundle config set build.debase --with-cflags="-Wno-error=implicit-function-declaration"
|
|
30
|
-
ruby -v | grep 2.5 && bundle config set build.ruby-debug-ide --with-cflags="-Wno-error=implicit-function-declaration"
|
|
31
|
-
bundle install --jobs 4 --retry 3
|
|
32
|
-
- name: Run specs
|
|
33
|
-
run: |
|
|
34
|
-
bundle exec rake
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: ubuntu
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-linux:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Ubuntu
|
|
14
|
-
runs-on: ubuntu-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
run: |
|
|
28
|
-
gem install bundler
|
|
29
|
-
bundle install --jobs 4 --retry 3
|
|
30
|
-
- name: Run specs
|
|
31
|
-
run: |
|
|
32
|
-
unset JAVA_TOOL_OPTIONS
|
|
33
|
-
bundle exec rake
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: windows
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-windows:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Windows
|
|
14
|
-
runs-on: windows-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
shell: pwsh
|
|
28
|
-
run: |
|
|
29
|
-
gem install bundler
|
|
30
|
-
bundle config --local path vendor/bundle
|
|
31
|
-
bundle update
|
|
32
|
-
bundle install --jobs 4 --retry 3
|
|
33
|
-
- name: Run specs
|
|
34
|
-
run: |
|
|
35
|
-
bundle exec rake
|