relaton-itu 1.7.0 → 1.7.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/rake.yml +46 -0
- data/README.adoc +34 -5
- data/grammars/itu.rng +13 -0
- data/lib/relaton_itu/editorial_group.rb +1 -1
- data/lib/relaton_itu/hash_converter.rb +2 -2
- data/lib/relaton_itu/hit.rb +2 -0
- data/lib/relaton_itu/hit_collection.rb +25 -14
- data/lib/relaton_itu/itu_bibliographic_item.rb +8 -1
- data/lib/relaton_itu/itu_bibliography.rb +39 -21
- data/lib/relaton_itu/itu_group.rb +1 -1
- data/lib/relaton_itu/processor.rb +1 -2
- data/lib/relaton_itu/scrapper.rb +52 -27
- data/lib/relaton_itu/version.rb +1 -1
- data/lib/relaton_itu/xml_parser.rb +1 -1
- data/relaton-itu.gemspec +2 -2
- metadata +4 -34
- data/.github/workflows/macos.yml +0 -34
- data/.github/workflows/ubuntu.yml +0 -33
- data/.github/workflows/windows.yml +0 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2764e22b822215ec681d22f20b979cc80cb498ffa49d87a164a6118575dd686f
|
4
|
+
data.tar.gz: 9863ca9b6b6d9f76bb3265ef148115de055dd82445c5c058564345e963ada182
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '087567111db88be5ee21f16942ce4b80fc07537665b565fb4b0913edcd3e6427196e541775dfaf142ddf9c0b1693d8e0067d7bbbf9602a87293890d6074089ac'
|
7
|
+
data.tar.gz: f9ca1d2439bcef8bc110a8cfe10cb22ea47e2000b289f3c446459c9e51d036bbf2bb7b1f8c99b798735b02522f7e4985aa78f1cefe36ff868f301795858abdaa
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# Auto-generated by Cimas: Do not edit it manually!
|
2
|
+
# See https://github.com/metanorma/cimas
|
3
|
+
name: rake
|
4
|
+
|
5
|
+
on:
|
6
|
+
push:
|
7
|
+
branches: [ master, main ]
|
8
|
+
tags: [ v* ]
|
9
|
+
pull_request:
|
10
|
+
|
11
|
+
jobs:
|
12
|
+
rake:
|
13
|
+
name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
|
14
|
+
runs-on: ${{ matrix.os }}
|
15
|
+
continue-on-error: ${{ matrix.experimental }}
|
16
|
+
strategy:
|
17
|
+
fail-fast: false
|
18
|
+
matrix:
|
19
|
+
ruby: [ '2.7', '2.6', '2.5', '2.4' ]
|
20
|
+
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
21
|
+
experimental: [ false ]
|
22
|
+
include:
|
23
|
+
- ruby: '3.0'
|
24
|
+
os: 'ubuntu-latest'
|
25
|
+
experimental: true
|
26
|
+
- ruby: '3.0'
|
27
|
+
os: 'windows-latest'
|
28
|
+
experimental: true
|
29
|
+
- ruby: '3.0'
|
30
|
+
os: 'macos-latest'
|
31
|
+
experimental: true
|
32
|
+
steps:
|
33
|
+
- uses: actions/checkout@v2
|
34
|
+
with:
|
35
|
+
submodules: true
|
36
|
+
|
37
|
+
# https://github.com/ruby-debug/debase/issues/89#issuecomment-686827382
|
38
|
+
- if: matrix.os == 'macos-latest' && matrix.ruby == '2.5'
|
39
|
+
run: echo BUNDLE_BUILD__DEBASE="--with-cflags=\"-Wno-error=implicit-function-declaration\"" >> $GITHUB_ENV
|
40
|
+
|
41
|
+
- uses: ruby/setup-ruby@v1
|
42
|
+
with:
|
43
|
+
ruby-version: ${{ matrix.ruby }}
|
44
|
+
bundler-cache: true
|
45
|
+
|
46
|
+
- run: bundle exec rake
|
data/README.adoc
CHANGED
@@ -87,6 +87,39 @@ RelatonItu::ItuBibliography.get("ITU-T L.163", "2018", {})
|
|
87
87
|
...
|
88
88
|
----
|
89
89
|
|
90
|
+
=== Get amendment
|
91
|
+
[source,ruby]
|
92
|
+
----
|
93
|
+
RelatonItu::ItuBibliography.get "ITU-T G.989.2/Amd 1"
|
94
|
+
[relaton-itu] ("ITU-T G.989.2/Amd 1") fetching...
|
95
|
+
[relaton-itu] ("ITU-T G.989.2/Amd 1") found ITU-T G.989.2/Amd 1
|
96
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fe5409e5840
|
97
|
+
...
|
98
|
+
----
|
99
|
+
|
100
|
+
=== Get ITU-R documents
|
101
|
+
|
102
|
+
[source,ruby]
|
103
|
+
----
|
104
|
+
RelatonItu::ItuBibliography.get "ITU-R REC-BO.600-1"
|
105
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") fetching...
|
106
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") found R-REC-BO.600-1
|
107
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007f84e9930498
|
108
|
+
...
|
109
|
+
----
|
110
|
+
|
111
|
+
=== Get ITU-R Radio Regulations (RR)
|
112
|
+
|
113
|
+
[source,ruby]
|
114
|
+
----
|
115
|
+
RelatonItu::ItuBibliography.get 'ITU-R RR (2020)'
|
116
|
+
[relaton-itu] ("ITU-R RR") fetching...
|
117
|
+
[relaton-itu] WARNING: invalid doctype: publication
|
118
|
+
[relaton-itu] ("ITU-R RR") found ITU-R RR
|
119
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fb4b55eb660
|
120
|
+
...
|
121
|
+
----
|
122
|
+
|
90
123
|
=== Create bibliographic item form YAML
|
91
124
|
[source,ruby]
|
92
125
|
----
|
@@ -94,11 +127,7 @@ hash = YAML.load_file 'spec/examples/itu_bib_item.yml'
|
|
94
127
|
=> {"id"=>"ITU-T L.163 (11/2018)",
|
95
128
|
...
|
96
129
|
|
97
|
-
|
98
|
-
=> {:id=>"ITU-T L.163 (11/2018)",
|
99
|
-
...
|
100
|
-
|
101
|
-
RelatonItu::ItuBibliographicItem.new bib_hash
|
130
|
+
RelatonItu::ItuBibliographicItem.from_hash hash
|
102
131
|
=> #<RelatonItu::ItuBibliographicItem:0x007fd88ac02aa0
|
103
132
|
...
|
104
133
|
----
|
data/grammars/itu.rng
CHANGED
@@ -42,6 +42,8 @@
|
|
42
42
|
<value>joint-itu-iso-iec</value>
|
43
43
|
<value>resolution</value>
|
44
44
|
<value>service-publication</value>
|
45
|
+
<value>handbook</value>
|
46
|
+
<value>question</value>
|
45
47
|
</choice>
|
46
48
|
</define>
|
47
49
|
<define name="editorialgroup">
|
@@ -98,6 +100,9 @@
|
|
98
100
|
<optional>
|
99
101
|
<ref name="meeting"/>
|
100
102
|
</optional>
|
103
|
+
<optional>
|
104
|
+
<ref name="meeting-place"/>
|
105
|
+
</optional>
|
101
106
|
<optional>
|
102
107
|
<ref name="meeting-date"/>
|
103
108
|
</optional>
|
@@ -128,6 +133,14 @@
|
|
128
133
|
</include>
|
129
134
|
<define name="meeting">
|
130
135
|
<element name="meeting">
|
136
|
+
<optional>
|
137
|
+
<attribute name="acronym"/>
|
138
|
+
</optional>
|
139
|
+
<text/>
|
140
|
+
</element>
|
141
|
+
</define>
|
142
|
+
<define name="meeting-place">
|
143
|
+
<element name="meeting-place">
|
131
144
|
<text/>
|
132
145
|
</element>
|
133
146
|
</define>
|
@@ -20,7 +20,7 @@ module RelatonItu
|
|
20
20
|
warn "[relaton-itu] WARNING: invalid bureau: #{bureau}"
|
21
21
|
end
|
22
22
|
@bureau = bureau
|
23
|
-
@group = group.is_a?(Hash) ? ItuGroup.new(group) : group
|
23
|
+
@group = group.is_a?(Hash) ? ItuGroup.new(**group) : group
|
24
24
|
@subgroup = subgroup.is_a?(Hash) ? ItuGroup.new(subgroup) : subgroup
|
25
25
|
@workgroup = workgroup.is_a?(Hash) ? ItuGroup.new(workgroup) : workgroup
|
26
26
|
end
|
@@ -7,7 +7,7 @@ module RelatonItu
|
|
7
7
|
eg = ret[:editorialgroup]
|
8
8
|
return unless eg
|
9
9
|
|
10
|
-
ret[:editorialgroup] = EditorialGroup.new eg
|
10
|
+
ret[:editorialgroup] = EditorialGroup.new **eg
|
11
11
|
end
|
12
12
|
|
13
13
|
# @param ret [Hash]
|
@@ -15,7 +15,7 @@ module RelatonItu
|
|
15
15
|
return unless ret[:structuredidentifier]
|
16
16
|
|
17
17
|
ret[:structuredidentifier] = StructuredIdentifier.new(
|
18
|
-
ret[:structuredidentifier]
|
18
|
+
**ret[:structuredidentifier]
|
19
19
|
)
|
20
20
|
end
|
21
21
|
end
|
data/lib/relaton_itu/hit.rb
CHANGED
@@ -14,23 +14,35 @@ module RelatonItu
|
|
14
14
|
|
15
15
|
# @param ref [String]
|
16
16
|
# @param year [String]
|
17
|
-
def initialize(ref, year = nil)
|
17
|
+
def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
18
18
|
text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
|
19
19
|
super text, year
|
20
20
|
@gi_imp = /\.Imp\d/.match?(ref)
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
21
|
+
if ref.match? /^(ITU-T|ITU-R\sRR)/
|
22
|
+
uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
|
23
|
+
data = { json: params.to_json }
|
24
|
+
resp = Net::HTTP.post(uri, data.to_json,
|
25
|
+
"Content-Type" => "application/json")
|
26
|
+
@array = hits JSON.parse(resp.body)
|
27
|
+
elsif ref.match? /^ITU-R/
|
28
|
+
rf = ref.sub(/^ITU-R\s/, "").upcase
|
29
|
+
url = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/data/#{rf}.yaml"
|
30
|
+
hash = YAML.safe_load Net::HTTP.get(URI(url))
|
31
|
+
item_hash = HashConverter.hash_to_bib(hash)
|
32
|
+
item = ItuBibliographicItem.new **item_hash
|
33
|
+
hit = Hit.new({ url: url }, self)
|
34
|
+
hit.fetch = item
|
35
|
+
@array = [hit]
|
36
|
+
end
|
26
37
|
end
|
27
38
|
|
28
39
|
private
|
29
40
|
|
30
41
|
# @return [String]
|
31
42
|
def group
|
32
|
-
@group ||=
|
33
|
-
|
43
|
+
@group ||= case text
|
44
|
+
when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
|
45
|
+
when %r{^ITU-T} then "Recommendations"
|
34
46
|
end
|
35
47
|
end
|
36
48
|
|
@@ -46,7 +58,7 @@ module RelatonItu
|
|
46
58
|
"ExactPhrase" => false,
|
47
59
|
"CollectionName" => "General",
|
48
60
|
"CollectionGroup" => group,
|
49
|
-
"Sector" =>
|
61
|
+
"Sector" => text.match(/(?<=^ITU-)\w/).to_s.downcase,
|
50
62
|
"Criterias" => [{
|
51
63
|
"Name" => "Search in",
|
52
64
|
"Criterias" => [
|
@@ -54,7 +66,7 @@ module RelatonItu
|
|
54
66
|
"Selected" => false,
|
55
67
|
"Value" => "",
|
56
68
|
"Label" => "Name",
|
57
|
-
"Target" => "
|
69
|
+
"Target" => "\\/name_s",
|
58
70
|
"TypeName" => "CHECKBOX",
|
59
71
|
"GetCriteriaType" => 0,
|
60
72
|
},
|
@@ -62,7 +74,7 @@ module RelatonItu
|
|
62
74
|
"Selected" => false,
|
63
75
|
"Value" => "",
|
64
76
|
"Label" => "Short description",
|
65
|
-
"Target" => "
|
77
|
+
"Target" => "\\/short_description_s",
|
66
78
|
"TypeName" => "CHECKBOX",
|
67
79
|
"GetCriteriaType" => 0,
|
68
80
|
},
|
@@ -70,7 +82,7 @@ module RelatonItu
|
|
70
82
|
"Selected" => false,
|
71
83
|
"Value" => "",
|
72
84
|
"Label" => "File content",
|
73
|
-
"Target" => "
|
85
|
+
"Target" => "\\/file",
|
74
86
|
"TypeName" => "CHECKBOX",
|
75
87
|
"GetCriteriaType" => 0,
|
76
88
|
},
|
@@ -79,9 +91,8 @@ module RelatonItu
|
|
79
91
|
"Selected" => false,
|
80
92
|
}],
|
81
93
|
"Topics" => "",
|
82
|
-
"ClientData" => {
|
94
|
+
"ClientData" => {},
|
83
95
|
"Language" => "en",
|
84
|
-
"IP" => "",
|
85
96
|
"SearchType" => "All",
|
86
97
|
}
|
87
98
|
end
|
@@ -4,7 +4,7 @@ module RelatonItu
|
|
4
4
|
recommendation recommendation-supplement recommendation-amendment
|
5
5
|
recommendation-corrigendum recommendation-errata recommendation-annex
|
6
6
|
focus-group implementers-guide technical-paper technical-report
|
7
|
-
joint-itu-iso-iec
|
7
|
+
joint-itu-iso-iec resolution service-publication handbook question
|
8
8
|
].freeze
|
9
9
|
|
10
10
|
# @params structuredidentifier [RelatonItu::StructuredIdentifier]
|
@@ -14,5 +14,12 @@ module RelatonItu
|
|
14
14
|
end
|
15
15
|
super
|
16
16
|
end
|
17
|
+
|
18
|
+
# @param hash [Hash]
|
19
|
+
# @return [RelatonItu::ItuBibliographicItem]
|
20
|
+
def self.from_hash(hash)
|
21
|
+
item_hash = ::RelatonItu::HashConverter.hash_to_bib(hash)
|
22
|
+
new **item_hash
|
23
|
+
end
|
17
24
|
end
|
18
25
|
end
|
@@ -30,9 +30,9 @@ module RelatonItu
|
|
30
30
|
# @param year [String] the year the standard was published (optional)
|
31
31
|
# @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
|
32
32
|
# @return [String] Relaton XML serialisation of reference
|
33
|
-
def get(code, year = nil, opts = {})
|
33
|
+
def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
34
34
|
if year.nil?
|
35
|
-
/^(?<code1>[^\s]+\s[^\s]+)\s\(\d{2}\/(?<year1>\d+)\)$/ =~ code
|
35
|
+
/^(?<code1>[^\s]+\s[^\s]+)\s\((\d{2}\/)?(?<year1>\d+)\)$/ =~ code
|
36
36
|
unless code1.nil?
|
37
37
|
code = code1
|
38
38
|
year = year1
|
@@ -50,13 +50,15 @@ module RelatonItu
|
|
50
50
|
|
51
51
|
private
|
52
52
|
|
53
|
-
def fetch_ref_err(code, year, missed_years)
|
53
|
+
def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
|
54
54
|
id = year ? "#{code}:#{year}" : code
|
55
55
|
warn "[relaton-itu] WARNING: no match found online for #{id}. "\
|
56
56
|
"The code must be exactly like it is on the standards website."
|
57
|
-
|
58
|
-
"
|
59
|
-
|
57
|
+
unless missed_years.empty?
|
58
|
+
warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
|
59
|
+
"found for #{missed_years.join(', ')}.)"
|
60
|
+
end
|
61
|
+
if /\d-\d/.match? code
|
60
62
|
warn "[relaton-itu] The provided document part may not exist, or the document "\
|
61
63
|
"may no longer be published in parts."
|
62
64
|
else
|
@@ -67,14 +69,35 @@ module RelatonItu
|
|
67
69
|
nil
|
68
70
|
end
|
69
71
|
|
70
|
-
def search_filter(code)
|
71
|
-
|
72
|
-
|
72
|
+
def search_filter(code, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
73
|
+
%r{
|
74
|
+
^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+)
|
75
|
+
(\s\(((?<month1>\d{2})\/)?(?<year1>\d{4})\))?
|
76
|
+
(\s-\s(?<buldate1>\d{2}\.\w{1,4}\.\d{4}))?
|
77
|
+
(\/(?<corr1>(Amd|Cor)\s?\d+))?
|
78
|
+
(\s\(((?<cormonth1>\d{2})\/)?(?<coryear1>\d{4})\))?
|
79
|
+
}x =~ code
|
80
|
+
year ||= year1
|
81
|
+
# docidrx = %r{\w+\.\d+|\w\sSuppl\.\s\d+} # %r{^ITU-T\s[^\s]+}
|
82
|
+
# c = code.sub(/Imp\s?/, "").match(docidrx).to_s
|
73
83
|
warn "[relaton-itu] (\"#{code}\") fetching..."
|
74
84
|
result = search(code)
|
85
|
+
code1.sub! /(?<=\.)Imp(?=\d)/, "" if result.gi_imp
|
75
86
|
result.select do |i|
|
76
|
-
i.hit[:code]
|
77
|
-
|
87
|
+
next true unless i.hit[:code]
|
88
|
+
|
89
|
+
%r{
|
90
|
+
^(?<pref2>ITU)?(-(?<type2>\w))?\s?(?<code2>[\S]+)
|
91
|
+
(\s\(((?<month2>\d{2})\/)?(?<year2>\d{4})\))?
|
92
|
+
(\s(?<corr2>(Amd|Cor)\.\s?\d+))?
|
93
|
+
(\s\(((?<cormonth2>\d{2})\/)?(?<coryear2>\d{4})\))?
|
94
|
+
}x =~ i.hit[:code]
|
95
|
+
/:[^\(]+\((?<buldate2>\d{2}\.\w{1,4}\.\d{4})\)/ =~ i.hit[:title]
|
96
|
+
corr2&.sub! /\.\s?/, " "
|
97
|
+
pref1 == pref2 && (!type1 || type1 == type2) && code1 == code2 &&
|
98
|
+
(!year || year == year2) && (!month1 || month1 == month2) &&
|
99
|
+
corr1 == corr2 && (!coryear1 || coryear1 == coryear2) &&
|
100
|
+
buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2)
|
78
101
|
end
|
79
102
|
end
|
80
103
|
|
@@ -86,24 +109,19 @@ module RelatonItu
|
|
86
109
|
# If no match, returns any years which caused mismatch, for error reporting
|
87
110
|
def isobib_results_filter(result, year)
|
88
111
|
missed_years = []
|
89
|
-
# result.each_slice(3) do |s| # ISO website only allows 3 connections
|
90
|
-
# fetch_pages(s, 3).each do |r|
|
91
112
|
result.each do |r|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
return { ret: r.fetch } if year == pyear
|
113
|
+
if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
|
114
|
+
ret = r.fetch
|
115
|
+
return { ret: ret } if ret
|
116
|
+
end
|
97
117
|
|
98
118
|
missed_years << pyear
|
99
|
-
# end
|
100
|
-
# end
|
101
119
|
end
|
102
120
|
{ years: missed_years }
|
103
121
|
end
|
104
122
|
|
105
123
|
def itubib_get1(code, year, _opts)
|
106
|
-
result = search_filter(code) || return
|
124
|
+
result = search_filter(code, year) || return
|
107
125
|
ret = isobib_results_filter(result, year)
|
108
126
|
if ret[:ret]
|
109
127
|
warn "[relaton-itu] (\"#{code}\") found #{ret[:ret].docidentifier.first&.id}"
|
@@ -26,8 +26,7 @@ module RelatonItu
|
|
26
26
|
# @param hash [Hash]
|
27
27
|
# @return [RelatonItu::ItuBibliographicItem]
|
28
28
|
def hash_to_bib(hash)
|
29
|
-
|
30
|
-
::RelatonItu::ItuBibliographicItem.new item_hash
|
29
|
+
::RelatonItu::ItuBibliographicItem.from_hash hash
|
31
30
|
end
|
32
31
|
|
33
32
|
# Returns hash of XML grammar
|
data/lib/relaton_itu/scrapper.rb
CHANGED
@@ -31,6 +31,8 @@ module RelatonItu
|
|
31
31
|
# @return [Hash]
|
32
32
|
def parse_page(hit_data, imp = false)
|
33
33
|
url, doc = get_page hit_data[:url]
|
34
|
+
return unless doc
|
35
|
+
|
34
36
|
if imp
|
35
37
|
a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
|
36
38
|
return unless a
|
@@ -44,7 +46,7 @@ module RelatonItu
|
|
44
46
|
ItuBibliographicItem.new(
|
45
47
|
fetched: Date.today.to_s,
|
46
48
|
type: "standard",
|
47
|
-
docid: fetch_docid(doc),
|
49
|
+
docid: fetch_docid(doc, hit_data[:title]),
|
48
50
|
edition: edition,
|
49
51
|
language: ["en"],
|
50
52
|
script: ["Latn"],
|
@@ -59,7 +61,7 @@ module RelatonItu
|
|
59
61
|
copyright: fetch_copyright(hit_data[:code], doc),
|
60
62
|
link: fetch_link(doc, url),
|
61
63
|
relation: fetch_relations(doc),
|
62
|
-
place: ["Geneva"]
|
64
|
+
place: ["Geneva"]
|
63
65
|
)
|
64
66
|
end
|
65
67
|
# rubocop:enable Metrics/AbcSize
|
@@ -69,17 +71,19 @@ module RelatonItu
|
|
69
71
|
# Fetch abstracts.
|
70
72
|
# @param doc [Nokigiri::HTML::Document]
|
71
73
|
# @return [Array<Array>]
|
72
|
-
def fetch_abstract(doc)
|
74
|
+
def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
|
73
75
|
abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
76
|
+
content = if abstract_url
|
77
|
+
url = abstract_url[:onclick].match(/https?[^']+/).to_s
|
78
|
+
d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
|
79
|
+
d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
|
80
|
+
elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
|
81
|
+
a.text.strip
|
82
|
+
end
|
83
|
+
return [] unless content
|
80
84
|
|
81
85
|
[{
|
82
|
-
content:
|
86
|
+
content: content,
|
83
87
|
language: "en",
|
84
88
|
script: "Latn",
|
85
89
|
}]
|
@@ -92,7 +96,9 @@ module RelatonItu
|
|
92
96
|
uri = URI url
|
93
97
|
resp = Net::HTTP.get_response(uri)
|
94
98
|
until resp.code == "200"
|
95
|
-
|
99
|
+
return if resp["location"] == "/en/publications/pages/notfound.aspx"
|
100
|
+
|
101
|
+
uri = URI resp["location"] if resp.code.match? /^30/
|
96
102
|
resp = Net::HTTP.get_response(uri)
|
97
103
|
end
|
98
104
|
[uri.to_s, Nokogiri::HTML(resp.body)]
|
@@ -105,18 +111,32 @@ module RelatonItu
|
|
105
111
|
|
106
112
|
# Fetch docid.
|
107
113
|
# @param doc [Nokogiri::HTML::Document]
|
114
|
+
# @param title [String]
|
108
115
|
# @return [Hash]
|
109
|
-
def fetch_docid(doc)
|
110
|
-
doc.xpath(
|
116
|
+
def fetch_docid(doc, title)
|
117
|
+
docids = doc.xpath(
|
111
118
|
"//span[@id='ctl00_content_main_uc_rec_main_info1_rpt_main_ctl00_lbl_rec']",
|
112
119
|
"//td[.='Identical standard:']/following-sibling::td",
|
113
|
-
"//div/table[1]/tr[4]/td/strong"
|
114
|
-
).map
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
+
"//div/table[1]/tr[4]/td/strong"
|
121
|
+
).map { |c| createdocid c.text }
|
122
|
+
docids << createdocid(title) unless docids.any?
|
123
|
+
docids
|
124
|
+
end
|
125
|
+
|
126
|
+
def createdocid(text) # rubocop:disable Metrics/MethodLength
|
127
|
+
%r{
|
128
|
+
^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
|
129
|
+
(\(((?<_month>\d{2})\/)?(?<_year>\d{4})\))?
|
130
|
+
(:[^\(]+\((?<buldate>\d{2}\.\w{1,4}\.\d{4})\))?
|
131
|
+
(\s(?<corr>(Amd|Cor)\.\s?\d+))?
|
132
|
+
# (\s\(((?<_cormonth>\d{2})\/)?(?<_coryear>\d{4})\))?
|
133
|
+
}x =~ text.squeeze(" ")
|
134
|
+
corr&.sub! /\.\s?/, " "
|
135
|
+
id = [code.sub(/[[:space:]]$/, ""), corr].compact.join "/"
|
136
|
+
id += " - #{buldate}" if buldate
|
137
|
+
type = id.match(%r{^\w+}).to_s
|
138
|
+
type = "ITU" if type == "G"
|
139
|
+
RelatonBib::DocumentIdentifier.new(type: type, id: id)
|
120
140
|
end
|
121
141
|
|
122
142
|
# Fetch status.
|
@@ -142,13 +162,13 @@ module RelatonItu
|
|
142
162
|
group = wg && itugroup(wg.text)
|
143
163
|
EditorialGroup.new(
|
144
164
|
bureau: code.match(/(?<=-)./).to_s,
|
145
|
-
group: group
|
165
|
+
group: group
|
146
166
|
)
|
147
167
|
end
|
148
168
|
|
149
169
|
# @param name [String]
|
150
170
|
# @return [RelatonItu::ItuGroup]
|
151
|
-
def itugroup(name)
|
171
|
+
def itugroup(name) # rubocop:disable Metrics/MethodLength
|
152
172
|
if name.include? "Study Group"
|
153
173
|
type = "study-group"
|
154
174
|
acronym = "SG"
|
@@ -190,13 +210,15 @@ module RelatonItu
|
|
190
210
|
# Fetch dates
|
191
211
|
# @param doc [Nokogiri::HTML::Document]
|
192
212
|
# @return [Array<Hash>]
|
193
|
-
def fetch_dates(doc)
|
213
|
+
def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
194
214
|
dates = []
|
195
215
|
date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]",
|
196
216
|
"//p[contains(.,'Approved in')]")
|
197
217
|
pdate = date&.text&.match(/\d{4}-\d{2}-\d{2}/).to_s || ob_date(doc)
|
198
218
|
if pdate && !pdate&.empty?
|
199
219
|
dates << { type: "published", on: pdate }
|
220
|
+
elsif pdate = ob_date(doc)
|
221
|
+
dates << { type: "published", on: pdate }
|
200
222
|
end
|
201
223
|
dates
|
202
224
|
end
|
@@ -208,7 +230,7 @@ module RelatonItu
|
|
208
230
|
pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
|
209
231
|
return unless pdate
|
210
232
|
|
211
|
-
roman_to_arabic pdate.text.match(%r{(?<=Year: )\d{2}.\w
|
233
|
+
roman_to_arabic pdate.text.match(%r{(?<=Year: )(\d{2}.\w+.)?\d{4}}).to_s
|
212
234
|
end
|
213
235
|
|
214
236
|
# Convert roman month number in string date to arabic number
|
@@ -216,8 +238,11 @@ module RelatonItu
|
|
216
238
|
# @return [String]
|
217
239
|
def roman_to_arabic(date)
|
218
240
|
%r{(?<rmonth>[IVX]+)} =~ date
|
219
|
-
|
220
|
-
|
241
|
+
if ROMAN_MONTHS.index(rmonth)
|
242
|
+
month = ROMAN_MONTHS.index(rmonth) + 1
|
243
|
+
Date.parse(date.sub(%r{[IVX]+}, month.to_s)).to_s
|
244
|
+
else date
|
245
|
+
end
|
221
246
|
end
|
222
247
|
|
223
248
|
# Fetch contributors
|
@@ -244,7 +269,7 @@ module RelatonItu
|
|
244
269
|
links = [{ type: "src", content: url }]
|
245
270
|
obp_elm = doc.at(
|
246
271
|
'//a[@title="Persistent link to download the PDF file"]',
|
247
|
-
"//font[contains(.,'PDF')]/../.."
|
272
|
+
"//font[contains(.,'PDF')]/../.."
|
248
273
|
)
|
249
274
|
links << typed_link("obp", obp_elm) if obp_elm
|
250
275
|
wrd_elm = doc.at("//font[contains(.,'Word')]/../..")
|
data/lib/relaton_itu/version.rb
CHANGED
data/relaton-itu.gemspec
CHANGED
@@ -26,12 +26,12 @@ Gem::Specification.new do |spec|
|
|
26
26
|
spec.require_paths = ["lib"]
|
27
27
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
28
28
|
|
29
|
-
spec.add_development_dependency "debase"
|
29
|
+
# spec.add_development_dependency "debase"
|
30
30
|
spec.add_development_dependency "equivalent-xml", "~> 0.6"
|
31
31
|
spec.add_development_dependency "pry-byebug"
|
32
32
|
spec.add_development_dependency "rake", "~> 10.0"
|
33
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
34
|
-
spec.add_development_dependency "ruby-debug-ide"
|
34
|
+
# spec.add_development_dependency "ruby-debug-ide"
|
35
35
|
spec.add_development_dependency "ruby-jing"
|
36
36
|
spec.add_development_dependency "simplecov"
|
37
37
|
spec.add_development_dependency "vcr", "~> 5.0.0"
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: relaton-itu
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.7.
|
4
|
+
version: 1.7.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: debase
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: equivalent-xml
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,20 +66,6 @@ dependencies:
|
|
80
66
|
- - "~>"
|
81
67
|
- !ruby/object:Gem::Version
|
82
68
|
version: '3.0'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: ruby-debug-ide
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - ">="
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '0'
|
90
|
-
type: :development
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - ">="
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '0'
|
97
69
|
- !ruby/object:Gem::Dependency
|
98
70
|
name: ruby-jing
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -172,9 +144,7 @@ executables: []
|
|
172
144
|
extensions: []
|
173
145
|
extra_rdoc_files: []
|
174
146
|
files:
|
175
|
-
- ".github/workflows/
|
176
|
-
- ".github/workflows/ubuntu.yml"
|
177
|
-
- ".github/workflows/windows.yml"
|
147
|
+
- ".github/workflows/rake.yml"
|
178
148
|
- ".gitignore"
|
179
149
|
- ".rspec"
|
180
150
|
- ".rubocop.yml"
|
@@ -222,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
222
192
|
- !ruby/object:Gem::Version
|
223
193
|
version: '0'
|
224
194
|
requirements: []
|
225
|
-
rubygems_version: 3.
|
195
|
+
rubygems_version: 3.2.3
|
226
196
|
signing_key:
|
227
197
|
specification_version: 4
|
228
198
|
summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
|
data/.github/workflows/macos.yml
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
2
|
-
# See https://github.com/metanorma/cimas
|
3
|
-
name: macos
|
4
|
-
|
5
|
-
on:
|
6
|
-
push:
|
7
|
-
branches: [ master ]
|
8
|
-
pull_request:
|
9
|
-
branches: [ '**' ]
|
10
|
-
|
11
|
-
jobs:
|
12
|
-
test-macos:
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} macOS
|
14
|
-
runs-on: macos-latest
|
15
|
-
strategy:
|
16
|
-
fail-fast: false
|
17
|
-
matrix:
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
19
|
-
steps:
|
20
|
-
- uses: actions/checkout@master
|
21
|
-
- name: Use Ruby
|
22
|
-
uses: actions/setup-ruby@v1
|
23
|
-
with:
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
25
|
-
architecture: 'x64'
|
26
|
-
- name: Update gems
|
27
|
-
run: |
|
28
|
-
sudo gem install bundler --force
|
29
|
-
ruby -v | grep 2.5 && bundle config set build.debase --with-cflags="-Wno-error=implicit-function-declaration"
|
30
|
-
ruby -v | grep 2.5 && bundle config set build.ruby-debug-ide --with-cflags="-Wno-error=implicit-function-declaration"
|
31
|
-
bundle install --jobs 4 --retry 3
|
32
|
-
- name: Run specs
|
33
|
-
run: |
|
34
|
-
bundle exec rake
|
@@ -1,33 +0,0 @@
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
2
|
-
# See https://github.com/metanorma/cimas
|
3
|
-
name: ubuntu
|
4
|
-
|
5
|
-
on:
|
6
|
-
push:
|
7
|
-
branches: [ master ]
|
8
|
-
pull_request:
|
9
|
-
branches: [ '**' ]
|
10
|
-
|
11
|
-
jobs:
|
12
|
-
test-linux:
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Ubuntu
|
14
|
-
runs-on: ubuntu-latest
|
15
|
-
strategy:
|
16
|
-
fail-fast: false
|
17
|
-
matrix:
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
19
|
-
steps:
|
20
|
-
- uses: actions/checkout@master
|
21
|
-
- name: Use Ruby
|
22
|
-
uses: actions/setup-ruby@v1
|
23
|
-
with:
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
25
|
-
architecture: 'x64'
|
26
|
-
- name: Update gems
|
27
|
-
run: |
|
28
|
-
gem install bundler
|
29
|
-
bundle install --jobs 4 --retry 3
|
30
|
-
- name: Run specs
|
31
|
-
run: |
|
32
|
-
unset JAVA_TOOL_OPTIONS
|
33
|
-
bundle exec rake
|
@@ -1,35 +0,0 @@
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
2
|
-
# See https://github.com/metanorma/cimas
|
3
|
-
name: windows
|
4
|
-
|
5
|
-
on:
|
6
|
-
push:
|
7
|
-
branches: [ master ]
|
8
|
-
pull_request:
|
9
|
-
branches: [ '**' ]
|
10
|
-
|
11
|
-
jobs:
|
12
|
-
test-windows:
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Windows
|
14
|
-
runs-on: windows-latest
|
15
|
-
strategy:
|
16
|
-
fail-fast: false
|
17
|
-
matrix:
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
19
|
-
steps:
|
20
|
-
- uses: actions/checkout@master
|
21
|
-
- name: Use Ruby
|
22
|
-
uses: actions/setup-ruby@v1
|
23
|
-
with:
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
25
|
-
architecture: 'x64'
|
26
|
-
- name: Update gems
|
27
|
-
shell: pwsh
|
28
|
-
run: |
|
29
|
-
gem install bundler
|
30
|
-
bundle config --local path vendor/bundle
|
31
|
-
bundle update
|
32
|
-
bundle install --jobs 4 --retry 3
|
33
|
-
- name: Run specs
|
34
|
-
run: |
|
35
|
-
bundle exec rake
|