relaton-itu 1.7.2 → 1.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/rake.yml +46 -0
- data/README.adoc +24 -5
- data/bin/rspec +29 -0
- data/grammars/itu.rng +13 -0
- data/lib/relaton_itu/editorial_group.rb +1 -1
- data/lib/relaton_itu/hash_converter.rb +2 -2
- data/lib/relaton_itu/hit.rb +2 -0
- data/lib/relaton_itu/hit_collection.rb +28 -10
- data/lib/relaton_itu/itu_bibliographic_item.rb +8 -1
- data/lib/relaton_itu/itu_bibliography.rb +32 -17
- data/lib/relaton_itu/itu_group.rb +1 -1
- data/lib/relaton_itu/processor.rb +1 -2
- data/lib/relaton_itu/scrapper.rb +32 -21
- data/lib/relaton_itu/version.rb +1 -1
- data/lib/relaton_itu/xml_parser.rb +1 -1
- data/relaton-itu.gemspec +3 -3
- metadata +6 -35
- data/.github/workflows/macos.yml +0 -34
- data/.github/workflows/ubuntu.yml +0 -33
- data/.github/workflows/windows.yml +0 -35
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 5c9e267804ed407243d999f68830546b55db550bd511fa9146ab87e2db146c1d
|
|
4
|
+
data.tar.gz: c52daf864de7aeacd2651ca1a5442c0cadd8982e43ff671ae8a730a8a9559096
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e412c86f5a31146d27ae6b05feef03c817e2004581efde5e3051a66b37fe061c1e984e417ba92e94ca15032bab17c3e6f70a2062842922c4b300dfc4224cac61
|
|
7
|
+
data.tar.gz: e6718282c40d4119d1b7f33bc0f553c611b87d8f46abe57ddf7eb8569f781d22fa54ea135808d7b895d2b9b48b267eafac4d8d25a43df841b35d21d5041dcaea
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
+
# See https://github.com/metanorma/cimas
|
|
3
|
+
name: rake
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [ master, main ]
|
|
8
|
+
tags: [ v* ]
|
|
9
|
+
pull_request:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
rake:
|
|
13
|
+
name: Test on Ruby ${{ matrix.ruby }} ${{ matrix.os }}
|
|
14
|
+
runs-on: ${{ matrix.os }}
|
|
15
|
+
continue-on-error: ${{ matrix.experimental }}
|
|
16
|
+
strategy:
|
|
17
|
+
fail-fast: false
|
|
18
|
+
matrix:
|
|
19
|
+
ruby: [ '2.7', '2.6', '2.5', '2.4' ]
|
|
20
|
+
os: [ ubuntu-latest, windows-latest, macos-latest ]
|
|
21
|
+
experimental: [ false ]
|
|
22
|
+
include:
|
|
23
|
+
- ruby: '3.0'
|
|
24
|
+
os: 'ubuntu-latest'
|
|
25
|
+
experimental: true
|
|
26
|
+
- ruby: '3.0'
|
|
27
|
+
os: 'windows-latest'
|
|
28
|
+
experimental: true
|
|
29
|
+
- ruby: '3.0'
|
|
30
|
+
os: 'macos-latest'
|
|
31
|
+
experimental: true
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v2
|
|
34
|
+
with:
|
|
35
|
+
submodules: true
|
|
36
|
+
|
|
37
|
+
# https://github.com/ruby-debug/debase/issues/89#issuecomment-686827382
|
|
38
|
+
- if: matrix.os == 'macos-latest' && matrix.ruby == '2.5'
|
|
39
|
+
run: echo BUNDLE_BUILD__DEBASE="--with-cflags=\"-Wno-error=implicit-function-declaration\"" >> $GITHUB_ENV
|
|
40
|
+
|
|
41
|
+
- uses: ruby/setup-ruby@v1
|
|
42
|
+
with:
|
|
43
|
+
ruby-version: ${{ matrix.ruby }}
|
|
44
|
+
bundler-cache: true
|
|
45
|
+
|
|
46
|
+
- run: bundle exec rake
|
data/README.adoc
CHANGED
|
@@ -97,6 +97,29 @@ RelatonItu::ItuBibliography.get "ITU-T G.989.2/Amd 1"
|
|
|
97
97
|
...
|
|
98
98
|
----
|
|
99
99
|
|
|
100
|
+
=== Get ITU-R documents
|
|
101
|
+
|
|
102
|
+
[source,ruby]
|
|
103
|
+
----
|
|
104
|
+
RelatonItu::ItuBibliography.get "ITU-R REC-BO.600-1"
|
|
105
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") fetching...
|
|
106
|
+
[relaton-itu] ("ITU-R REC-BO.600-1") found R-REC-BO.600-1
|
|
107
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007f84e9930498
|
|
108
|
+
...
|
|
109
|
+
----
|
|
110
|
+
|
|
111
|
+
=== Get ITU-R Radio Regulations (RR)
|
|
112
|
+
|
|
113
|
+
[source,ruby]
|
|
114
|
+
----
|
|
115
|
+
RelatonItu::ItuBibliography.get 'ITU-R RR (2020)'
|
|
116
|
+
[relaton-itu] ("ITU-R RR") fetching...
|
|
117
|
+
[relaton-itu] WARNING: invalid doctype: publication
|
|
118
|
+
[relaton-itu] ("ITU-R RR") found ITU-R RR
|
|
119
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fb4b55eb660
|
|
120
|
+
...
|
|
121
|
+
----
|
|
122
|
+
|
|
100
123
|
=== Create bibliographic item form YAML
|
|
101
124
|
[source,ruby]
|
|
102
125
|
----
|
|
@@ -104,11 +127,7 @@ hash = YAML.load_file 'spec/examples/itu_bib_item.yml'
|
|
|
104
127
|
=> {"id"=>"ITU-T L.163 (11/2018)",
|
|
105
128
|
...
|
|
106
129
|
|
|
107
|
-
|
|
108
|
-
=> {:id=>"ITU-T L.163 (11/2018)",
|
|
109
|
-
...
|
|
110
|
-
|
|
111
|
-
RelatonItu::ItuBibliographicItem.new bib_hash
|
|
130
|
+
RelatonItu::ItuBibliographicItem.from_hash hash
|
|
112
131
|
=> #<RelatonItu::ItuBibliographicItem:0x007fd88ac02aa0
|
|
113
132
|
...
|
|
114
133
|
----
|
data/bin/rspec
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
#
|
|
5
|
+
# This file was generated by Bundler.
|
|
6
|
+
#
|
|
7
|
+
# The application 'rspec' is installed as part of a gem, and
|
|
8
|
+
# this file is here to facilitate running it.
|
|
9
|
+
#
|
|
10
|
+
|
|
11
|
+
require "pathname"
|
|
12
|
+
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
|
|
13
|
+
Pathname.new(__FILE__).realpath)
|
|
14
|
+
|
|
15
|
+
bundle_binstub = File.expand_path("../bundle", __FILE__)
|
|
16
|
+
|
|
17
|
+
if File.file?(bundle_binstub)
|
|
18
|
+
if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
|
|
19
|
+
load(bundle_binstub)
|
|
20
|
+
else
|
|
21
|
+
abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
|
|
22
|
+
Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
require "rubygems"
|
|
27
|
+
require "bundler/setup"
|
|
28
|
+
|
|
29
|
+
load Gem.bin_path("rspec-core", "rspec")
|
data/grammars/itu.rng
CHANGED
|
@@ -42,6 +42,8 @@
|
|
|
42
42
|
<value>joint-itu-iso-iec</value>
|
|
43
43
|
<value>resolution</value>
|
|
44
44
|
<value>service-publication</value>
|
|
45
|
+
<value>handbook</value>
|
|
46
|
+
<value>question</value>
|
|
45
47
|
</choice>
|
|
46
48
|
</define>
|
|
47
49
|
<define name="editorialgroup">
|
|
@@ -98,6 +100,9 @@
|
|
|
98
100
|
<optional>
|
|
99
101
|
<ref name="meeting"/>
|
|
100
102
|
</optional>
|
|
103
|
+
<optional>
|
|
104
|
+
<ref name="meeting-place"/>
|
|
105
|
+
</optional>
|
|
101
106
|
<optional>
|
|
102
107
|
<ref name="meeting-date"/>
|
|
103
108
|
</optional>
|
|
@@ -128,6 +133,14 @@
|
|
|
128
133
|
</include>
|
|
129
134
|
<define name="meeting">
|
|
130
135
|
<element name="meeting">
|
|
136
|
+
<optional>
|
|
137
|
+
<attribute name="acronym"/>
|
|
138
|
+
</optional>
|
|
139
|
+
<text/>
|
|
140
|
+
</element>
|
|
141
|
+
</define>
|
|
142
|
+
<define name="meeting-place">
|
|
143
|
+
<element name="meeting-place">
|
|
131
144
|
<text/>
|
|
132
145
|
</element>
|
|
133
146
|
</define>
|
|
@@ -20,7 +20,7 @@ module RelatonItu
|
|
|
20
20
|
warn "[relaton-itu] WARNING: invalid bureau: #{bureau}"
|
|
21
21
|
end
|
|
22
22
|
@bureau = bureau
|
|
23
|
-
@group = group.is_a?(Hash) ? ItuGroup.new(group) : group
|
|
23
|
+
@group = group.is_a?(Hash) ? ItuGroup.new(**group) : group
|
|
24
24
|
@subgroup = subgroup.is_a?(Hash) ? ItuGroup.new(subgroup) : subgroup
|
|
25
25
|
@workgroup = workgroup.is_a?(Hash) ? ItuGroup.new(workgroup) : workgroup
|
|
26
26
|
end
|
|
@@ -7,7 +7,7 @@ module RelatonItu
|
|
|
7
7
|
eg = ret[:editorialgroup]
|
|
8
8
|
return unless eg
|
|
9
9
|
|
|
10
|
-
ret[:editorialgroup] = EditorialGroup.new eg
|
|
10
|
+
ret[:editorialgroup] = EditorialGroup.new **eg
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
# @param ret [Hash]
|
|
@@ -15,7 +15,7 @@ module RelatonItu
|
|
|
15
15
|
return unless ret[:structuredidentifier]
|
|
16
16
|
|
|
17
17
|
ret[:structuredidentifier] = StructuredIdentifier.new(
|
|
18
|
-
ret[:structuredidentifier]
|
|
18
|
+
**ret[:structuredidentifier]
|
|
19
19
|
)
|
|
20
20
|
end
|
|
21
21
|
end
|
data/lib/relaton_itu/hit.rb
CHANGED
|
@@ -14,23 +14,41 @@ module RelatonItu
|
|
|
14
14
|
|
|
15
15
|
# @param ref [String]
|
|
16
16
|
# @param year [String]
|
|
17
|
-
def initialize(ref, year = nil)
|
|
17
|
+
def initialize(ref, year = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
|
18
18
|
text = ref.sub /(?<=\.)Imp\s?(?=\d)/, ""
|
|
19
19
|
super text, year
|
|
20
20
|
@gi_imp = /\.Imp\d/.match?(ref)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
if ref.match? /^(ITU-T|ITU-R\sRR)/
|
|
22
|
+
uri = URI "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
|
|
23
|
+
data = { json: params.to_json }
|
|
24
|
+
resp = Net::HTTP.post(uri, data.to_json,
|
|
25
|
+
"Content-Type" => "application/json")
|
|
26
|
+
@array = hits JSON.parse(resp.body)
|
|
27
|
+
elsif ref.match? /^ITU-R/
|
|
28
|
+
rf = ref.sub(/^ITU-R\s/, "").upcase
|
|
29
|
+
url = "https://raw.githubusercontent.com/relaton/relaton-data-itu-r/master/data/#{rf}.yaml"
|
|
30
|
+
resp = Net::HTTP.get_response(URI(url))
|
|
31
|
+
if resp.code == "404"
|
|
32
|
+
@array = []
|
|
33
|
+
return
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
hash = YAML.safe_load resp.body
|
|
37
|
+
item_hash = HashConverter.hash_to_bib(hash)
|
|
38
|
+
item = ItuBibliographicItem.new **item_hash
|
|
39
|
+
hit = Hit.new({ url: url }, self)
|
|
40
|
+
hit.fetch = item
|
|
41
|
+
@array = [hit]
|
|
42
|
+
end
|
|
26
43
|
end
|
|
27
44
|
|
|
28
45
|
private
|
|
29
46
|
|
|
30
47
|
# @return [String]
|
|
31
48
|
def group
|
|
32
|
-
@group ||=
|
|
33
|
-
|
|
49
|
+
@group ||= case text
|
|
50
|
+
when %r{OB|Operational Bulletin}, %r{^ITU-R\sRR} then "Publications"
|
|
51
|
+
when %r{^ITU-T} then "Recommendations"
|
|
34
52
|
end
|
|
35
53
|
end
|
|
36
54
|
|
|
@@ -46,7 +64,7 @@ module RelatonItu
|
|
|
46
64
|
"ExactPhrase" => false,
|
|
47
65
|
"CollectionName" => "General",
|
|
48
66
|
"CollectionGroup" => group,
|
|
49
|
-
"Sector" =>
|
|
67
|
+
"Sector" => text.match(/(?<=^ITU-)\w/).to_s.downcase,
|
|
50
68
|
"Criterias" => [{
|
|
51
69
|
"Name" => "Search in",
|
|
52
70
|
"Criterias" => [
|
|
@@ -93,7 +111,7 @@ module RelatonItu
|
|
|
93
111
|
code = h["Media"]["Name"]
|
|
94
112
|
title = h["Title"]
|
|
95
113
|
url = h["Redirection"]
|
|
96
|
-
type =
|
|
114
|
+
type = h["Collection"]["Group"].downcase[0...-1]
|
|
97
115
|
Hit.new({ code: code, title: title, url: url, type: type }, self)
|
|
98
116
|
end
|
|
99
117
|
end
|
|
@@ -4,7 +4,7 @@ module RelatonItu
|
|
|
4
4
|
recommendation recommendation-supplement recommendation-amendment
|
|
5
5
|
recommendation-corrigendum recommendation-errata recommendation-annex
|
|
6
6
|
focus-group implementers-guide technical-paper technical-report
|
|
7
|
-
joint-itu-iso-iec
|
|
7
|
+
joint-itu-iso-iec resolution service-publication handbook question
|
|
8
8
|
].freeze
|
|
9
9
|
|
|
10
10
|
# @params structuredidentifier [RelatonItu::StructuredIdentifier]
|
|
@@ -14,5 +14,12 @@ module RelatonItu
|
|
|
14
14
|
end
|
|
15
15
|
super
|
|
16
16
|
end
|
|
17
|
+
|
|
18
|
+
# @param hash [Hash]
|
|
19
|
+
# @return [RelatonItu::ItuBibliographicItem]
|
|
20
|
+
def self.from_hash(hash)
|
|
21
|
+
item_hash = ::RelatonItu::HashConverter.hash_to_bib(hash)
|
|
22
|
+
new **item_hash
|
|
23
|
+
end
|
|
17
24
|
end
|
|
18
25
|
end
|
|
@@ -19,6 +19,12 @@ module RelatonItu
|
|
|
19
19
|
# @param text [String]
|
|
20
20
|
# @return [RelatonItu::HitCollection]
|
|
21
21
|
def search(text, year = nil)
|
|
22
|
+
# code = text.sub(/(?<=ITU-T\s\w)\.(\w+\.)(?=\d+)/, ' \1')
|
|
23
|
+
if text =~ /(ITU-T\s\w)\.(Suppl\.|Annex)\s?(\w?\d+)/
|
|
24
|
+
correct_ref = "#{$~[1]} #{$~[2]} #{$~[3]}"
|
|
25
|
+
warn "[relaton-itu] WARNING: Incorrect reference #{text}"
|
|
26
|
+
warn "[relaton-itu] the reference should be #{correct_ref}"
|
|
27
|
+
end
|
|
22
28
|
HitCollection.new text, year
|
|
23
29
|
rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
|
|
24
30
|
EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
|
|
@@ -30,9 +36,9 @@ module RelatonItu
|
|
|
30
36
|
# @param year [String] the year the standard was published (optional)
|
|
31
37
|
# @param opts [Hash] options; restricted to :all_parts if all-parts reference is required
|
|
32
38
|
# @return [String] Relaton XML serialisation of reference
|
|
33
|
-
def get(code, year = nil, opts = {})
|
|
39
|
+
def get(code, year = nil, opts = {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
34
40
|
if year.nil?
|
|
35
|
-
/^(?<code1>[^\s]+\s[^\s]+)\s\(\d{2}\/(?<year1>\d+)\)$/ =~ code
|
|
41
|
+
/^(?<code1>[^\s]+\s[^\s]+)\s\((\d{2}\/)?(?<year1>\d+)\)$/ =~ code
|
|
36
42
|
unless code1.nil?
|
|
37
43
|
code = code1
|
|
38
44
|
year = year1
|
|
@@ -50,13 +56,15 @@ module RelatonItu
|
|
|
50
56
|
|
|
51
57
|
private
|
|
52
58
|
|
|
53
|
-
def fetch_ref_err(code, year, missed_years)
|
|
59
|
+
def fetch_ref_err(code, year, missed_years) # rubocop:disable Metrics/MethodLength
|
|
54
60
|
id = year ? "#{code}:#{year}" : code
|
|
55
61
|
warn "[relaton-itu] WARNING: no match found online for #{id}. "\
|
|
56
62
|
"The code must be exactly like it is on the standards website."
|
|
57
|
-
|
|
58
|
-
"
|
|
59
|
-
|
|
63
|
+
unless missed_years.empty?
|
|
64
|
+
warn "[relaton-itu] (There was no match for #{year}, though there were matches "\
|
|
65
|
+
"found for #{missed_years.join(', ')}.)"
|
|
66
|
+
end
|
|
67
|
+
if /\d-\d/.match? code
|
|
60
68
|
warn "[relaton-itu] The provided document part may not exist, or the document "\
|
|
61
69
|
"may no longer be published in parts."
|
|
62
70
|
else
|
|
@@ -67,12 +75,13 @@ module RelatonItu
|
|
|
67
75
|
nil
|
|
68
76
|
end
|
|
69
77
|
|
|
70
|
-
def search_filter(code, year)
|
|
78
|
+
def search_filter(code, year) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
|
|
71
79
|
%r{
|
|
72
|
-
^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+)
|
|
80
|
+
^(?<pref1>ITU)?(-(?<type1>\w))?\s?(?<code1>[^\s\/]+(?:\/\w[\.\d]+)?)
|
|
81
|
+
(\s\(?(?<ver1>v\d+)\)?)?
|
|
73
82
|
(\s\(((?<month1>\d{2})\/)?(?<year1>\d{4})\))?
|
|
74
83
|
(\s-\s(?<buldate1>\d{2}\.\w{1,4}\.\d{4}))?
|
|
75
|
-
(
|
|
84
|
+
(\s(?<corr1>(Amd|Cor|Amendment|Corrigendum)\.?\s?\d+))?
|
|
76
85
|
(\s\(((?<cormonth1>\d{2})\/)?(?<coryear1>\d{4})\))?
|
|
77
86
|
}x =~ code
|
|
78
87
|
year ||= year1
|
|
@@ -81,21 +90,27 @@ module RelatonItu
|
|
|
81
90
|
warn "[relaton-itu] (\"#{code}\") fetching..."
|
|
82
91
|
result = search(code)
|
|
83
92
|
code1.sub! /(?<=\.)Imp(?=\d)/, "" if result.gi_imp
|
|
93
|
+
if corr1
|
|
94
|
+
corr1.sub!(/[\.\s]+/, " ").sub!("Amendment", "Amd")
|
|
95
|
+
corr1.sub!("Corrigendum", "Corr")
|
|
96
|
+
end
|
|
84
97
|
result.select do |i|
|
|
98
|
+
next true unless i.hit[:code]
|
|
99
|
+
|
|
85
100
|
%r{
|
|
86
101
|
^(?<pref2>ITU)?(-(?<type2>\w))?\s?(?<code2>[\S]+)
|
|
102
|
+
(\s\(?(?<ver2>v\d+)\)?)?
|
|
87
103
|
(\s\(((?<month2>\d{2})\/)?(?<year2>\d{4})\))?
|
|
88
104
|
(\s(?<corr2>(Amd|Cor)\.\s?\d+))?
|
|
89
105
|
(\s\(((?<cormonth2>\d{2})\/)?(?<coryear2>\d{4})\))?
|
|
90
106
|
}x =~ i.hit[:code]
|
|
91
107
|
/:[^\(]+\((?<buldate2>\d{2}\.\w{1,4}\.\d{4})\)/ =~ i.hit[:title]
|
|
92
108
|
corr2&.sub! /\.\s?/, " "
|
|
93
|
-
pref1 == pref2 && (!type1 || type1 == type2) && code1
|
|
109
|
+
pref1 == pref2 && (!type1 || type1 == type2) && code2.include?(code1) &&
|
|
94
110
|
(!year || year == year2) && (!month1 || month1 == month2) &&
|
|
95
111
|
corr1 == corr2 && (!coryear1 || coryear1 == coryear2) &&
|
|
96
|
-
buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2)
|
|
97
|
-
|
|
98
|
-
# i.hit[:code].match(docidrx).to_s == c
|
|
112
|
+
buldate1 == buldate2 && (!cormonth1 || cormonth1 == cormonth2) &&
|
|
113
|
+
(!ver1 || ver1 == ver2)
|
|
99
114
|
end
|
|
100
115
|
end
|
|
101
116
|
|
|
@@ -108,10 +123,10 @@ module RelatonItu
|
|
|
108
123
|
def isobib_results_filter(result, year)
|
|
109
124
|
missed_years = []
|
|
110
125
|
result.each do |r|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
126
|
+
if !year || /\((\d{2}\/)?(?<pyear>\d{4})\)/ =~ r.hit[:code]
|
|
127
|
+
ret = r.fetch
|
|
128
|
+
return { ret: ret } if ret
|
|
129
|
+
end
|
|
115
130
|
|
|
116
131
|
missed_years << pyear
|
|
117
132
|
end
|
|
@@ -26,8 +26,7 @@ module RelatonItu
|
|
|
26
26
|
# @param hash [Hash]
|
|
27
27
|
# @return [RelatonItu::ItuBibliographicItem]
|
|
28
28
|
def hash_to_bib(hash)
|
|
29
|
-
|
|
30
|
-
::RelatonItu::ItuBibliographicItem.new item_hash
|
|
29
|
+
::RelatonItu::ItuBibliographicItem.from_hash hash
|
|
31
30
|
end
|
|
32
31
|
|
|
33
32
|
# Returns hash of XML grammar
|
data/lib/relaton_itu/scrapper.rb
CHANGED
|
@@ -31,6 +31,8 @@ module RelatonItu
|
|
|
31
31
|
# @return [Hash]
|
|
32
32
|
def parse_page(hit_data, imp = false)
|
|
33
33
|
url, doc = get_page hit_data[:url]
|
|
34
|
+
return unless doc
|
|
35
|
+
|
|
34
36
|
if imp
|
|
35
37
|
a = doc.at "//span[contains(@id, 'tab_ig_uc_rec')]/a"
|
|
36
38
|
return unless a
|
|
@@ -59,7 +61,7 @@ module RelatonItu
|
|
|
59
61
|
copyright: fetch_copyright(hit_data[:code], doc),
|
|
60
62
|
link: fetch_link(doc, url),
|
|
61
63
|
relation: fetch_relations(doc),
|
|
62
|
-
place: ["Geneva"]
|
|
64
|
+
place: ["Geneva"]
|
|
63
65
|
)
|
|
64
66
|
end
|
|
65
67
|
# rubocop:enable Metrics/AbcSize
|
|
@@ -69,17 +71,19 @@ module RelatonItu
|
|
|
69
71
|
# Fetch abstracts.
|
|
70
72
|
# @param doc [Nokigiri::HTML::Document]
|
|
71
73
|
# @return [Array<Array>]
|
|
72
|
-
def fetch_abstract(doc)
|
|
74
|
+
def fetch_abstract(doc) # rubocop:disable Metrics/AbcSize
|
|
73
75
|
abstract_url = doc.at('//table/tr/td/span[contains(@id, "lbl_dms")]/div')
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
content = if abstract_url
|
|
77
|
+
url = abstract_url[:onclick].match(/https?[^']+/).to_s
|
|
78
|
+
d = Nokogiri::HTML Net::HTTP.get(URI(url)).encode(undef: :replace, replace: "")
|
|
79
|
+
d.css("p.MsoNormal").text.gsub(/\r\n/, "").squeeze(" ").gsub(/\u00a0/, "")
|
|
80
|
+
elsif a = doc.at('//table/tr/td/span[contains(@class, "observation")]/text()')
|
|
81
|
+
a.text.strip
|
|
82
|
+
end
|
|
83
|
+
return [] unless content
|
|
80
84
|
|
|
81
85
|
[{
|
|
82
|
-
content:
|
|
86
|
+
content: content,
|
|
83
87
|
language: "en",
|
|
84
88
|
script: "Latn",
|
|
85
89
|
}]
|
|
@@ -92,7 +96,9 @@ module RelatonItu
|
|
|
92
96
|
uri = URI url
|
|
93
97
|
resp = Net::HTTP.get_response(uri)
|
|
94
98
|
until resp.code == "200"
|
|
95
|
-
|
|
99
|
+
return if resp["location"] == "/en/publications/pages/notfound.aspx"
|
|
100
|
+
|
|
101
|
+
uri = URI resp["location"] if resp.code.match? /^30/
|
|
96
102
|
resp = Net::HTTP.get_response(uri)
|
|
97
103
|
end
|
|
98
104
|
[uri.to_s, Nokogiri::HTML(resp.body)]
|
|
@@ -117,16 +123,16 @@ module RelatonItu
|
|
|
117
123
|
docids
|
|
118
124
|
end
|
|
119
125
|
|
|
120
|
-
def createdocid(text)
|
|
126
|
+
def createdocid(text) # rubocop:disable Metrics/MethodLength
|
|
121
127
|
%r{
|
|
122
128
|
^(?<code>((ITU-\w|ISO\/IEC)\s)?[^\(:]+)
|
|
123
|
-
(\(((?<
|
|
129
|
+
(\(((?<_month>\d{2})\/)?(?<_year>\d{4})\))?
|
|
124
130
|
(:[^\(]+\((?<buldate>\d{2}\.\w{1,4}\.\d{4})\))?
|
|
125
131
|
(\s(?<corr>(Amd|Cor)\.\s?\d+))?
|
|
126
|
-
# (\s\(((?<
|
|
132
|
+
# (\s\(((?<_cormonth>\d{2})\/)?(?<_coryear>\d{4})\))?
|
|
127
133
|
}x =~ text.squeeze(" ")
|
|
128
134
|
corr&.sub! /\.\s?/, " "
|
|
129
|
-
id = [code.sub(/[[:space:]]$/, ""), corr].compact.join "
|
|
135
|
+
id = [code.sub(/[[:space:]]$/, ""), corr].compact.join " "
|
|
130
136
|
id += " - #{buldate}" if buldate
|
|
131
137
|
type = id.match(%r{^\w+}).to_s
|
|
132
138
|
type = "ITU" if type == "G"
|
|
@@ -156,13 +162,13 @@ module RelatonItu
|
|
|
156
162
|
group = wg && itugroup(wg.text)
|
|
157
163
|
EditorialGroup.new(
|
|
158
164
|
bureau: code.match(/(?<=-)./).to_s,
|
|
159
|
-
group: group
|
|
165
|
+
group: group
|
|
160
166
|
)
|
|
161
167
|
end
|
|
162
168
|
|
|
163
169
|
# @param name [String]
|
|
164
170
|
# @return [RelatonItu::ItuGroup]
|
|
165
|
-
def itugroup(name)
|
|
171
|
+
def itugroup(name) # rubocop:disable Metrics/MethodLength
|
|
166
172
|
if name.include? "Study Group"
|
|
167
173
|
type = "study-group"
|
|
168
174
|
acronym = "SG"
|
|
@@ -204,13 +210,15 @@ module RelatonItu
|
|
|
204
210
|
# Fetch dates
|
|
205
211
|
# @param doc [Nokogiri::HTML::Document]
|
|
206
212
|
# @return [Array<Hash>]
|
|
207
|
-
def fetch_dates(doc)
|
|
213
|
+
def fetch_dates(doc) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
|
|
208
214
|
dates = []
|
|
209
215
|
date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]",
|
|
210
216
|
"//p[contains(.,'Approved in')]")
|
|
211
217
|
pdate = date&.text&.match(/\d{4}-\d{2}-\d{2}/).to_s || ob_date(doc)
|
|
212
218
|
if pdate && !pdate&.empty?
|
|
213
219
|
dates << { type: "published", on: pdate }
|
|
220
|
+
elsif pdate = ob_date(doc)
|
|
221
|
+
dates << { type: "published", on: pdate }
|
|
214
222
|
end
|
|
215
223
|
dates
|
|
216
224
|
end
|
|
@@ -222,7 +230,7 @@ module RelatonItu
|
|
|
222
230
|
pdate = doc.at('//table/tbody/tr/td[contains(text(), "Year:")]')
|
|
223
231
|
return unless pdate
|
|
224
232
|
|
|
225
|
-
roman_to_arabic pdate.text.match(%r{(?<=Year: )\d{2}.\w
|
|
233
|
+
roman_to_arabic pdate.text.match(%r{(?<=Year: )(\d{2}.\w+.)?\d{4}}).to_s
|
|
226
234
|
end
|
|
227
235
|
|
|
228
236
|
# Convert roman month number in string date to arabic number
|
|
@@ -230,8 +238,11 @@ module RelatonItu
|
|
|
230
238
|
# @return [String]
|
|
231
239
|
def roman_to_arabic(date)
|
|
232
240
|
%r{(?<rmonth>[IVX]+)} =~ date
|
|
233
|
-
|
|
234
|
-
|
|
241
|
+
if ROMAN_MONTHS.index(rmonth)
|
|
242
|
+
month = ROMAN_MONTHS.index(rmonth) + 1
|
|
243
|
+
Date.parse(date.sub(%r{[IVX]+}, month.to_s)).to_s
|
|
244
|
+
else date
|
|
245
|
+
end
|
|
235
246
|
end
|
|
236
247
|
|
|
237
248
|
# Fetch contributors
|
|
@@ -258,7 +269,7 @@ module RelatonItu
|
|
|
258
269
|
links = [{ type: "src", content: url }]
|
|
259
270
|
obp_elm = doc.at(
|
|
260
271
|
'//a[@title="Persistent link to download the PDF file"]',
|
|
261
|
-
"//font[contains(.,'PDF')]/../.."
|
|
272
|
+
"//font[contains(.,'PDF')]/../.."
|
|
262
273
|
)
|
|
263
274
|
links << typed_link("obp", obp_elm) if obp_elm
|
|
264
275
|
wrd_elm = doc.at("//font[contains(.,'Word')]/../..")
|
data/lib/relaton_itu/version.rb
CHANGED
data/relaton-itu.gemspec
CHANGED
|
@@ -26,12 +26,12 @@ Gem::Specification.new do |spec|
|
|
|
26
26
|
spec.require_paths = ["lib"]
|
|
27
27
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
|
28
28
|
|
|
29
|
-
spec.add_development_dependency "debase"
|
|
29
|
+
# spec.add_development_dependency "debase"
|
|
30
30
|
spec.add_development_dependency "equivalent-xml", "~> 0.6"
|
|
31
31
|
spec.add_development_dependency "pry-byebug"
|
|
32
|
-
spec.add_development_dependency "rake", "~>
|
|
32
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
33
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
|
34
|
-
spec.add_development_dependency "ruby-debug-ide"
|
|
34
|
+
# spec.add_development_dependency "ruby-debug-ide"
|
|
35
35
|
spec.add_development_dependency "ruby-jing"
|
|
36
36
|
spec.add_development_dependency "simplecov"
|
|
37
37
|
spec.add_development_dependency "vcr", "~> 5.0.0"
|
metadata
CHANGED
|
@@ -1,29 +1,15 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton-itu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.7.
|
|
4
|
+
version: 1.7.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2021-04-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
|
-
- !ruby/object:Gem::Dependency
|
|
14
|
-
name: debase
|
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
|
16
|
-
requirements:
|
|
17
|
-
- - ">="
|
|
18
|
-
- !ruby/object:Gem::Version
|
|
19
|
-
version: '0'
|
|
20
|
-
type: :development
|
|
21
|
-
prerelease: false
|
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
-
requirements:
|
|
24
|
-
- - ">="
|
|
25
|
-
- !ruby/object:Gem::Version
|
|
26
|
-
version: '0'
|
|
27
13
|
- !ruby/object:Gem::Dependency
|
|
28
14
|
name: equivalent-xml
|
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -58,14 +44,14 @@ dependencies:
|
|
|
58
44
|
requirements:
|
|
59
45
|
- - "~>"
|
|
60
46
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '
|
|
47
|
+
version: '13.0'
|
|
62
48
|
type: :development
|
|
63
49
|
prerelease: false
|
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
51
|
requirements:
|
|
66
52
|
- - "~>"
|
|
67
53
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '
|
|
54
|
+
version: '13.0'
|
|
69
55
|
- !ruby/object:Gem::Dependency
|
|
70
56
|
name: rspec
|
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -80,20 +66,6 @@ dependencies:
|
|
|
80
66
|
- - "~>"
|
|
81
67
|
- !ruby/object:Gem::Version
|
|
82
68
|
version: '3.0'
|
|
83
|
-
- !ruby/object:Gem::Dependency
|
|
84
|
-
name: ruby-debug-ide
|
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
|
86
|
-
requirements:
|
|
87
|
-
- - ">="
|
|
88
|
-
- !ruby/object:Gem::Version
|
|
89
|
-
version: '0'
|
|
90
|
-
type: :development
|
|
91
|
-
prerelease: false
|
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
-
requirements:
|
|
94
|
-
- - ">="
|
|
95
|
-
- !ruby/object:Gem::Version
|
|
96
|
-
version: '0'
|
|
97
69
|
- !ruby/object:Gem::Dependency
|
|
98
70
|
name: ruby-jing
|
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -172,9 +144,7 @@ executables: []
|
|
|
172
144
|
extensions: []
|
|
173
145
|
extra_rdoc_files: []
|
|
174
146
|
files:
|
|
175
|
-
- ".github/workflows/
|
|
176
|
-
- ".github/workflows/ubuntu.yml"
|
|
177
|
-
- ".github/workflows/windows.yml"
|
|
147
|
+
- ".github/workflows/rake.yml"
|
|
178
148
|
- ".gitignore"
|
|
179
149
|
- ".rspec"
|
|
180
150
|
- ".rubocop.yml"
|
|
@@ -183,6 +153,7 @@ files:
|
|
|
183
153
|
- README.adoc
|
|
184
154
|
- Rakefile
|
|
185
155
|
- bin/console
|
|
156
|
+
- bin/rspec
|
|
186
157
|
- bin/setup
|
|
187
158
|
- grammars/basicdoc.rng
|
|
188
159
|
- grammars/biblio.rng
|
data/.github/workflows/macos.yml
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: macos
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-macos:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} macOS
|
|
14
|
-
runs-on: macos-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
run: |
|
|
28
|
-
sudo gem install bundler --force
|
|
29
|
-
ruby -v | grep 2.5 && bundle config set build.debase --with-cflags="-Wno-error=implicit-function-declaration"
|
|
30
|
-
ruby -v | grep 2.5 && bundle config set build.ruby-debug-ide --with-cflags="-Wno-error=implicit-function-declaration"
|
|
31
|
-
bundle install --jobs 4 --retry 3
|
|
32
|
-
- name: Run specs
|
|
33
|
-
run: |
|
|
34
|
-
bundle exec rake
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: ubuntu
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-linux:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Ubuntu
|
|
14
|
-
runs-on: ubuntu-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
run: |
|
|
28
|
-
gem install bundler
|
|
29
|
-
bundle install --jobs 4 --retry 3
|
|
30
|
-
- name: Run specs
|
|
31
|
-
run: |
|
|
32
|
-
unset JAVA_TOOL_OPTIONS
|
|
33
|
-
bundle exec rake
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
# Auto-generated by Cimas: Do not edit it manually!
|
|
2
|
-
# See https://github.com/metanorma/cimas
|
|
3
|
-
name: windows
|
|
4
|
-
|
|
5
|
-
on:
|
|
6
|
-
push:
|
|
7
|
-
branches: [ master ]
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ '**' ]
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
test-windows:
|
|
13
|
-
name: Test on Ruby ${{ matrix.ruby }} Windows
|
|
14
|
-
runs-on: windows-latest
|
|
15
|
-
strategy:
|
|
16
|
-
fail-fast: false
|
|
17
|
-
matrix:
|
|
18
|
-
ruby: [ '2.6', '2.5', '2.4' ]
|
|
19
|
-
steps:
|
|
20
|
-
- uses: actions/checkout@master
|
|
21
|
-
- name: Use Ruby
|
|
22
|
-
uses: actions/setup-ruby@v1
|
|
23
|
-
with:
|
|
24
|
-
ruby-version: ${{ matrix.ruby }}
|
|
25
|
-
architecture: 'x64'
|
|
26
|
-
- name: Update gems
|
|
27
|
-
shell: pwsh
|
|
28
|
-
run: |
|
|
29
|
-
gem install bundler
|
|
30
|
-
bundle config --local path vendor/bundle
|
|
31
|
-
bundle update
|
|
32
|
-
bundle install --jobs 4 --retry 3
|
|
33
|
-
- name: Run specs
|
|
34
|
-
run: |
|
|
35
|
-
bundle exec rake
|