relaton-itu 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +3 -1
- data/Gemfile +0 -3
- data/Gemfile.lock +15 -17
- data/README.adoc +98 -18
- data/appveyor.yml +8 -1
- data/itubib.gemspec +5 -4
- data/lib/relaton/processor.rb +1 -1
- data/lib/relaton_itu/editorial_group.rb +37 -0
- data/lib/relaton_itu/hit_collection.rb +56 -36
- data/lib/relaton_itu/itu_bibliographic_item.rb +27 -0
- data/lib/relaton_itu/itu_bibliography.rb +18 -11
- data/lib/relaton_itu/itu_group.rb +60 -0
- data/lib/relaton_itu/scrapper.rb +104 -90
- data/lib/relaton_itu/version.rb +1 -1
- data/lib/relaton_itu/xml_parser.rb +55 -0
- data/lib/relaton_itu.rb +2 -2
- metadata +33 -30
- data/lib/relaton_itu/workers_pool.rb +0 -41
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: cb8e6a79f5332e451c0aa9d243a702c3b06d1699b2230b17ae15ee89e03670e9
|
|
4
|
+
data.tar.gz: 97fedc98d4f4737545529acc6ef6d42bfa6491f783d69a1ac5c7a068e37c701e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a139715a49f88d302d7625134864bf916186f7a61aa4cf9498a614e3a917cc3b168dacb948c6d08c4ad297498d03587d6286b6e9dc5fab65745821c434ae882a
|
|
7
|
+
data.tar.gz: a645b6d4286de51c636eeeab9508f76e075e0c8082b026bdd923470d02181977f0053bfe8d954b2c645bfd9e60b16848c21b1b7727074a6cb9a1de66611f465b
|
data/.travis.yml
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# Auto-generated !!! Do not edit it manually
|
|
2
|
+
# use ci-master https://github.com/metanorma/metanorma-build-scripts
|
|
1
3
|
language: ruby
|
|
2
4
|
cache: bundler
|
|
3
5
|
os:
|
|
@@ -7,7 +9,7 @@ rvm:
|
|
|
7
9
|
- 2.5
|
|
8
10
|
- 2.4
|
|
9
11
|
- ruby-head
|
|
10
|
-
before_install:
|
|
12
|
+
before_install:
|
|
11
13
|
- gem install bundler -v 2.0.1
|
|
12
14
|
- bundle update
|
|
13
15
|
matrix:
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,15 +1,8 @@
|
|
|
1
|
-
GIT
|
|
2
|
-
remote: https://github.com/vcr/vcr.git
|
|
3
|
-
revision: a0bb140184e8518baad640a705e04155911bf31f
|
|
4
|
-
ref: a0bb140184e8518baad640a705e04155911bf31f
|
|
5
|
-
specs:
|
|
6
|
-
vcr (4.0.0)
|
|
7
|
-
|
|
8
1
|
PATH
|
|
9
2
|
remote: .
|
|
10
3
|
specs:
|
|
11
|
-
relaton-itu (0.
|
|
12
|
-
iso-bib
|
|
4
|
+
relaton-itu (0.2.0)
|
|
5
|
+
relaton-iso-bib (~> 0.2.0)
|
|
13
6
|
|
|
14
7
|
GEM
|
|
15
8
|
remote: https://rubygems.org/
|
|
@@ -27,11 +20,7 @@ GEM
|
|
|
27
20
|
docile (1.3.1)
|
|
28
21
|
equivalent-xml (0.6.0)
|
|
29
22
|
nokogiri (>= 1.4.3)
|
|
30
|
-
hashdiff (0.
|
|
31
|
-
iso-bib-item (0.4.5)
|
|
32
|
-
isoics (~> 0.1.6)
|
|
33
|
-
nokogiri (~> 1.8.4)
|
|
34
|
-
ruby_deep_clone (~> 0.8.0)
|
|
23
|
+
hashdiff (0.4.0)
|
|
35
24
|
isoics (0.1.7)
|
|
36
25
|
json (2.2.0)
|
|
37
26
|
method_source (0.9.2)
|
|
@@ -44,15 +33,23 @@ GEM
|
|
|
44
33
|
pry-byebug (3.7.0)
|
|
45
34
|
byebug (~> 11.0)
|
|
46
35
|
pry (~> 0.10)
|
|
47
|
-
public_suffix (3.0
|
|
36
|
+
public_suffix (3.1.0)
|
|
48
37
|
rake (10.5.0)
|
|
38
|
+
relaton-bib (0.2.0)
|
|
39
|
+
addressable
|
|
40
|
+
nokogiri (~> 1.8.4)
|
|
41
|
+
relaton-iso-bib (0.2.0)
|
|
42
|
+
isoics (~> 0.1.6)
|
|
43
|
+
nokogiri (~> 1.8.4)
|
|
44
|
+
relaton-bib (~> 0.2.0)
|
|
45
|
+
ruby_deep_clone (~> 0.8.0)
|
|
49
46
|
rspec (3.8.0)
|
|
50
47
|
rspec-core (~> 3.8.0)
|
|
51
48
|
rspec-expectations (~> 3.8.0)
|
|
52
49
|
rspec-mocks (~> 3.8.0)
|
|
53
50
|
rspec-core (3.8.0)
|
|
54
51
|
rspec-support (~> 3.8.0)
|
|
55
|
-
rspec-expectations (3.8.
|
|
52
|
+
rspec-expectations (3.8.3)
|
|
56
53
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
57
54
|
rspec-support (~> 3.8.0)
|
|
58
55
|
rspec-mocks (3.8.0)
|
|
@@ -68,6 +65,7 @@ GEM
|
|
|
68
65
|
json (>= 1.8, < 3)
|
|
69
66
|
simplecov-html (~> 0.10.0)
|
|
70
67
|
simplecov-html (0.10.2)
|
|
68
|
+
vcr (5.0.0)
|
|
71
69
|
webmock (3.5.1)
|
|
72
70
|
addressable (>= 2.3.6)
|
|
73
71
|
crack (>= 0.3.2)
|
|
@@ -86,7 +84,7 @@ DEPENDENCIES
|
|
|
86
84
|
rspec (~> 3.0)
|
|
87
85
|
ruby-debug-ide
|
|
88
86
|
simplecov
|
|
89
|
-
vcr
|
|
87
|
+
vcr (~> 5.0.0)
|
|
90
88
|
webmock
|
|
91
89
|
|
|
92
90
|
BUNDLED WITH
|
data/README.adoc
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
= RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem model
|
|
2
2
|
|
|
3
|
-
image:https://img.shields.io/gem/v/
|
|
4
|
-
image:https://travis-ci.com/metanorma/
|
|
5
|
-
image:https://ci.appveyor.com/api/projects/status/fd39m2762jo8ve04?svg=true["Appveyor Build Status", link="https://ci.appveyor.com/project/ribose/
|
|
3
|
+
image:https://img.shields.io/gem/v/relaton-bib.svg["Gem Version", link="https://rubygems.org/gems/relaton-bib"]
|
|
4
|
+
image:https://travis-ci.com/metanorma/relaton-bib.svg?branch=master["Build Status", link="https://travis-ci.com/metanorma/relaton-bib"]
|
|
5
|
+
image:https://ci.appveyor.com/api/projects/status/fd39m2762jo8ve04?svg=true["Appveyor Build Status", link="https://ci.appveyor.com/project/ribose/relaton-bib"]
|
|
6
6
|
|
|
7
7
|
RelatonItu is a Ruby gem that implements the https://github.com/metanorma/metanorma-model-iso#iso-bibliographic-item[IsoBibliographicItem model].
|
|
8
8
|
|
|
@@ -12,7 +12,7 @@ Add this line to your application's Gemfile:
|
|
|
12
12
|
|
|
13
13
|
[source,ruby]
|
|
14
14
|
----
|
|
15
|
-
gem '
|
|
15
|
+
gem 'relaton-bib'
|
|
16
16
|
----
|
|
17
17
|
|
|
18
18
|
And then execute:
|
|
@@ -21,7 +21,7 @@ And then execute:
|
|
|
21
21
|
|
|
22
22
|
Or install it yourself as:
|
|
23
23
|
|
|
24
|
-
$ gem install
|
|
24
|
+
$ gem install relaton-bib
|
|
25
25
|
|
|
26
26
|
== Usage
|
|
27
27
|
|
|
@@ -32,12 +32,19 @@ Or install it yourself as:
|
|
|
32
32
|
require 'relaton_bib'
|
|
33
33
|
|
|
34
34
|
hit_collection = RelatonItu::ItuBibliography.search("ITU-T L.163")
|
|
35
|
-
=> [<RelatonItu::Hit:
|
|
36
|
-
<RelatonItu::Hit:
|
|
37
|
-
|
|
35
|
+
=> [<RelatonItu::Hit:0x007fe50780a270 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T L.163 (11/2018)">,
|
|
36
|
+
<RelatonItu::Hit:0x007fe50780a220 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T J.163 (12/2007)">,
|
|
37
|
+
<RelatonItu::Hit:0x007fe50780a1d0 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T F.163 (08/1992)">,
|
|
38
|
+
<RelatonItu::Hit:0x007fe50780a180 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T J.163 (11/2005)">,
|
|
39
|
+
<RelatonItu::Hit:0x007fe50780a130 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T J.163 (03/2004)">,
|
|
40
|
+
<RelatonItu::Hit:0x007fe50780a0e0 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T Q.163 (11/1988)">,
|
|
41
|
+
<RelatonItu::Hit:0x007fe50780a090 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T J.163 (03/2001)">,
|
|
42
|
+
<RelatonItu::Hit:0x007fe50780a040 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T F.163 (07/1996)">,
|
|
43
|
+
<RelatonItu::Hit:0x007fe507809ff0 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T X.163 (04/1995)">,
|
|
44
|
+
<RelatonItu::Hit:0x007fe507809fa0 @text="ITU-T L.163" @fetched="false" @fullIdentifier="" @title="ITU-T O.163 (11/1988)">]
|
|
38
45
|
|
|
39
46
|
item = hit_collection[1].fetch
|
|
40
|
-
=> #<
|
|
47
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fe5024f3028
|
|
41
48
|
...
|
|
42
49
|
----
|
|
43
50
|
|
|
@@ -45,13 +52,86 @@ hit_collection = RelatonItu::ItuBibliography.search("ITU-T L.163")
|
|
|
45
52
|
[source,ruby]
|
|
46
53
|
----
|
|
47
54
|
item.to_xml
|
|
48
|
-
=> "<bibitem
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
+
=> "<bibitem id="ITU-TJ.163(12/2007)">
|
|
56
|
+
<fetched>2019-06-07</fetched>
|
|
57
|
+
<title type="title-main" format="text/plain" language="en" script="Latn">ITU-T J.163 (12/2007): Dynamic quality of service for the provision of real-time services over cable television networks using cable modems</title>
|
|
58
|
+
<title type="main" format="text/plain" language="en" script="Latn">ITU-T J.163 (12/2007): Dynamic quality of service for the provision of real-time services over cable television networks using cable modems</title>
|
|
59
|
+
<uri type="src">https://www.itu.int/net4/ITU-T/search/api/redirection?dest=http%3A%2F%2Fhandle.itu.int%2F11.1002%2F1000%2F9306-en&position=9&page=1</uri>
|
|
60
|
+
<uri type="obp">https://www.itu.int/rec/T-REC-J.163-200712-I</uri>
|
|
61
|
+
<docidentifier type="ITU">ITU-T J.163 (12/2007)</docidentifier>
|
|
62
|
+
<date type="published">
|
|
63
|
+
<on>2007</on>
|
|
64
|
+
</date>
|
|
65
|
+
<contributor>
|
|
66
|
+
<role type="publisher"/>
|
|
67
|
+
<organization>
|
|
68
|
+
<name>International Telecommunication Union</name>
|
|
69
|
+
<abbreviation>ITU</abbreviation>
|
|
70
|
+
<uri>www.itu.int</uri>
|
|
71
|
+
</organization>
|
|
72
|
+
</contributor>
|
|
73
|
+
<edition>4</edition>
|
|
74
|
+
<language>en</language>
|
|
75
|
+
<script>Latn</script>
|
|
76
|
+
<abstract format="text/plain" language="en" script="Latn">ITU-T Recommendation J.163 addresses requirements for a client device to obtain access to network resources. In particular, it specifies a comprehensive mechanism for a client device to request a specific quality of service from the DOCSIS network. Extensive examples illustrate the use of this Recommendation. The scope of this Recommendation is to define the QoS architecture for the "Access" portion of the IPCablecom network, provided to requesting applications on a per-flow basis. The access portion of the network is defined to be between the multimedia terminal adapter (MTA) and the cable modem termination system (CMTS), including the DOCSIS network. The method of QoS allocation over the backbone is unspecified in this Recommendation. Interface to the managed IP backbone and issues related to IP multicast are not within the scope of this Recommendation. This Recommendation also recognizes that per-flow reservations may be required within the customer premises, and the protocol developed addresses this potential need.</abstract>
|
|
77
|
+
<status>
|
|
78
|
+
<stage>Published</stage>
|
|
79
|
+
</status>
|
|
80
|
+
<copyright>
|
|
81
|
+
<from>2007</from>
|
|
82
|
+
<owner>
|
|
83
|
+
<organization>
|
|
84
|
+
<name>International Telecommunication Union</name>
|
|
85
|
+
<abbreviation>ITU</abbreviation>
|
|
86
|
+
<uri>www.itu.int</uri>
|
|
87
|
+
</organization>
|
|
88
|
+
</owner>
|
|
89
|
+
</copyright>
|
|
90
|
+
<relation type="published">
|
|
91
|
+
<bibitem id="JSuppl.1(11/1998)">
|
|
92
|
+
<formattedref format="text/plain" language="en" script="Latn">J Suppl. 1 (11/1998)</formattedref>
|
|
93
|
+
</bibitem>
|
|
94
|
+
</relation>
|
|
95
|
+
<relation type="published">
|
|
96
|
+
<bibitem id="JSuppl.2(11/1998)">
|
|
97
|
+
<formattedref format="text/plain" language="en" script="Latn">J Suppl. 2 (11/1998)</formattedref>
|
|
98
|
+
</bibitem>
|
|
99
|
+
</relation>
|
|
100
|
+
<relation type="published">
|
|
101
|
+
<bibitem id="JSuppl.3(11/1998)">
|
|
102
|
+
<formattedref format="text/plain" language="en" script="Latn">J Suppl. 3 (11/1998)</formattedref>
|
|
103
|
+
</bibitem>
|
|
104
|
+
</relation>
|
|
105
|
+
<relation type="published">
|
|
106
|
+
<bibitem id="JSuppl.5(09/1999)">
|
|
107
|
+
<formattedref format="text/plain" language="en" script="Latn">J Suppl. 5 (09/1999)</formattedref>
|
|
108
|
+
</bibitem>
|
|
109
|
+
</relation>
|
|
110
|
+
<relation type="published">
|
|
111
|
+
<bibitem id="JSuppl.6(09/2016)">
|
|
112
|
+
<formattedref format="text/plain" language="en" script="Latn">J Suppl. 6 (09/2016)</formattedref>
|
|
113
|
+
</bibitem>
|
|
114
|
+
</relation>
|
|
115
|
+
</bibitem>"
|
|
116
|
+
----
|
|
117
|
+
With `bibdata: true` option XML output wrapped with `bibdata` element and `ext` element added.
|
|
118
|
+
[source,ruby]
|
|
119
|
+
----
|
|
120
|
+
item.to_xml bibdata: true
|
|
121
|
+
=> "<bibdata>
|
|
122
|
+
<fetched>2019-06-07</fetched>
|
|
123
|
+
...
|
|
124
|
+
<ext>
|
|
125
|
+
<doctype>recommendation</doctype>
|
|
126
|
+
<editorialgroup>
|
|
127
|
+
<bureau>T</bureau>
|
|
128
|
+
<group type="study-group">
|
|
129
|
+
<name>ITU-T Study Group 9</name>
|
|
130
|
+
<acronym>SG</acronym>
|
|
131
|
+
</group>
|
|
132
|
+
</editorialgroup>
|
|
133
|
+
</ext>
|
|
134
|
+
</bibdata>"
|
|
55
135
|
----
|
|
56
136
|
|
|
57
137
|
=== Get code, and year
|
|
@@ -59,7 +139,7 @@ item.to_xml
|
|
|
59
139
|
----
|
|
60
140
|
RelatonItu::ItuBibliography.get("ITU-T L.163", "2018", {})
|
|
61
141
|
fetching ITU-T L.163...
|
|
62
|
-
=> #<
|
|
142
|
+
=> #<RelatonItu::ItuBibliographicItem:0x007fe502509dc8
|
|
63
143
|
...
|
|
64
144
|
----
|
|
65
145
|
|
|
@@ -71,7 +151,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
|
71
151
|
|
|
72
152
|
== Contributing
|
|
73
153
|
|
|
74
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/
|
|
154
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/metanorma/relaton-bib.
|
|
75
155
|
|
|
76
156
|
== License
|
|
77
157
|
|
data/appveyor.yml
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
|
+
# Auto-generated !!! Do not edit it manually
|
|
2
|
+
# use ci-master https://github.com/metanorma/metanorma-build-scripts
|
|
1
3
|
version: '{build}'
|
|
2
4
|
|
|
5
|
+
cache:
|
|
6
|
+
- vendor/bundle
|
|
7
|
+
|
|
3
8
|
environment:
|
|
4
9
|
matrix:
|
|
5
10
|
- RUBY_VERSION: 25
|
|
@@ -16,6 +21,8 @@ install:
|
|
|
16
21
|
|
|
17
22
|
build_script:
|
|
18
23
|
- set PATH=C:\Ruby%RUBY_VERSION%\bin;%PATH%
|
|
24
|
+
- set GIT_TERMINAL_PROMPT=0
|
|
25
|
+
- bundle config --local path vendor/bundle
|
|
19
26
|
- bundle update
|
|
20
27
|
- bundle install
|
|
21
28
|
|
|
@@ -25,4 +32,4 @@ before_test:
|
|
|
25
32
|
- bundle -v
|
|
26
33
|
|
|
27
34
|
test_script:
|
|
28
|
-
- bundle exec rake
|
|
35
|
+
- bundle exec rake
|
data/itubib.gemspec
CHANGED
|
@@ -24,17 +24,18 @@ Gem::Specification.new do |spec|
|
|
|
24
24
|
spec.bindir = "exe"
|
|
25
25
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
26
26
|
spec.require_paths = ["lib"]
|
|
27
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
|
|
27
28
|
|
|
28
29
|
spec.add_development_dependency "bundler", "~> 2.0"
|
|
30
|
+
spec.add_development_dependency "debase"
|
|
29
31
|
spec.add_development_dependency "equivalent-xml", "~> 0.6"
|
|
30
32
|
spec.add_development_dependency "pry-byebug"
|
|
31
33
|
spec.add_development_dependency "rake", "~> 10.0"
|
|
32
34
|
spec.add_development_dependency "rspec", "~> 3.0"
|
|
35
|
+
spec.add_development_dependency "ruby-debug-ide"
|
|
33
36
|
spec.add_development_dependency "simplecov"
|
|
34
|
-
spec.add_development_dependency "vcr"
|
|
37
|
+
spec.add_development_dependency "vcr", "~> 5.0.0"
|
|
35
38
|
spec.add_development_dependency "webmock"
|
|
36
|
-
spec.add_development_dependency "ruby-debug-ide"
|
|
37
|
-
spec.add_development_dependency "debase"
|
|
38
39
|
|
|
39
|
-
spec.add_dependency "iso-bib
|
|
40
|
+
spec.add_dependency "relaton-iso-bib", "~> 0.2.0"
|
|
40
41
|
end
|
data/lib/relaton/processor.rb
CHANGED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module RelatonItu
|
|
2
|
+
class EditorialGroup
|
|
3
|
+
BUREAUS = %w[T D R].freeze
|
|
4
|
+
|
|
5
|
+
# @return [String]
|
|
6
|
+
attr_reader :bureau
|
|
7
|
+
|
|
8
|
+
# @returnn [RelatonItu::ItuGroup]
|
|
9
|
+
attr_reader :group
|
|
10
|
+
|
|
11
|
+
# @return [RelatonItu::ItuGroup, NilClass]
|
|
12
|
+
attr_reader :subgroup, :workgroup
|
|
13
|
+
|
|
14
|
+
# @param bureau [String]
|
|
15
|
+
# @param group [RelatonItu::ItuGroup]
|
|
16
|
+
# @param subgroup [RelatonItu::ItuGroup, NilClass]
|
|
17
|
+
# @param workgroup [RelatonItu::ItuGroup, NilClass]
|
|
18
|
+
def initialize(bureau:, group:, subgroup: nil, workgroup: nil)
|
|
19
|
+
raise ArgumentError, "invalid bureau: #{bureau}" unless BUREAUS.include? bureau
|
|
20
|
+
|
|
21
|
+
@bureau = bureau
|
|
22
|
+
@group = group
|
|
23
|
+
@subgroup = subgroup
|
|
24
|
+
@workgroup = workgroup
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param builder [Nokogiri::XML::Builder]
|
|
28
|
+
def to_xml(builder)
|
|
29
|
+
builder.editorialgroup do
|
|
30
|
+
builder.bureau bureau
|
|
31
|
+
builder.group { |b| group.to_xml b }
|
|
32
|
+
builder.subgroup { |b| group.to_xml b } if subgroup
|
|
33
|
+
builder.workgroup { |b| group.to_xml b } if workgroup
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
3
|
+
require "relaton_itu/hit"
|
|
4
4
|
require "addressable/uri"
|
|
5
|
-
require
|
|
5
|
+
require "net/http"
|
|
6
6
|
|
|
7
7
|
module RelatonItu
|
|
8
8
|
# Page of hit collection.
|
|
9
9
|
class HitCollection < Array
|
|
10
|
-
|
|
11
|
-
DOMAIN = 'https://www.itu.int'.freeze
|
|
10
|
+
DOMAIN = "https://www.itu.int".freeze
|
|
12
11
|
|
|
13
12
|
# @return [TrueClass, FalseClass]
|
|
14
13
|
attr_reader :fetched
|
|
@@ -24,44 +23,65 @@ module RelatonItu
|
|
|
24
23
|
def initialize(ref_nbr, year = nil) #(text, hit_pages = nil)
|
|
25
24
|
@text = ref_nbr
|
|
26
25
|
@year = year
|
|
27
|
-
from, to = nil
|
|
28
|
-
if year
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
end
|
|
32
|
-
url
|
|
26
|
+
# from, to = nil
|
|
27
|
+
# if year
|
|
28
|
+
# from = Date.strptime year, "%Y"
|
|
29
|
+
# to = from.next_year.prev_day
|
|
30
|
+
# end
|
|
31
|
+
url = "#{DOMAIN}/net4/ITU-T/search/GlobalSearch/Search"
|
|
33
32
|
params = {
|
|
34
|
-
"Input"=>"163",
|
|
35
|
-
"Start"=>0,
|
|
36
|
-
"Rows"=>10,
|
|
37
|
-
"SortBy"=>"RELEVANCE",
|
|
38
|
-
"ExactPhrase"=>false,
|
|
39
|
-
"CollectionName"=>"General",
|
|
40
|
-
"CollectionGroup"=>"Recommendations",
|
|
41
|
-
"Sector"=>"t",
|
|
42
|
-
"Criterias"=> [{
|
|
43
|
-
"Name"=>"Search in",
|
|
33
|
+
"Input" => "163",
|
|
34
|
+
"Start" => 0,
|
|
35
|
+
"Rows" => 10,
|
|
36
|
+
"SortBy" => "RELEVANCE",
|
|
37
|
+
"ExactPhrase" => false,
|
|
38
|
+
"CollectionName" => "General",
|
|
39
|
+
"CollectionGroup" => "Recommendations",
|
|
40
|
+
"Sector" => "t",
|
|
41
|
+
"Criterias" => [{
|
|
42
|
+
"Name" => "Search in",
|
|
44
43
|
"Criterias" => [
|
|
45
|
-
{
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
{
|
|
45
|
+
"Selected" => false,
|
|
46
|
+
"Value" => "",
|
|
47
|
+
"Label" => "Name",
|
|
48
|
+
"Target" => "/name_s",
|
|
49
|
+
"TypeName" => "CHECKBOX",
|
|
50
|
+
"GetCriteriaType" => 0,
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"Selected" => false,
|
|
54
|
+
"Value" => "",
|
|
55
|
+
"Label" => "Short description",
|
|
56
|
+
"Target" => "/short_description_s",
|
|
57
|
+
"TypeName" => "CHECKBOX",
|
|
58
|
+
"GetCriteriaType" => 0,
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"Selected" => false,
|
|
62
|
+
"Value" => "",
|
|
63
|
+
"Label" => "File content",
|
|
64
|
+
"Target" => "/file",
|
|
65
|
+
"TypeName" => "CHECKBOX",
|
|
66
|
+
"GetCriteriaType" => 0,
|
|
67
|
+
},
|
|
48
68
|
],
|
|
49
|
-
"ShowCheckbox"=>true,
|
|
50
|
-
"Selected"=>false
|
|
69
|
+
"ShowCheckbox" => true,
|
|
70
|
+
"Selected" => false,
|
|
51
71
|
}],
|
|
52
|
-
"Topics"=>"",
|
|
53
|
-
"ClientData"=>{"ip"=>""},
|
|
54
|
-
"Language"=>"en",
|
|
55
|
-
"IP"=>"",
|
|
56
|
-
"SearchType"=>"All"
|
|
72
|
+
"Topics" => "",
|
|
73
|
+
"ClientData" => { "ip" => "" },
|
|
74
|
+
"Language" => "en",
|
|
75
|
+
"IP" => "",
|
|
76
|
+
"SearchType" => "All",
|
|
57
77
|
}
|
|
58
78
|
data = { json: params.to_json }
|
|
59
|
-
resp = Net::HTTP.post(URI(url), data.to_json,
|
|
79
|
+
resp = Net::HTTP.post(URI(url), data.to_json, "Content-Type" => "application/json")
|
|
60
80
|
doc = JSON.parse resp.body
|
|
61
|
-
hits = doc[
|
|
62
|
-
code = h[
|
|
63
|
-
title = h[
|
|
64
|
-
url = h[
|
|
81
|
+
hits = doc["results"].map do |h|
|
|
82
|
+
code = h["Media"]["Name"]
|
|
83
|
+
title = h["Title"]
|
|
84
|
+
url = h["Redirection"]
|
|
65
85
|
Hit.new({ code: code, title: title, url: url }, self)
|
|
66
86
|
end
|
|
67
87
|
concat hits
|
|
@@ -70,7 +90,7 @@ module RelatonItu
|
|
|
70
90
|
|
|
71
91
|
# @return [RelatonItu::HitCollection]
|
|
72
92
|
def fetch
|
|
73
|
-
workers = WorkersPool.new 4
|
|
93
|
+
workers = RelatonBib::WorkersPool.new 4
|
|
74
94
|
workers.worker(&:fetch)
|
|
75
95
|
each do |hit|
|
|
76
96
|
workers << hit
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
module RelatonItu
|
|
2
|
+
class ItuBibliographicItem < RelatonIsoBib::IsoBibliographicItem
|
|
3
|
+
TYPES = %w[
|
|
4
|
+
recommendation recommendation-supplement recommendation-amendment
|
|
5
|
+
recommendation-corrigendum recommendation-errata recommendation-annex
|
|
6
|
+
focus-group implementers-guide technical-paper technical-report
|
|
7
|
+
joint-itu-iso-iec
|
|
8
|
+
].freeze
|
|
9
|
+
|
|
10
|
+
def initialize(**args)
|
|
11
|
+
@doctype = args.delete :type
|
|
12
|
+
if doctype && !TYPES.include?(doctype)
|
|
13
|
+
raise ArgumentError, "invalid type: #{doctype}"
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
super
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def to_xml(builder = nil, **opts)
|
|
20
|
+
super builder, **opts do |b|
|
|
21
|
+
if opts[:bibdata]
|
|
22
|
+
# b.doctype doctype if doctype
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
require
|
|
5
|
-
require
|
|
3
|
+
require "relaton_iso_bib"
|
|
4
|
+
require "relaton_itu/itu_bibliographic_item"
|
|
5
|
+
require "relaton_itu/editorial_group"
|
|
6
|
+
require "relaton_itu/itu_group"
|
|
7
|
+
require "relaton_itu/scrapper"
|
|
8
|
+
require "relaton_itu/hit_collection"
|
|
9
|
+
require "relaton_itu/hit"
|
|
10
|
+
require "relaton_itu/xml_parser"
|
|
6
11
|
require "date"
|
|
7
12
|
|
|
8
13
|
module RelatonItu
|
|
@@ -12,12 +17,10 @@ module RelatonItu
|
|
|
12
17
|
# @param text [String]
|
|
13
18
|
# @return [RelatonItu::HitCollection]
|
|
14
19
|
def search(text, year = nil)
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
[]
|
|
20
|
-
end
|
|
20
|
+
HitCollection.new text, year
|
|
21
|
+
rescue
|
|
22
|
+
warn "Could not access http://www.itu.int"
|
|
23
|
+
[]
|
|
21
24
|
end
|
|
22
25
|
|
|
23
26
|
# @param code [String] the ISO standard Code to look up (e..g "ISO 9000")
|
|
@@ -33,9 +36,10 @@ module RelatonItu
|
|
|
33
36
|
end
|
|
34
37
|
end
|
|
35
38
|
|
|
36
|
-
code +=
|
|
39
|
+
code += "-1" if opts[:all_parts]
|
|
37
40
|
ret = itubib_get1(code, year, opts)
|
|
38
41
|
return nil if ret.nil?
|
|
42
|
+
|
|
39
43
|
ret.to_most_recent_reference unless year || opts[:keep_year]
|
|
40
44
|
ret.to_all_parts if opts[:all_parts]
|
|
41
45
|
ret
|
|
@@ -61,7 +65,7 @@ module RelatonItu
|
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
def fetch_pages(s, n)
|
|
64
|
-
workers = WorkersPool.new n
|
|
68
|
+
workers = RelatonBib::WorkersPool.new n
|
|
65
69
|
workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
|
|
66
70
|
s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
|
|
67
71
|
workers.end
|
|
@@ -89,8 +93,10 @@ module RelatonItu
|
|
|
89
93
|
result.each_slice(3) do |s| # ISO website only allows 3 connections
|
|
90
94
|
fetch_pages(s, 3).each_with_index do |r, i|
|
|
91
95
|
return { ret: r } if !year
|
|
96
|
+
|
|
92
97
|
r.dates.select { |d| d.type == "published" }.each do |d|
|
|
93
98
|
return { ret: r } if year.to_i == d.on.year
|
|
99
|
+
|
|
94
100
|
missed_years << d.on.year
|
|
95
101
|
end
|
|
96
102
|
end
|
|
@@ -102,6 +108,7 @@ module RelatonItu
|
|
|
102
108
|
result = search_filter(code) or return nil
|
|
103
109
|
ret = isobib_results_filter(result, year)
|
|
104
110
|
return ret[:ret] if ret[:ret]
|
|
111
|
+
|
|
105
112
|
fetch_ref_err(code, year, ret[:years])
|
|
106
113
|
end
|
|
107
114
|
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
module RelatonItu
|
|
2
|
+
class ItuGroup
|
|
3
|
+
class Period
|
|
4
|
+
# @return [String] group period start year
|
|
5
|
+
attr_reader :start
|
|
6
|
+
|
|
7
|
+
# @return [String, NilClass] group period end year
|
|
8
|
+
attr_reader :finish
|
|
9
|
+
|
|
10
|
+
# @params start [String]
|
|
11
|
+
# @params finish [String, NilClass]
|
|
12
|
+
def initialize(start:, finish: nil)
|
|
13
|
+
@start = start
|
|
14
|
+
@finish = finish
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# @param builder [Nokogiri::XML::Builder]
|
|
18
|
+
def to_xml(builder)
|
|
19
|
+
builder.period do
|
|
20
|
+
builder.start start
|
|
21
|
+
builder.end finish if finish
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
TYPES = %w[tsag study-group work-group].freeze
|
|
27
|
+
|
|
28
|
+
# @return [String]
|
|
29
|
+
attr_reader :name
|
|
30
|
+
|
|
31
|
+
# @return [String, NilClass]
|
|
32
|
+
attr_reader :type, :acronym
|
|
33
|
+
|
|
34
|
+
# @return [RelatonItu::ItuGroup::Period, NilClass] group period
|
|
35
|
+
attr_reader :period
|
|
36
|
+
|
|
37
|
+
# @param type [String, NilClass]
|
|
38
|
+
# @param name [String]
|
|
39
|
+
# @param acronym [String, NilClass]
|
|
40
|
+
# @param period [RelatonItu::ItuGroup::Period, NilClass]
|
|
41
|
+
def initialize(type: nil, name:, acronym: nil, period: nil)
|
|
42
|
+
if type && !TYPES.include?(type)
|
|
43
|
+
raise ArgumentError, "invalid type: #{type}"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
@type = type
|
|
47
|
+
@name = name
|
|
48
|
+
@acronym = acronym
|
|
49
|
+
@period = period
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @param builder [Nokogiri::XML::Builder]
|
|
53
|
+
def to_xml(builder)
|
|
54
|
+
builder.parent[:type] = type if type
|
|
55
|
+
builder.name name
|
|
56
|
+
builder.acronym acronym if acronym
|
|
57
|
+
period&.to_xml builder
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
data/lib/relaton_itu/scrapper.rb
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require
|
|
4
|
-
require
|
|
5
|
-
require 'nokogiri'
|
|
6
|
-
require 'net/http'
|
|
7
|
-
require 'relaton_itu/workers_pool'
|
|
3
|
+
require "nokogiri"
|
|
4
|
+
require "net/http"
|
|
8
5
|
|
|
9
6
|
# Capybara.request_driver :poltergeist do |app|
|
|
10
7
|
# Capybara::Poltergeist::Driver.new app, js_errors: false
|
|
@@ -15,21 +12,21 @@ module RelatonItu
|
|
|
15
12
|
# Scrapper.
|
|
16
13
|
# rubocop:disable Metrics/ModuleLength
|
|
17
14
|
module Scrapper
|
|
18
|
-
DOMAIN =
|
|
15
|
+
DOMAIN = "https://www.itu.int"
|
|
19
16
|
|
|
20
17
|
TYPES = {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
18
|
+
"ISO" => "international-standard",
|
|
19
|
+
"TS" => "technicalSpecification",
|
|
20
|
+
"TR" => "technicalReport",
|
|
21
|
+
"PAS" => "publiclyAvailableSpecification",
|
|
22
|
+
"AWI" => "appruvedWorkItem",
|
|
23
|
+
"CD" => "committeeDraft",
|
|
24
|
+
"FDIS" => "finalDraftInternationalStandard",
|
|
25
|
+
"NP" => "newProposal",
|
|
26
|
+
"DIS" => "draftInternationalStandard",
|
|
27
|
+
"WD" => "workingDraft",
|
|
28
|
+
"R" => "recommendation",
|
|
29
|
+
"Guide" => "guide",
|
|
33
30
|
}.freeze
|
|
34
31
|
|
|
35
32
|
class << self
|
|
@@ -55,22 +52,23 @@ module RelatonItu
|
|
|
55
52
|
# Fetch edition.
|
|
56
53
|
edition = doc.at("//table/tr/td/span[contains(@id, 'Label8')]/b").text
|
|
57
54
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
55
|
+
ItuBibliographicItem.new(
|
|
56
|
+
fetched: Date.today.to_s,
|
|
57
|
+
docid: fetch_docid(hit_data[:code]),
|
|
58
|
+
edition: edition,
|
|
59
|
+
language: ["en"],
|
|
60
|
+
script: ["Latn"],
|
|
61
|
+
titles: fetch_titles(hit_data),
|
|
62
|
+
type: fetch_type(doc),
|
|
63
|
+
docstatus: fetch_status(doc),
|
|
64
|
+
ics: [], # fetch_ics(doc),
|
|
65
|
+
dates: fetch_dates(doc),
|
|
68
66
|
contributors: fetch_contributors(hit_data[:code]),
|
|
69
|
-
|
|
70
|
-
abstract:
|
|
71
|
-
copyright:
|
|
72
|
-
link:
|
|
73
|
-
relations:
|
|
67
|
+
editorialgroup: fetch_workgroup(doc),
|
|
68
|
+
abstract: fetch_abstract(doc),
|
|
69
|
+
copyright: fetch_copyright(hit_data[:code], doc),
|
|
70
|
+
link: fetch_link(doc, hit_data[:url]),
|
|
71
|
+
relations: fetch_relations(doc),
|
|
74
72
|
)
|
|
75
73
|
end
|
|
76
74
|
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
@@ -86,13 +84,13 @@ module RelatonItu
|
|
|
86
84
|
|
|
87
85
|
url = abstract_url[:onclick].match(/https?[^']+/).to_s
|
|
88
86
|
d = Nokogiri::HTML Net::HTTP.get(URI(url))
|
|
89
|
-
abstract_content = d.css(
|
|
90
|
-
.gsub(/\s{2,}/,
|
|
87
|
+
abstract_content = d.css("p.MsoNormal").text.gsub(/\r\n/, "")
|
|
88
|
+
.gsub(/\s{2,}/, " ").gsub(/\u00a0/, "")
|
|
91
89
|
|
|
92
90
|
[{
|
|
93
|
-
content:
|
|
94
|
-
language:
|
|
95
|
-
script:
|
|
91
|
+
content: abstract_content,
|
|
92
|
+
language: "en",
|
|
93
|
+
script: "Latn",
|
|
96
94
|
}]
|
|
97
95
|
end
|
|
98
96
|
|
|
@@ -115,10 +113,10 @@ module RelatonItu
|
|
|
115
113
|
# @return [Array<Nokogiri::HTML::Document, String>]
|
|
116
114
|
def get_page(url)
|
|
117
115
|
uri = URI url
|
|
118
|
-
resp = Net::HTTP.get_response(uri)
|
|
119
|
-
while resp.code ==
|
|
120
|
-
uri = URI resp[
|
|
121
|
-
resp = Net::HTTP.get_response(uri)
|
|
116
|
+
resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
|
|
117
|
+
while resp.code == "301" || resp.code == "302" || resp.code == "303"
|
|
118
|
+
uri = URI resp["location"]
|
|
119
|
+
resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
|
|
122
120
|
end
|
|
123
121
|
Nokogiri::HTML(resp.body)
|
|
124
122
|
end
|
|
@@ -128,15 +126,11 @@ module RelatonItu
|
|
|
128
126
|
# @param doc [Nokogiri::HTML::Document]
|
|
129
127
|
# @return [Hash]
|
|
130
128
|
def fetch_docid(code)
|
|
131
|
-
m = code.match(/(?<=\s)(?<project>[^\s]+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
prefix: nil,
|
|
137
|
-
type: 'ITU',
|
|
138
|
-
id: code
|
|
139
|
-
}
|
|
129
|
+
# m = code.match(/(?<=\s)(?<project>[^\s]+)-?(?<part>(?<=-)\d+|)-?(?<subpart>(?<=-)\d+|)/)
|
|
130
|
+
# project_number: m[:project],
|
|
131
|
+
# part_number: m[:part],
|
|
132
|
+
# subpart_number: m[:subpart],
|
|
133
|
+
[RelatonBib::DocumentIdentifier.new(type: "ITU", id: code)]
|
|
140
134
|
end
|
|
141
135
|
|
|
142
136
|
# Fetch status.
|
|
@@ -145,62 +139,82 @@ module RelatonItu
|
|
|
145
139
|
# @return [Hash]
|
|
146
140
|
def fetch_status(doc)
|
|
147
141
|
s = doc.at("//table/tr/td/span[contains(@id, 'Label7')]").text
|
|
148
|
-
if s ==
|
|
149
|
-
status =
|
|
150
|
-
stage =
|
|
151
|
-
substage =
|
|
142
|
+
if s == "In force"
|
|
143
|
+
status = "Published"
|
|
144
|
+
# stage = "60"
|
|
145
|
+
# substage = "60"
|
|
152
146
|
else
|
|
153
|
-
status =
|
|
154
|
-
stage =
|
|
155
|
-
substage =
|
|
147
|
+
status = "Withdrawal"
|
|
148
|
+
# stage = "95"
|
|
149
|
+
# substage = "99"
|
|
156
150
|
end
|
|
157
|
-
|
|
151
|
+
RelatonBib::DocumentStatus.new(stage: status)
|
|
158
152
|
end
|
|
159
153
|
|
|
160
154
|
# Fetch workgroup.
|
|
161
155
|
# @param doc [Nokogiri::HTML::Document]
|
|
162
|
-
# @return [
|
|
156
|
+
# @return [RelatonItu::EditorialGroup]
|
|
163
157
|
def fetch_workgroup(doc)
|
|
164
158
|
wg = doc.at('//table/tr/td/span[contains(@id, "Label8")]/a').text
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
159
|
+
EditorialGroup.new(
|
|
160
|
+
bureau: wg.match(/(?<=-)./).to_s,
|
|
161
|
+
group: itugroup(wg),
|
|
162
|
+
# name: "International Telecommunication Union",
|
|
163
|
+
# abbreviation: "ITU",
|
|
164
|
+
# url: "www.itu.int",
|
|
165
|
+
# technical_committee: tc,
|
|
166
|
+
)
|
|
173
167
|
end
|
|
174
168
|
|
|
169
|
+
# @param name [String]
|
|
170
|
+
# @return [RelatonItu::ItuGroup]
|
|
171
|
+
def itugroup(name)
|
|
172
|
+
if name.include? "Study Group"
|
|
173
|
+
type = "study-group"
|
|
174
|
+
acronym = "SG"
|
|
175
|
+
elsif name.include? "Telecommunication Standardization Advisory Group"
|
|
176
|
+
type = "tsag"
|
|
177
|
+
acronym = "TSAG"
|
|
178
|
+
else
|
|
179
|
+
type = "work-group"
|
|
180
|
+
acronym = "WG"
|
|
181
|
+
end
|
|
182
|
+
ItuGroup.new name: name, type: type, acronym: acronym
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# rubocop:disable Metrics/MethodLength
|
|
186
|
+
|
|
175
187
|
# Fetch relations.
|
|
176
188
|
# @param doc [Nokogiri::HTML::Document]
|
|
177
189
|
# @return [Array<Hash>]
|
|
178
|
-
# rubocop:disable Metrics/MethodLength
|
|
179
190
|
def fetch_relations(doc)
|
|
180
191
|
doc.xpath('//div[contains(@id, "tab_sup")]//table/tr[position()>2]').map do |r|
|
|
181
192
|
r_type = r.at('./td/span[contains(@id, "Label4")]/nobr').text.downcase
|
|
182
193
|
type = case r_type
|
|
183
|
-
when
|
|
194
|
+
when "in force" then "published"
|
|
184
195
|
else r_type
|
|
185
196
|
end
|
|
186
197
|
ref = r.at('./td/span[contains(@id, "title_e")]/nobr/a')
|
|
187
|
-
url = DOMAIN + ref[:href].sub(/^\./,
|
|
188
|
-
|
|
198
|
+
# url = DOMAIN + ref[:href].sub(/^\./, "/ITU-T/recommendations")
|
|
199
|
+
fref = RelatonBib::FormattedRef.new(content: ref.text, language: "en", script: "Latn")
|
|
200
|
+
bibitem = RelatonIsoBib::IsoBibliographicItem.new(formattedref: fref)
|
|
201
|
+
{ type: type, bibitem: bibitem }
|
|
189
202
|
end
|
|
190
203
|
end
|
|
204
|
+
# rubocop:enable Metrics/MethodLength
|
|
191
205
|
|
|
192
206
|
# Fetch type.
|
|
193
207
|
# @param doc [Nokogiri::HTML::Document]
|
|
194
208
|
# @return [String]
|
|
195
|
-
def fetch_type(
|
|
196
|
-
|
|
209
|
+
def fetch_type(_doc)
|
|
210
|
+
"recommendation"
|
|
197
211
|
end
|
|
198
212
|
|
|
199
213
|
# Fetch titles.
|
|
200
214
|
# @param hit_data [Hash]
|
|
201
215
|
# @return [Array<Hash>]
|
|
202
216
|
def fetch_titles(hit_data)
|
|
203
|
-
titles = hit_data[:title].split
|
|
217
|
+
titles = hit_data[:title].split " - "
|
|
204
218
|
case titles.size
|
|
205
219
|
when 0
|
|
206
220
|
intro, main, part = nil, "", nil
|
|
@@ -219,10 +233,10 @@ module RelatonItu
|
|
|
219
233
|
end
|
|
220
234
|
[{
|
|
221
235
|
title_intro: intro,
|
|
222
|
-
title_main:
|
|
223
|
-
title_part:
|
|
224
|
-
language:
|
|
225
|
-
script:
|
|
236
|
+
title_main: main,
|
|
237
|
+
title_part: part,
|
|
238
|
+
language: "en",
|
|
239
|
+
script: "Latn",
|
|
226
240
|
}]
|
|
227
241
|
end
|
|
228
242
|
|
|
@@ -233,7 +247,7 @@ module RelatonItu
|
|
|
233
247
|
dates = []
|
|
234
248
|
publish_date = doc.at("//table/tr/td/span[contains(@id, 'Label5')]").text
|
|
235
249
|
unless publish_date.empty?
|
|
236
|
-
dates << { type:
|
|
250
|
+
dates << { type: "published", on: publish_date }
|
|
237
251
|
end
|
|
238
252
|
dates
|
|
239
253
|
end
|
|
@@ -242,13 +256,13 @@ module RelatonItu
|
|
|
242
256
|
# @param doc [Nokogiri::HTML::Document]
|
|
243
257
|
# @return [Array<Hash>]
|
|
244
258
|
def fetch_contributors(code)
|
|
245
|
-
abbrev = code.sub(/-\w\s.*/,
|
|
259
|
+
abbrev = code.sub(/-\w\s.*/, "")
|
|
246
260
|
case abbrev
|
|
247
|
-
when
|
|
248
|
-
name =
|
|
249
|
-
url =
|
|
261
|
+
when "ITU"
|
|
262
|
+
name = "International Telecommunication Union"
|
|
263
|
+
url = "www.itu.int"
|
|
250
264
|
end
|
|
251
|
-
[{ entity: { name: name, url: url, abbreviation: abbrev }, roles: [
|
|
265
|
+
[{ entity: { name: name, url: url, abbreviation: abbrev }, roles: ["publisher"] }]
|
|
252
266
|
end
|
|
253
267
|
|
|
254
268
|
# Fetch ICS.
|
|
@@ -266,9 +280,9 @@ module RelatonItu
|
|
|
266
280
|
# @param url [String]
|
|
267
281
|
# @return [Array<Hash>]
|
|
268
282
|
def fetch_link(doc, url)
|
|
269
|
-
links = [{ type:
|
|
283
|
+
links = [{ type: "src", content: url }]
|
|
270
284
|
obp_elms = doc.at('//table/tr/td/span[contains(@id, "Label4")]/a')
|
|
271
|
-
links << { type:
|
|
285
|
+
links << { type: "obp", content: DOMAIN + obp_elms[:href] } if obp_elms
|
|
272
286
|
links
|
|
273
287
|
end
|
|
274
288
|
|
|
@@ -279,9 +293,9 @@ module RelatonItu
|
|
|
279
293
|
def fetch_copyright(code, doc)
|
|
280
294
|
abbreviation = code.match(/^[^-]+/).to_s
|
|
281
295
|
case abbreviation
|
|
282
|
-
when
|
|
283
|
-
name =
|
|
284
|
-
url =
|
|
296
|
+
when "ITU"
|
|
297
|
+
name = "International Telecommunication Union"
|
|
298
|
+
url = "www.itu.int"
|
|
285
299
|
end
|
|
286
300
|
from = doc.at("//table/tr/td/span[contains(@id, 'Label5')]").text
|
|
287
301
|
{ owner: { name: name, abbreviation: abbreviation, url: url }, from: from }
|
data/lib/relaton_itu/version.rb
CHANGED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
require "nokogiri"
|
|
2
|
+
|
|
3
|
+
module RelatonItu
|
|
4
|
+
class XMLParser < RelatonIsoBib::XMLParser
|
|
5
|
+
class << self
|
|
6
|
+
# Override RelatonIsoBib::XMLParser.form_xml method.
|
|
7
|
+
# @param xml [String]
|
|
8
|
+
# @return [RelatonItu::ItuBibliographicItem]
|
|
9
|
+
def from_xml(xml)
|
|
10
|
+
doc = Nokogiri::XML(xml)
|
|
11
|
+
ituitem = doc.at "/bibitem|/bibdata"
|
|
12
|
+
ItuBibliographicItem.new item_data(ituitem)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
# @param ext [Nokogiri::XML::Element]
|
|
18
|
+
# @return [RelatonItu::EditorialGroup]
|
|
19
|
+
def fetch_editorialgroup(ext)
|
|
20
|
+
eg = ext.at("./editorialgroup")
|
|
21
|
+
return unless eg
|
|
22
|
+
|
|
23
|
+
EditorialGroup.new(
|
|
24
|
+
bureau: eg.at("bureau")&.text,
|
|
25
|
+
group: itugroup(eg.at("group")),
|
|
26
|
+
subgroup: itugroup(eg.at("subgroup")),
|
|
27
|
+
workgroup: itugroup(eg.at("workgroup")),
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# @param com [Nokogiri::XML::Element]
|
|
32
|
+
# @return [RelatonItu::ItuGroup]
|
|
33
|
+
def itugroup(group)
|
|
34
|
+
return unless group
|
|
35
|
+
|
|
36
|
+
ItuGroup.new(
|
|
37
|
+
type: group[:type],
|
|
38
|
+
name: group.at("name").text,
|
|
39
|
+
acronym: group.at("acronym").text,
|
|
40
|
+
period: itugroupperiod(group.at("period")),
|
|
41
|
+
)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @param com [Nokogiri::XML::Element]
|
|
45
|
+
# @return [RelatonItu::ItuGroup::Period]
|
|
46
|
+
def itugroupperiod(period)
|
|
47
|
+
return until period
|
|
48
|
+
|
|
49
|
+
ItuGroup::Period.new(
|
|
50
|
+
start: period.at("start").text, finish: period.at("end")&.text,
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
data/lib/relaton_itu.rb
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
require "relaton_itu/version"
|
|
2
|
-
require
|
|
2
|
+
require "relaton_itu/itu_bibliography"
|
|
3
3
|
|
|
4
4
|
if defined? Relaton
|
|
5
|
-
require_relative
|
|
5
|
+
require_relative "relaton/processor"
|
|
6
6
|
Relaton::Registry.instance.register(Relaton::RelatonItu::Processor)
|
|
7
7
|
end
|
|
8
8
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton-itu
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-
|
|
11
|
+
date: 2019-06-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -24,6 +24,20 @@ dependencies:
|
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '2.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: debase
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
27
41
|
- !ruby/object:Gem::Dependency
|
|
28
42
|
name: equivalent-xml
|
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -81,21 +95,7 @@ dependencies:
|
|
|
81
95
|
- !ruby/object:Gem::Version
|
|
82
96
|
version: '3.0'
|
|
83
97
|
- !ruby/object:Gem::Dependency
|
|
84
|
-
name:
|
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
|
86
|
-
requirements:
|
|
87
|
-
- - ">="
|
|
88
|
-
- !ruby/object:Gem::Version
|
|
89
|
-
version: '0'
|
|
90
|
-
type: :development
|
|
91
|
-
prerelease: false
|
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
-
requirements:
|
|
94
|
-
- - ">="
|
|
95
|
-
- !ruby/object:Gem::Version
|
|
96
|
-
version: '0'
|
|
97
|
-
- !ruby/object:Gem::Dependency
|
|
98
|
-
name: vcr
|
|
98
|
+
name: ruby-debug-ide
|
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
|
100
100
|
requirements:
|
|
101
101
|
- - ">="
|
|
@@ -109,7 +109,7 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: '0'
|
|
111
111
|
- !ruby/object:Gem::Dependency
|
|
112
|
-
name:
|
|
112
|
+
name: simplecov
|
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
|
114
114
|
requirements:
|
|
115
115
|
- - ">="
|
|
@@ -123,21 +123,21 @@ dependencies:
|
|
|
123
123
|
- !ruby/object:Gem::Version
|
|
124
124
|
version: '0'
|
|
125
125
|
- !ruby/object:Gem::Dependency
|
|
126
|
-
name:
|
|
126
|
+
name: vcr
|
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
|
128
128
|
requirements:
|
|
129
|
-
- - "
|
|
129
|
+
- - "~>"
|
|
130
130
|
- !ruby/object:Gem::Version
|
|
131
|
-
version:
|
|
131
|
+
version: 5.0.0
|
|
132
132
|
type: :development
|
|
133
133
|
prerelease: false
|
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
|
135
135
|
requirements:
|
|
136
|
-
- - "
|
|
136
|
+
- - "~>"
|
|
137
137
|
- !ruby/object:Gem::Version
|
|
138
|
-
version:
|
|
138
|
+
version: 5.0.0
|
|
139
139
|
- !ruby/object:Gem::Dependency
|
|
140
|
-
name:
|
|
140
|
+
name: webmock
|
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
|
142
142
|
requirements:
|
|
143
143
|
- - ">="
|
|
@@ -151,19 +151,19 @@ dependencies:
|
|
|
151
151
|
- !ruby/object:Gem::Version
|
|
152
152
|
version: '0'
|
|
153
153
|
- !ruby/object:Gem::Dependency
|
|
154
|
-
name: iso-bib
|
|
154
|
+
name: relaton-iso-bib
|
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
|
156
156
|
requirements:
|
|
157
157
|
- - "~>"
|
|
158
158
|
- !ruby/object:Gem::Version
|
|
159
|
-
version: 0.
|
|
159
|
+
version: 0.2.0
|
|
160
160
|
type: :runtime
|
|
161
161
|
prerelease: false
|
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
|
163
163
|
requirements:
|
|
164
164
|
- - "~>"
|
|
165
165
|
- !ruby/object:Gem::Version
|
|
166
|
-
version: 0.
|
|
166
|
+
version: 0.2.0
|
|
167
167
|
description: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
|
|
168
168
|
model'
|
|
169
169
|
email:
|
|
@@ -187,12 +187,15 @@ files:
|
|
|
187
187
|
- itubib.gemspec
|
|
188
188
|
- lib/relaton/processor.rb
|
|
189
189
|
- lib/relaton_itu.rb
|
|
190
|
+
- lib/relaton_itu/editorial_group.rb
|
|
190
191
|
- lib/relaton_itu/hit.rb
|
|
191
192
|
- lib/relaton_itu/hit_collection.rb
|
|
193
|
+
- lib/relaton_itu/itu_bibliographic_item.rb
|
|
192
194
|
- lib/relaton_itu/itu_bibliography.rb
|
|
195
|
+
- lib/relaton_itu/itu_group.rb
|
|
193
196
|
- lib/relaton_itu/scrapper.rb
|
|
194
197
|
- lib/relaton_itu/version.rb
|
|
195
|
-
- lib/relaton_itu/
|
|
198
|
+
- lib/relaton_itu/xml_parser.rb
|
|
196
199
|
homepage: https://github.com/metanorma/relaton-itu
|
|
197
200
|
licenses:
|
|
198
201
|
- MIT
|
|
@@ -205,7 +208,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
205
208
|
requirements:
|
|
206
209
|
- - ">="
|
|
207
210
|
- !ruby/object:Gem::Version
|
|
208
|
-
version:
|
|
211
|
+
version: 2.4.0
|
|
209
212
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
210
213
|
requirements:
|
|
211
214
|
- - ">="
|
|
@@ -213,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
213
216
|
version: '0'
|
|
214
217
|
requirements: []
|
|
215
218
|
rubyforge_project:
|
|
216
|
-
rubygems_version: 2.6
|
|
219
|
+
rubygems_version: 2.7.6
|
|
217
220
|
signing_key:
|
|
218
221
|
specification_version: 4
|
|
219
222
|
summary: 'RelatonItu: retrieve ITU Standards for bibliographic use using the BibliographicItem
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# Workers poll.
|
|
4
|
-
class WorkersPool
|
|
5
|
-
attr_accessor :nb_hits
|
|
6
|
-
|
|
7
|
-
def initialize(num_workers = 2)
|
|
8
|
-
@num_workers = num_workers < 2 ? 2 : num_workers
|
|
9
|
-
@queue = SizedQueue.new(num_workers * 2)
|
|
10
|
-
@result = []
|
|
11
|
-
@nb_hits = 0
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def worker(&block)
|
|
15
|
-
@threads = Array.new @num_workers do
|
|
16
|
-
Thread.new do
|
|
17
|
-
until (item = @queue.pop) == :END
|
|
18
|
-
@result << yield(item) if block
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def result
|
|
25
|
-
@threads.each(&:join)
|
|
26
|
-
@result
|
|
27
|
-
end
|
|
28
|
-
|
|
29
|
-
def <<(item)
|
|
30
|
-
@queue << item
|
|
31
|
-
self
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def end
|
|
35
|
-
@num_workers.times { @queue << :END }
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
def size
|
|
39
|
-
@result.size
|
|
40
|
-
end
|
|
41
|
-
end
|