relaton-nist 0.2.3 → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/Gemfile.lock +9 -9
- data/README.adoc +16 -0
- data/lib/relaton_nist.rb +4 -20
- data/lib/relaton_nist/comment_period.rb +9 -1
- data/lib/relaton_nist/hash_converter.rb +30 -0
- data/lib/relaton_nist/hit.rb +1 -30
- data/lib/relaton_nist/hit_collection.rb +54 -57
- data/lib/relaton_nist/keyword.rb +7 -0
- data/lib/relaton_nist/nist_bibliographic_item.rb +24 -16
- data/lib/relaton_nist/nist_bibliography.rb +8 -11
- data/lib/relaton_nist/processor.rb +33 -0
- data/lib/relaton_nist/scrapper.rb +24 -23
- data/lib/relaton_nist/version.rb +1 -1
- data/lib/relaton_nist/xml_parser.rb +4 -1
- data/relaton_nist.gemspec +1 -1
- metadata +6 -5
- data/lib/relaton/processor.rb +0 -23
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6ad28b9e4eda9f14ac735a69842d2e4d63c7c88e
|
|
4
|
+
data.tar.gz: e20f4b0086adbe773a420d22d0ee30c352001149
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8ac638788da72901abd8373f68a1c6228584b4754823251e597b20b4898a3f14de21ad870b5dc7e27deccb09f2c18e0af7dee8e8952f5395ea5f001ca824094e
|
|
7
|
+
data.tar.gz: 46f960c6178eebaeb1ed9a36bf9989746e59b3266691e07aa4d5ce457f2c7e6d871905d8e51ba025d3409457126bfc8c1dcbdd4924935843243b143fd012528f
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
relaton-nist (0.
|
|
5
|
-
relaton-bib (~> 0.
|
|
4
|
+
relaton-nist (0.3.4)
|
|
5
|
+
relaton-bib (~> 0.3.0)
|
|
6
6
|
rubyzip
|
|
7
7
|
|
|
8
8
|
GEM
|
|
9
9
|
remote: https://rubygems.org/
|
|
10
10
|
specs:
|
|
11
|
-
addressable (2.
|
|
12
|
-
public_suffix (>= 2.0.2, <
|
|
11
|
+
addressable (2.7.0)
|
|
12
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
13
13
|
byebug (11.0.1)
|
|
14
14
|
coderay (1.1.2)
|
|
15
15
|
crack (0.4.3)
|
|
@@ -25,7 +25,7 @@ GEM
|
|
|
25
25
|
json (2.2.0)
|
|
26
26
|
method_source (0.9.2)
|
|
27
27
|
mini_portile2 (2.4.0)
|
|
28
|
-
nokogiri (1.10.
|
|
28
|
+
nokogiri (1.10.4)
|
|
29
29
|
mini_portile2 (~> 2.4.0)
|
|
30
30
|
pry (0.12.2)
|
|
31
31
|
coderay (~> 1.1.0)
|
|
@@ -33,11 +33,11 @@ GEM
|
|
|
33
33
|
pry-byebug (3.7.0)
|
|
34
34
|
byebug (~> 11.0)
|
|
35
35
|
pry (~> 0.10)
|
|
36
|
-
public_suffix (
|
|
36
|
+
public_suffix (4.0.1)
|
|
37
37
|
rake (10.5.0)
|
|
38
|
-
relaton-bib (0.
|
|
38
|
+
relaton-bib (0.3.6)
|
|
39
39
|
addressable
|
|
40
|
-
nokogiri
|
|
40
|
+
nokogiri
|
|
41
41
|
rspec (3.8.0)
|
|
42
42
|
rspec-core (~> 3.8.0)
|
|
43
43
|
rspec-expectations (~> 3.8.0)
|
|
@@ -53,7 +53,7 @@ GEM
|
|
|
53
53
|
rspec-support (3.8.2)
|
|
54
54
|
ruby-debug-ide (0.7.0)
|
|
55
55
|
rake (>= 0.8.1)
|
|
56
|
-
rubyzip (1.2.
|
|
56
|
+
rubyzip (1.2.4)
|
|
57
57
|
safe_yaml (1.0.5)
|
|
58
58
|
simplecov (0.16.1)
|
|
59
59
|
docile (~> 1.1)
|
data/README.adoc
CHANGED
|
@@ -113,6 +113,22 @@ fetching 8200...
|
|
|
113
113
|
...
|
|
114
114
|
----
|
|
115
115
|
|
|
116
|
+
=== Create bibliographic item from YAML
|
|
117
|
+
[source,ruby]
|
|
118
|
+
----
|
|
119
|
+
hash = YAML.load_file 'spec/examples/nist_bib_item.yml'
|
|
120
|
+
=> {"id"=>"NISTIR 8011 Vol. 3",
|
|
121
|
+
...
|
|
122
|
+
|
|
123
|
+
bib_hash = RelatonNist::HashConverter.hash_to_bib hash
|
|
124
|
+
=> {:id=>"NISTIR 8011 Vol. 3",
|
|
125
|
+
...
|
|
126
|
+
|
|
127
|
+
RelatonNist::NistBibliographicItem.new bib_hash
|
|
128
|
+
=> #<RelatonNist::NistBibliographicItem:0x007f8b708505b8
|
|
129
|
+
...
|
|
130
|
+
----
|
|
131
|
+
|
|
116
132
|
== Development
|
|
117
133
|
|
|
118
134
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/lib/relaton_nist.rb
CHANGED
|
@@ -1,27 +1,11 @@
|
|
|
1
1
|
require "relaton_nist/version"
|
|
2
2
|
require "relaton_nist/nist_bibliography"
|
|
3
3
|
|
|
4
|
-
if defined? Relaton
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
end
|
|
4
|
+
# if defined? Relaton
|
|
5
|
+
# require_relative "relaton/processor"
|
|
6
|
+
# Relaton::Registry.instance.register(Relaton::RelatonNist::Processor)
|
|
7
|
+
# end
|
|
8
8
|
|
|
9
9
|
module RelatonNist
|
|
10
10
|
class Error < StandardError; end
|
|
11
|
-
|
|
12
|
-
class << self
|
|
13
|
-
# @param date [String]
|
|
14
|
-
# @return [Date, NilClass]
|
|
15
|
-
def parse_date(sdate)
|
|
16
|
-
if /(?<date>\w+\s\d{4})/ =~ sdate # February 2012
|
|
17
|
-
Date.strptime(date, "%B %Y")
|
|
18
|
-
elsif /(?<date>\w+\s\d{1,2},\s\d{4})/ =~ sdate # February 11, 2012
|
|
19
|
-
Date.strptime(date, "%B %d, %Y")
|
|
20
|
-
elsif /(?<date>\d{4}-\d{2}-\d{2})/ =~ sdate # 2012-02-11
|
|
21
|
-
Date.parse(date)
|
|
22
|
-
elsif /(?<date>\d{4}-\d{2})/ =~ sdate # 2012-02
|
|
23
|
-
Date.strptime date, "%Y-%m"
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
11
|
end
|
|
@@ -12,7 +12,7 @@ module RelatonNist
|
|
|
12
12
|
# @param from [Date]
|
|
13
13
|
# @param to [Date, NilClass]
|
|
14
14
|
# @param extended [Date, NilClass]
|
|
15
|
-
def initialize(from
|
|
15
|
+
def initialize(from:, to: nil, extended: nil)
|
|
16
16
|
@from = from
|
|
17
17
|
@to = to
|
|
18
18
|
@extended = extended
|
|
@@ -26,5 +26,13 @@ module RelatonNist
|
|
|
26
26
|
builder.extended extended.to_s if extended
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
|
+
|
|
30
|
+
# @return [Hash]
|
|
31
|
+
def to_hash
|
|
32
|
+
hash = { "from" => from.to_s }
|
|
33
|
+
hash["to"] = to.to_s if to
|
|
34
|
+
hash["extended"] = extended.to_s if extended
|
|
35
|
+
hash
|
|
36
|
+
end
|
|
29
37
|
end
|
|
30
38
|
end
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
module RelatonNist
|
|
2
|
+
class HashConverter < RelatonBib::HashConverter
|
|
3
|
+
class << self
|
|
4
|
+
# @override RelatonBib::HashConverter.hash_to_bib
|
|
5
|
+
# @param args [Hash]
|
|
6
|
+
# @param nested [TrueClass, FalseClass]
|
|
7
|
+
# @return [Hash]
|
|
8
|
+
def hash_to_bib(args, nested = false)
|
|
9
|
+
ret = super
|
|
10
|
+
return if ret.nil?
|
|
11
|
+
|
|
12
|
+
keyword_hash_to_bib(ret)
|
|
13
|
+
commentperiod_hash_to_bib(ret)
|
|
14
|
+
ret
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def keyword_hash_to_bib(ret)
|
|
20
|
+
ret[:keyword]&.map! { |kw| Keyword.new kw }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def commentperiod_hash_to_bib(ret)
|
|
24
|
+
return unless ret[:commentperiod]
|
|
25
|
+
|
|
26
|
+
ret[:commentperiod] = CommentPeriod.new ret[:commentperiod]
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
data/lib/relaton_nist/hit.rb
CHANGED
|
@@ -2,45 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
module RelatonNist
|
|
4
4
|
# Hit.
|
|
5
|
-
class Hit
|
|
5
|
+
class Hit < RelatonBib::Hit
|
|
6
6
|
# @return [RelatonNist::HitCollection]
|
|
7
7
|
attr_reader :hit_collection
|
|
8
8
|
|
|
9
|
-
# @return [Array<Hash>]
|
|
10
|
-
attr_reader :hit
|
|
11
|
-
|
|
12
|
-
# @param hit [Hash]
|
|
13
|
-
# @param hit_collection [RelatonNist:HitCollection]
|
|
14
|
-
def initialize(hit, hit_collection = nil)
|
|
15
|
-
@hit = hit
|
|
16
|
-
@hit_collection = hit_collection
|
|
17
|
-
end
|
|
18
|
-
|
|
19
9
|
# Parse page.
|
|
20
10
|
# @return [RelatonNist::NistBliographicItem]
|
|
21
11
|
def fetch
|
|
22
12
|
@fetch ||= Scrapper.parse_page @hit
|
|
23
13
|
end
|
|
24
14
|
|
|
25
|
-
# @return [String]
|
|
26
|
-
def to_s
|
|
27
|
-
inspect
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
# @return [String]
|
|
31
|
-
def inspect
|
|
32
|
-
"<#{self.class}:#{format('%#.14x', object_id << 1)} "\
|
|
33
|
-
"@text=\"#{@hit_collection&.text}\" "\
|
|
34
|
-
"@fetched=\"#{!@fetch.nil?}\" "\
|
|
35
|
-
"@fullIdentifier=\"#{@fetch&.shortref(nil)}\" "\
|
|
36
|
-
"@title=\"#{@hit[:code]}\">"
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
# @return [String]
|
|
40
|
-
def to_xml(**opts)
|
|
41
|
-
fetch.to_xml **opts
|
|
42
|
-
end
|
|
43
|
-
|
|
44
15
|
# @return [Iteger]
|
|
45
16
|
def sort_value
|
|
46
17
|
@sort_value ||= begin
|
|
@@ -8,19 +8,10 @@ require "open-uri"
|
|
|
8
8
|
|
|
9
9
|
module RelatonNist
|
|
10
10
|
# Page of hit collection.
|
|
11
|
-
class HitCollection <
|
|
11
|
+
class HitCollection < RelatonBib::HitCollection
|
|
12
12
|
DOMAIN = "https://csrc.nist.gov"
|
|
13
13
|
DATAFILE = File.expand_path "data/pubs-export.zip", __dir__
|
|
14
14
|
|
|
15
|
-
# @return [TrueClass, FalseClass]
|
|
16
|
-
attr_reader :fetched
|
|
17
|
-
|
|
18
|
-
# @return [String]
|
|
19
|
-
attr_reader :text
|
|
20
|
-
|
|
21
|
-
# @return [String]
|
|
22
|
-
attr_reader :year
|
|
23
|
-
|
|
24
15
|
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
25
16
|
|
|
26
17
|
# @param ref_nbr [String]
|
|
@@ -46,28 +37,6 @@ module RelatonNist
|
|
|
46
37
|
end
|
|
47
38
|
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
48
39
|
|
|
49
|
-
# @return [Iecbib::HitCollection]
|
|
50
|
-
def fetch
|
|
51
|
-
workers = RelatonBib::WorkersPool.new 4
|
|
52
|
-
workers.worker(&:fetch)
|
|
53
|
-
each do |hit|
|
|
54
|
-
workers << hit
|
|
55
|
-
end
|
|
56
|
-
workers.end
|
|
57
|
-
workers.result
|
|
58
|
-
@fetched = true
|
|
59
|
-
self
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
def to_s
|
|
63
|
-
inspect
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
# @return [String]
|
|
67
|
-
def inspect
|
|
68
|
-
"<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
|
|
69
|
-
end
|
|
70
|
-
|
|
71
40
|
private
|
|
72
41
|
|
|
73
42
|
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
@@ -107,49 +76,78 @@ module RelatonNist
|
|
|
107
76
|
)
|
|
108
77
|
end
|
|
109
78
|
end
|
|
79
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
110
80
|
|
|
111
81
|
# Fetches data form json
|
|
112
82
|
# @param docid [String]
|
|
83
|
+
# @param stage [String]
|
|
84
|
+
# @return [Array<RelatonNist::Hit>]
|
|
113
85
|
def from_json(docid, **opts)
|
|
86
|
+
select_data(docid, **opts).map do |h|
|
|
87
|
+
/(?<serie>(?<=-)\w+$)/ =~ h["series"]
|
|
88
|
+
title = [h["title-main"], h["title-sub"]].compact.join " - "
|
|
89
|
+
release_date = RelatonBib.parse_date h["published-date"]
|
|
90
|
+
Hit.new({ code: h["docidentifier"], serie: serie.upcase, title: title,
|
|
91
|
+
url: h["uri"], status: h["status"],
|
|
92
|
+
release_date: release_date, json: h }, self)
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# @param docid [String]
|
|
97
|
+
# @param stage [String]
|
|
98
|
+
# @return [Array<Hach>]
|
|
99
|
+
def select_data(docid, **opts)
|
|
100
|
+
d = Date.strptime year, "%Y" if year
|
|
114
101
|
data.select do |doc|
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
idate = RelatonNist.parse_date doc["issued-date"]
|
|
118
|
-
next unless idate.between? d, d.next_year.prev_day
|
|
119
|
-
end
|
|
102
|
+
next unless match_year?(doc, d)
|
|
103
|
+
|
|
120
104
|
if /PD/ =~ opts[:stage]
|
|
121
105
|
next unless %w[draft-public draft-prelim].include? doc["status"]
|
|
122
106
|
else
|
|
123
107
|
next unless doc["status"] == "final"
|
|
124
108
|
end
|
|
125
109
|
doc["docidentifier"] =~ Regexp.new(docid)
|
|
126
|
-
end.map do |h|
|
|
127
|
-
/(?<serie>(?<=-)\w+$)/ =~ h["series"]
|
|
128
|
-
title = [h["title-main"], h["title-sub"]].compact.join " - "
|
|
129
|
-
release_date = RelatonNist.parse_date h["published-date"]
|
|
130
|
-
Hit.new(
|
|
131
|
-
{
|
|
132
|
-
code: h["docidentifier"], serie: serie.upcase, title: title,
|
|
133
|
-
url: h["uri"], status: h["status"], release_date: release_date,
|
|
134
|
-
json: h
|
|
135
|
-
}, self
|
|
136
|
-
)
|
|
137
110
|
end
|
|
138
111
|
end
|
|
139
112
|
|
|
140
|
-
#
|
|
113
|
+
# @param doc [Hash]
|
|
114
|
+
# @param date [Date] first day of year
|
|
115
|
+
# @return [TrueClass, FalseClass]
|
|
116
|
+
def match_year?(doc, date)
|
|
117
|
+
return true unless year
|
|
118
|
+
|
|
119
|
+
idate = RelatonBib.parse_date doc["issued-date"]
|
|
120
|
+
idate.between? date, date.next_year.prev_day
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Fetches json data form server
|
|
141
124
|
# @return [Hash]
|
|
142
125
|
def data
|
|
143
126
|
ctime = File.ctime DATAFILE if File.exist? DATAFILE
|
|
144
127
|
if !ctime || ctime.to_date < Date.today
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
128
|
+
fetch_data(ctime)
|
|
129
|
+
end
|
|
130
|
+
unzip
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Fetch data form server and save it to file
|
|
134
|
+
#
|
|
135
|
+
# @prarm ctime [Time, NilClass]
|
|
136
|
+
def fetch_data(ctime)
|
|
137
|
+
resp = OpenURI.open_uri("https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.meta")
|
|
138
|
+
if !ctime || ctime < resp.last_modified
|
|
139
|
+
@data = nil
|
|
140
|
+
zip = OpenURI.open_uri "https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.zip"
|
|
141
|
+
zip.close
|
|
142
|
+
FileUtils.mv zip.path, DATAFILE
|
|
151
143
|
end
|
|
152
|
-
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# upack zip file
|
|
147
|
+
#
|
|
148
|
+
# @return [Hash]
|
|
149
|
+
def unzip
|
|
150
|
+
return @data if @data
|
|
153
151
|
|
|
154
152
|
Zip::File.open(DATAFILE) do |zf|
|
|
155
153
|
zf.each do |f|
|
|
@@ -159,6 +157,5 @@ module RelatonNist
|
|
|
159
157
|
end
|
|
160
158
|
@data
|
|
161
159
|
end
|
|
162
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
|
163
160
|
end
|
|
164
161
|
end
|
data/lib/relaton_nist/keyword.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
module RelatonNist
|
|
2
2
|
class Keyword
|
|
3
|
+
include RelatonBib
|
|
4
|
+
|
|
3
5
|
# @return [Nokogiri::XML::DocumentFragment]
|
|
4
6
|
attr_reader :element
|
|
5
7
|
|
|
@@ -12,5 +14,10 @@ module RelatonNist
|
|
|
12
14
|
def to_xml(builder)
|
|
13
15
|
builder.keyword element.to_xml
|
|
14
16
|
end
|
|
17
|
+
|
|
18
|
+
# @return [String]
|
|
19
|
+
def to_hash
|
|
20
|
+
element.text
|
|
21
|
+
end
|
|
15
22
|
end
|
|
16
23
|
end
|
|
@@ -6,7 +6,7 @@ module RelatonNist
|
|
|
6
6
|
# @return [Array<RelatonNist::Keyword>]
|
|
7
7
|
attr_reader :keyword
|
|
8
8
|
|
|
9
|
-
# @return [RelatonNist::CommentPeriod]
|
|
9
|
+
# @return [RelatonNist::CommentPeriod, NilClass]
|
|
10
10
|
attr_reader :commentperiod
|
|
11
11
|
|
|
12
12
|
# @param id [String, NilClass]
|
|
@@ -31,19 +31,19 @@ module RelatonNist
|
|
|
31
31
|
# @param fetched [Date, NilClass] default nil
|
|
32
32
|
# @param doctype [String]
|
|
33
33
|
# @param keyword [Array<RelatonNist::Keyword>]
|
|
34
|
-
# @param commentperiod [RelatonNist::CommentPeriod]
|
|
34
|
+
# @param commentperiod [RelatonNist::CommentPeriod, NilClass]
|
|
35
35
|
#
|
|
36
|
-
# @param
|
|
37
|
-
# @option
|
|
38
|
-
# @option
|
|
39
|
-
# @option
|
|
36
|
+
# @param date [Array<Hash>]
|
|
37
|
+
# @option date [String] :type
|
|
38
|
+
# @option date [String] :from
|
|
39
|
+
# @option date [String] :to
|
|
40
40
|
#
|
|
41
|
-
# @param
|
|
42
|
-
# @option
|
|
43
|
-
# @option
|
|
44
|
-
# @option
|
|
45
|
-
# @option
|
|
46
|
-
# @option
|
|
41
|
+
# @param contributor [Array<Hash>]
|
|
42
|
+
# @option contributor [String] :type
|
|
43
|
+
# @option contributor [String] :from
|
|
44
|
+
# @option contributor [String] :to
|
|
45
|
+
# @option contributor [String] :abbreviation
|
|
46
|
+
# @option contributor [Array<String>] :role
|
|
47
47
|
#
|
|
48
48
|
# @param abstract [Array<Hash>]
|
|
49
49
|
# @option abstract [String] :content
|
|
@@ -51,10 +51,10 @@ module RelatonNist
|
|
|
51
51
|
# @option abstract [String] :script
|
|
52
52
|
# @option abstract [String] :type
|
|
53
53
|
#
|
|
54
|
-
# @param
|
|
55
|
-
# @option
|
|
56
|
-
# @option
|
|
57
|
-
# @option
|
|
54
|
+
# @param relation [Array<Hash>]
|
|
55
|
+
# @option relation [String] :type
|
|
56
|
+
# @option relation [RelatonBib::BibliographicItem] :bibitem
|
|
57
|
+
# @option relation [Array<RelatonBib::BibItemLocality>] :bib_locality
|
|
58
58
|
def initialize(**args)
|
|
59
59
|
@doctype = args.delete(:doctype) || "standard"
|
|
60
60
|
@keyword = args.delete(:keyword) || []
|
|
@@ -74,5 +74,13 @@ module RelatonNist
|
|
|
74
74
|
end
|
|
75
75
|
end
|
|
76
76
|
end
|
|
77
|
+
|
|
78
|
+
# @return [Hash]
|
|
79
|
+
def to_hash
|
|
80
|
+
hash = super
|
|
81
|
+
hash["keyword"] = single_element_array(keyword) if keyword&.any?
|
|
82
|
+
hash["commentperiod"] = commentperiod.to_hash if commentperiod
|
|
83
|
+
hash
|
|
84
|
+
end
|
|
77
85
|
end
|
|
78
86
|
end
|
|
@@ -6,6 +6,7 @@ require "relaton_nist/xml_parser"
|
|
|
6
6
|
require "relaton_nist/keyword"
|
|
7
7
|
require "relaton_nist/comment_period"
|
|
8
8
|
require "relaton_nist/document_status"
|
|
9
|
+
require "relaton_nist/hash_converter"
|
|
9
10
|
|
|
10
11
|
module RelatonNist
|
|
11
12
|
class NistBibliography
|
|
@@ -14,7 +15,7 @@ module RelatonNist
|
|
|
14
15
|
# @return [RelatonNist::HitCollection]
|
|
15
16
|
def search(text, year = nil, opts = {})
|
|
16
17
|
HitCollection.new text, year, opts
|
|
17
|
-
rescue OpenURI::HTTPError, SocketError
|
|
18
|
+
rescue OpenURI::HTTPError, SocketError, OpenSSL::SSL::SSLError
|
|
18
19
|
raise RelatonBib::RequestError, "Could not access https://www.nist.gov"
|
|
19
20
|
end
|
|
20
21
|
|
|
@@ -33,9 +34,9 @@ module RelatonNist
|
|
|
33
34
|
code = code2.strip
|
|
34
35
|
if date2
|
|
35
36
|
if /\w+\s\d{4}/ =~ date2
|
|
36
|
-
opts[:issued_date] =
|
|
37
|
+
opts[:issued_date] = Date.strptime date2, "%B %Y"
|
|
37
38
|
elsif /\w+\s\d{2},\s\d{4}/ =~ date2
|
|
38
|
-
opts[:updated_date] =
|
|
39
|
+
opts[:updated_date] = Date.strptime date2, "%B %d, %Y"
|
|
39
40
|
end
|
|
40
41
|
end
|
|
41
42
|
opts[:stage] = stage if stage
|
|
@@ -50,11 +51,7 @@ module RelatonNist
|
|
|
50
51
|
end
|
|
51
52
|
|
|
52
53
|
code += "-1" if opts[:all_parts]
|
|
53
|
-
|
|
54
|
-
# return nil if ret.nil?
|
|
55
|
-
# ret.to_most_recent_reference unless year || opts[:keep_year]
|
|
56
|
-
# ret.to_all_parts if opts[:all_parts]
|
|
57
|
-
ret
|
|
54
|
+
nistbib_get1(code, year, opts)
|
|
58
55
|
end
|
|
59
56
|
|
|
60
57
|
private
|
|
@@ -91,16 +88,16 @@ module RelatonNist
|
|
|
91
88
|
result.each_slice(3) do |s| # ISO website only allows 3 connections
|
|
92
89
|
fetch_pages(s, 3).each_with_index do |r, _i|
|
|
93
90
|
if opts[:issued_date]
|
|
94
|
-
ids = r.
|
|
91
|
+
ids = r.date.select { |d| d.type == "issued" && d.on == opts[:issued_date] }
|
|
95
92
|
next if ids.empty?
|
|
96
93
|
elsif opts[:updated_date]
|
|
97
|
-
pds = r.
|
|
94
|
+
pds = r.date.select { |d| d.type == "published" && d.on == opts[:updated_date] }
|
|
98
95
|
next if pds.empty?
|
|
99
96
|
end
|
|
100
97
|
next if iter && r.status.iteration != iteration
|
|
101
98
|
return { ret: r } if !year
|
|
102
99
|
|
|
103
|
-
r.
|
|
100
|
+
r.date.select { |d| d.type == "published" }.each do |d|
|
|
104
101
|
return { ret: r } if year.to_i == d.on.year
|
|
105
102
|
|
|
106
103
|
missed_years << d.on.year
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
require "relaton/processor"
|
|
2
|
+
|
|
3
|
+
module RelatonNist
|
|
4
|
+
class Processor < Relaton::Processor
|
|
5
|
+
def initialize
|
|
6
|
+
@short = :relaton_nist
|
|
7
|
+
@prefix = "NIST"
|
|
8
|
+
@defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC)[ /]}
|
|
9
|
+
@idtype = "NIST"
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# @param code [String]
|
|
13
|
+
# @param date [String, NilClass] year
|
|
14
|
+
# @param opts [Hash]
|
|
15
|
+
# @return [RelatonNist::GbBibliographicItem]
|
|
16
|
+
def get(code, date = nil, opts = {})
|
|
17
|
+
::RelatonNist::NistBibliography.get(code, date, opts)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# @param xml [String]
|
|
21
|
+
# @return [RelatonNist::GbBibliographicItem]
|
|
22
|
+
def from_xml(xml)
|
|
23
|
+
::RelatonNist::XMLParser.from_xml xml
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# @param hash [Hash]
|
|
27
|
+
# @return [RelatonNist::GbBibliographicItem]
|
|
28
|
+
def hash_to_bib(hash)
|
|
29
|
+
item_hash = ::RelatonNist::HashConverter.hash_to_bib(hash)
|
|
30
|
+
::RelatonNist::NistBibliographicItem.new item_hash
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -26,7 +26,7 @@ module RelatonNist
|
|
|
26
26
|
end
|
|
27
27
|
item_data[:fetched] = Date.today.to_s
|
|
28
28
|
item_data[:type] = "standard"
|
|
29
|
-
item_data[:
|
|
29
|
+
item_data[:title] = titles
|
|
30
30
|
item_data[:doctype] = doctype
|
|
31
31
|
|
|
32
32
|
NistBibliographicItem.new(**item_data)
|
|
@@ -39,16 +39,14 @@ module RelatonNist
|
|
|
39
39
|
{
|
|
40
40
|
link: fetch_link(json),
|
|
41
41
|
docid: fetch_docid(json["docidentifier"]),
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
date: fetch_dates(json, hit_data[:release_date]),
|
|
43
|
+
contributor: fetch_contributors(json),
|
|
44
44
|
edition: fetch_edition(json),
|
|
45
45
|
language: [json["language"]],
|
|
46
46
|
script: [json["script"]],
|
|
47
|
-
# abstract: fetch_abstract(doc),
|
|
48
47
|
docstatus: fetch_status(json, hit_data[:status]),
|
|
49
48
|
copyright: fetch_copyright(json["published-date"]),
|
|
50
|
-
|
|
51
|
-
# series: fetch_series(json),
|
|
49
|
+
relation: fetch_relations_json(json),
|
|
52
50
|
keyword: fetch_keywords(json),
|
|
53
51
|
commentperiod: fetch_commentperiod_json(json),
|
|
54
52
|
}
|
|
@@ -60,15 +58,15 @@ module RelatonNist
|
|
|
60
58
|
# id: fetch_id(doc),
|
|
61
59
|
link: fetch_link(doc),
|
|
62
60
|
docid: fetch_docid(doc),
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
date: fetch_dates(doc, hit_data[:release_date]),
|
|
62
|
+
contributor: fetch_contributors(doc),
|
|
65
63
|
edition: fetch_edition(hit_data[:code]),
|
|
66
64
|
language: ["en"],
|
|
67
65
|
script: ["Latn"],
|
|
68
66
|
abstract: fetch_abstract(doc),
|
|
69
67
|
docstatus: fetch_status(doc, hit_data[:status]),
|
|
70
68
|
copyright: fetch_copyright(doc),
|
|
71
|
-
|
|
69
|
+
relation: fetch_relations(doc),
|
|
72
70
|
series: fetch_series(doc),
|
|
73
71
|
keyword: fetch_keywords(doc),
|
|
74
72
|
commentperiod: fetch_commentperiod(doc),
|
|
@@ -91,7 +89,8 @@ module RelatonNist
|
|
|
91
89
|
resp = Net::HTTP.get_response(uri) # .encode("UTF-8")
|
|
92
90
|
Nokogiri::HTML(resp.body)
|
|
93
91
|
rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
|
|
94
|
-
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError
|
|
92
|
+
Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
|
|
93
|
+
OpenSSL::SSL::SSLError
|
|
95
94
|
raise RelatonBib::RequestError, "Could not access #{url}"
|
|
96
95
|
end
|
|
97
96
|
|
|
@@ -178,14 +177,14 @@ module RelatonNist
|
|
|
178
177
|
dates = [{ type: "published", on: release_date.to_s }]
|
|
179
178
|
|
|
180
179
|
if doc.is_a? Hash
|
|
181
|
-
issued =
|
|
182
|
-
updated =
|
|
180
|
+
issued = RelatonBib.parse_date doc["issued-date"]
|
|
181
|
+
updated = RelatonBib.parse_date doc["updated-date"]
|
|
183
182
|
dates << { type: "updated", on: updated.to_s } if updated
|
|
184
|
-
obsoleted =
|
|
183
|
+
obsoleted = RelatonBib.parse_date doc["obsoleted-date"]
|
|
185
184
|
dates << { type: "obsoleted", on: obsoleted.to_s } if obsoleted
|
|
186
185
|
else
|
|
187
186
|
d = doc.at("//span[@id='pub-release-date']").text.strip
|
|
188
|
-
issued =
|
|
187
|
+
issued = RelatonBib.parse_date d
|
|
189
188
|
end
|
|
190
189
|
dates << { type: "issued", on: issued.to_s }
|
|
191
190
|
dates
|
|
@@ -208,7 +207,7 @@ module RelatonNist
|
|
|
208
207
|
org = RelatonBib::Organization.new(
|
|
209
208
|
name: name, url: "www.nist.gov", abbreviation: "NIST",
|
|
210
209
|
)
|
|
211
|
-
contribs << RelatonBib::ContributionInfo.new(entity: org, role: ["publisher"])
|
|
210
|
+
contribs << RelatonBib::ContributionInfo.new(entity: org, role: [type: "publisher"])
|
|
212
211
|
authors = doc.at('//h4[.="Author(s)"]/following-sibling::p')
|
|
213
212
|
contribs += contributors(authors, "author")
|
|
214
213
|
editors = doc.at('//h4[.="Editor(s)"]/following-sibling::p')
|
|
@@ -230,14 +229,14 @@ module RelatonNist
|
|
|
230
229
|
)
|
|
231
230
|
end
|
|
232
231
|
if contr["surname"]
|
|
233
|
-
affiliation = RelatonBib::Affilation.new org
|
|
232
|
+
affiliation = RelatonBib::Affilation.new organization: org
|
|
234
233
|
entity = RelatonBib::Person.new(
|
|
235
234
|
name: full_name(contr, lang, script), affiliation: [affiliation],
|
|
236
235
|
)
|
|
237
236
|
else
|
|
238
237
|
entity = org
|
|
239
238
|
end
|
|
240
|
-
RelatonBib::ContributionInfo.new entity: entity, role: [role]
|
|
239
|
+
RelatonBib::ContributionInfo.new entity: entity, role: [type: role]
|
|
241
240
|
end
|
|
242
241
|
end
|
|
243
242
|
|
|
@@ -266,14 +265,14 @@ module RelatonNist
|
|
|
266
265
|
url = nil
|
|
267
266
|
end
|
|
268
267
|
org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
|
|
269
|
-
affiliation = RelatonBib::Affilation.new org
|
|
268
|
+
affiliation = RelatonBib::Affilation.new organization: org
|
|
270
269
|
entity = RelatonBib::Person.new(
|
|
271
270
|
name: fullname, affiliation: [affiliation],
|
|
272
271
|
)
|
|
273
272
|
else
|
|
274
273
|
entity = RelatonBib::Organization.new name: an, abbreviation: abbrev
|
|
275
274
|
end
|
|
276
|
-
RelatonBib::ContributionInfo.new entity: entity, role: [role]
|
|
275
|
+
RelatonBib::ContributionInfo.new entity: entity, role: [type: role]
|
|
277
276
|
end
|
|
278
277
|
end
|
|
279
278
|
# rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize, Metrics/MethodLength
|
|
@@ -285,8 +284,8 @@ module RelatonNist
|
|
|
285
284
|
def full_name(name, lang, script)
|
|
286
285
|
RelatonBib::FullName.new(
|
|
287
286
|
surname: RelatonBib::LocalizedString.new(name["surname"], lang, script),
|
|
288
|
-
|
|
289
|
-
|
|
287
|
+
forename: name_parts(name["givenName"], lang, script),
|
|
288
|
+
addition: name_parts(name["suffix"], lang, script),
|
|
290
289
|
prefix: name_parts(name["title"], lang, script),
|
|
291
290
|
completename: RelatonBib::LocalizedString.new(name["fullName"], lang, script),
|
|
292
291
|
)
|
|
@@ -455,13 +454,15 @@ module RelatonNist
|
|
|
455
454
|
ex = doc.at "//strong[contains(.,'The comment closing date has been extended to')]"
|
|
456
455
|
ext = ex&.text&.match(/\w+\s\d{2},\s\d{4}/).to_s
|
|
457
456
|
extended = ext.empty? ? nil : Date.strptime(ext, "%B %d, %Y")
|
|
458
|
-
CommentPeriod.new from, to, extended
|
|
457
|
+
CommentPeriod.new from: from, to: to, extended: extended
|
|
459
458
|
end
|
|
460
459
|
|
|
461
460
|
# @param json [Hash]
|
|
462
461
|
# @return [RelatonNist::CommentPeriod, NilClass]
|
|
463
462
|
def fetch_commentperiod_json(json)
|
|
464
|
-
|
|
463
|
+
return unless json["comment-from"]
|
|
464
|
+
|
|
465
|
+
CommentPeriod.new from: json["comment-from"], to: json["comment-to"]
|
|
465
466
|
end
|
|
466
467
|
end
|
|
467
468
|
end
|
data/lib/relaton_nist/version.rb
CHANGED
|
@@ -34,7 +34,10 @@ module RelatonNist
|
|
|
34
34
|
cp = item.at "./commentperiod"
|
|
35
35
|
return unless cp
|
|
36
36
|
|
|
37
|
-
CommentPeriod.new
|
|
37
|
+
CommentPeriod.new(
|
|
38
|
+
from: cp.at("from").text, to: cp.at("to")&.text,
|
|
39
|
+
extended: cp.at("extended")&.text
|
|
40
|
+
)
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def fetch_keyword(item)
|
data/relaton_nist.gemspec
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: relaton-nist
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ribose Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2019-
|
|
11
|
+
date: 2019-09-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -170,14 +170,14 @@ dependencies:
|
|
|
170
170
|
requirements:
|
|
171
171
|
- - "~>"
|
|
172
172
|
- !ruby/object:Gem::Version
|
|
173
|
-
version: 0.
|
|
173
|
+
version: 0.3.0
|
|
174
174
|
type: :runtime
|
|
175
175
|
prerelease: false
|
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
|
177
177
|
requirements:
|
|
178
178
|
- - "~>"
|
|
179
179
|
- !ruby/object:Gem::Version
|
|
180
|
-
version: 0.
|
|
180
|
+
version: 0.3.0
|
|
181
181
|
- !ruby/object:Gem::Dependency
|
|
182
182
|
name: rubyzip
|
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -211,16 +211,17 @@ files:
|
|
|
211
211
|
- appveyor.yml
|
|
212
212
|
- bin/console
|
|
213
213
|
- bin/setup
|
|
214
|
-
- lib/relaton/processor.rb
|
|
215
214
|
- lib/relaton_nist.rb
|
|
216
215
|
- lib/relaton_nist/comment_period.rb
|
|
217
216
|
- lib/relaton_nist/data/pubs-export.zip
|
|
218
217
|
- lib/relaton_nist/document_status.rb
|
|
218
|
+
- lib/relaton_nist/hash_converter.rb
|
|
219
219
|
- lib/relaton_nist/hit.rb
|
|
220
220
|
- lib/relaton_nist/hit_collection.rb
|
|
221
221
|
- lib/relaton_nist/keyword.rb
|
|
222
222
|
- lib/relaton_nist/nist_bibliographic_item.rb
|
|
223
223
|
- lib/relaton_nist/nist_bibliography.rb
|
|
224
|
+
- lib/relaton_nist/processor.rb
|
|
224
225
|
- lib/relaton_nist/scrapper.rb
|
|
225
226
|
- lib/relaton_nist/version.rb
|
|
226
227
|
- lib/relaton_nist/xml_parser.rb
|
data/lib/relaton/processor.rb
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
require "relaton/processor"
|
|
2
|
-
|
|
3
|
-
module Relaton
|
|
4
|
-
module RelatonNist
|
|
5
|
-
class Processor < Relaton::Processor
|
|
6
|
-
|
|
7
|
-
def initialize
|
|
8
|
-
@short = :relaton_nist
|
|
9
|
-
@prefix = "NIST"
|
|
10
|
-
@defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC)[ /]}
|
|
11
|
-
@idtype = "NIST"
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
def get(code, date = nil, opts = {})
|
|
15
|
-
::RelatonNist::NistBibliography.get(code, date, opts)
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
def from_xml(xml)
|
|
19
|
-
::RelatonNist::XMLParser.from_xml xml
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
end
|