relaton-nist 0.3.3 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/Gemfile.lock +7 -7
- data/README.adoc +1 -1
- data/lib/relaton_nist.rb +4 -20
- data/lib/relaton_nist/hit.rb +1 -30
- data/lib/relaton_nist/hit_collection.rb +57 -59
- data/lib/relaton_nist/nist_bibliography.rb +1 -5
- data/lib/relaton_nist/processor.rb +33 -0
- data/lib/relaton_nist/scrapper.rb +5 -7
- data/lib/relaton_nist/version.rb +1 -1
- metadata +4 -6
- data/lib/relaton/processor.rb +0 -23
- data/lib/relaton_nist/data/pubs-export.zip +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: a9aa2e893b8a1b9004089a2a51daf8bec520c9da94092eb159d60e8eb5f38e29
|
4
|
+
data.tar.gz: 1e52d069f7c446704f190fa48594529344df75cc07050a7c050e53c0a39bbaba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2df94a8d67d12ecf5027361c13a866acda3b8d5f276ef52b5b85b768d7621d23b25bcc61bed190d8ee405d61dad08bda6b40d67c54a8f09df264b3eb5f0a9f94
|
7
|
+
data.tar.gz: 48f57b587993181848688807beba9ec0117c088619f9896e55067004437a40db43d80e708b69a5232da0da54f05e1a358803d4e20887feb0e2c2ecee82d87238
|
data/Gemfile.lock
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
relaton-nist (0.3.
|
4
|
+
relaton-nist (0.3.8)
|
5
5
|
relaton-bib (~> 0.3.0)
|
6
6
|
rubyzip
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
|
-
addressable (2.
|
12
|
-
public_suffix (>= 2.0.2, <
|
11
|
+
addressable (2.7.0)
|
12
|
+
public_suffix (>= 2.0.2, < 5.0)
|
13
13
|
byebug (11.0.1)
|
14
14
|
coderay (1.1.2)
|
15
15
|
crack (0.4.3)
|
@@ -33,11 +33,11 @@ GEM
|
|
33
33
|
pry-byebug (3.7.0)
|
34
34
|
byebug (~> 11.0)
|
35
35
|
pry (~> 0.10)
|
36
|
-
public_suffix (
|
36
|
+
public_suffix (4.0.1)
|
37
37
|
rake (10.5.0)
|
38
|
-
relaton-bib (0.3.
|
38
|
+
relaton-bib (0.3.12)
|
39
39
|
addressable
|
40
|
-
nokogiri
|
40
|
+
nokogiri
|
41
41
|
rspec (3.8.0)
|
42
42
|
rspec-core (~> 3.8.0)
|
43
43
|
rspec-expectations (~> 3.8.0)
|
@@ -53,7 +53,7 @@ GEM
|
|
53
53
|
rspec-support (3.8.2)
|
54
54
|
ruby-debug-ide (0.7.0)
|
55
55
|
rake (>= 0.8.1)
|
56
|
-
rubyzip (
|
56
|
+
rubyzip (2.0.0)
|
57
57
|
safe_yaml (1.0.5)
|
58
58
|
simplecov (0.16.1)
|
59
59
|
docile (~> 1.1)
|
data/README.adoc
CHANGED
data/lib/relaton_nist.rb
CHANGED
@@ -1,27 +1,11 @@
|
|
1
1
|
require "relaton_nist/version"
|
2
2
|
require "relaton_nist/nist_bibliography"
|
3
3
|
|
4
|
-
if defined? Relaton
|
5
|
-
|
6
|
-
|
7
|
-
end
|
4
|
+
# if defined? Relaton
|
5
|
+
# require_relative "relaton/processor"
|
6
|
+
# Relaton::Registry.instance.register(Relaton::RelatonNist::Processor)
|
7
|
+
# end
|
8
8
|
|
9
9
|
module RelatonNist
|
10
10
|
class Error < StandardError; end
|
11
|
-
|
12
|
-
class << self
|
13
|
-
# @param date [String]
|
14
|
-
# @return [Date, NilClass]
|
15
|
-
def parse_date(sdate)
|
16
|
-
if /(?<date>\w+\s\d{4})/ =~ sdate # February 2012
|
17
|
-
Date.strptime(date, "%B %Y")
|
18
|
-
elsif /(?<date>\w+\s\d{1,2},\s\d{4})/ =~ sdate # February 11, 2012
|
19
|
-
Date.strptime(date, "%B %d, %Y")
|
20
|
-
elsif /(?<date>\d{4}-\d{2}-\d{2})/ =~ sdate # 2012-02-11
|
21
|
-
Date.parse(date)
|
22
|
-
elsif /(?<date>\d{4}-\d{2})/ =~ sdate # 2012-02
|
23
|
-
Date.strptime date, "%Y-%m"
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
11
|
end
|
data/lib/relaton_nist/hit.rb
CHANGED
@@ -2,45 +2,16 @@
|
|
2
2
|
|
3
3
|
module RelatonNist
|
4
4
|
# Hit.
|
5
|
-
class Hit
|
5
|
+
class Hit < RelatonBib::Hit
|
6
6
|
# @return [RelatonNist::HitCollection]
|
7
7
|
attr_reader :hit_collection
|
8
8
|
|
9
|
-
# @return [Array<Hash>]
|
10
|
-
attr_reader :hit
|
11
|
-
|
12
|
-
# @param hit [Hash]
|
13
|
-
# @param hit_collection [RelatonNist:HitCollection]
|
14
|
-
def initialize(hit, hit_collection = nil)
|
15
|
-
@hit = hit
|
16
|
-
@hit_collection = hit_collection
|
17
|
-
end
|
18
|
-
|
19
9
|
# Parse page.
|
20
10
|
# @return [RelatonNist::NistBliographicItem]
|
21
11
|
def fetch
|
22
12
|
@fetch ||= Scrapper.parse_page @hit
|
23
13
|
end
|
24
14
|
|
25
|
-
# @return [String]
|
26
|
-
def to_s
|
27
|
-
inspect
|
28
|
-
end
|
29
|
-
|
30
|
-
# @return [String]
|
31
|
-
def inspect
|
32
|
-
"<#{self.class}:#{format('%#.14x', object_id << 1)} "\
|
33
|
-
"@text=\"#{@hit_collection&.text}\" "\
|
34
|
-
"@fetched=\"#{!@fetch.nil?}\" "\
|
35
|
-
"@fullIdentifier=\"#{@fetch&.shortref(nil)}\" "\
|
36
|
-
"@title=\"#{@hit[:code]}\">"
|
37
|
-
end
|
38
|
-
|
39
|
-
# @return [String]
|
40
|
-
def to_xml(**opts)
|
41
|
-
fetch.to_xml **opts
|
42
|
-
end
|
43
|
-
|
44
15
|
# @return [Iteger]
|
45
16
|
def sort_value
|
46
17
|
@sort_value ||= begin
|
@@ -8,18 +8,10 @@ require "open-uri"
|
|
8
8
|
|
9
9
|
module RelatonNist
|
10
10
|
# Page of hit collection.
|
11
|
-
class HitCollection <
|
11
|
+
class HitCollection < RelatonBib::HitCollection
|
12
12
|
DOMAIN = "https://csrc.nist.gov"
|
13
|
-
|
14
|
-
|
15
|
-
# @return [TrueClass, FalseClass]
|
16
|
-
attr_reader :fetched
|
17
|
-
|
18
|
-
# @return [String]
|
19
|
-
attr_reader :text
|
20
|
-
|
21
|
-
# @return [String]
|
22
|
-
attr_reader :year
|
13
|
+
DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
|
14
|
+
DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
|
23
15
|
|
24
16
|
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
25
17
|
|
@@ -46,28 +38,6 @@ module RelatonNist
|
|
46
38
|
end
|
47
39
|
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
48
40
|
|
49
|
-
# @return [Iecbib::HitCollection]
|
50
|
-
def fetch
|
51
|
-
workers = RelatonBib::WorkersPool.new 4
|
52
|
-
workers.worker(&:fetch)
|
53
|
-
each do |hit|
|
54
|
-
workers << hit
|
55
|
-
end
|
56
|
-
workers.end
|
57
|
-
workers.result
|
58
|
-
@fetched = true
|
59
|
-
self
|
60
|
-
end
|
61
|
-
|
62
|
-
def to_s
|
63
|
-
inspect
|
64
|
-
end
|
65
|
-
|
66
|
-
# @return [String]
|
67
|
-
def inspect
|
68
|
-
"<#{self.class}:#{format('%#.14x', object_id << 1)} @fetched=#{@fetched}>"
|
69
|
-
end
|
70
|
-
|
71
41
|
private
|
72
42
|
|
73
43
|
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
@@ -107,50 +77,79 @@ module RelatonNist
|
|
107
77
|
)
|
108
78
|
end
|
109
79
|
end
|
80
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
110
81
|
|
111
82
|
# Fetches data form json
|
112
83
|
# @param docid [String]
|
84
|
+
# @param stage [String]
|
85
|
+
# @return [Array<RelatonNist::Hit>]
|
113
86
|
def from_json(docid, **opts)
|
87
|
+
select_data(docid, **opts).map do |h|
|
88
|
+
/(?<serie>(?<=-)\w+$)/ =~ h["series"]
|
89
|
+
title = [h["title-main"], h["title-sub"]].compact.join " - "
|
90
|
+
release_date = RelatonBib.parse_date h["published-date"]
|
91
|
+
Hit.new({ code: h["docidentifier"], serie: serie.upcase, title: title,
|
92
|
+
url: h["uri"], status: h["status"],
|
93
|
+
release_date: release_date, json: h }, self)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
# @param docid [String]
|
98
|
+
# @param stage [String]
|
99
|
+
# @return [Array<Hach>]
|
100
|
+
def select_data(docid, **opts)
|
101
|
+
d = Date.strptime year, "%Y" if year
|
114
102
|
data.select do |doc|
|
115
|
-
|
116
|
-
|
117
|
-
idate = RelatonNist.parse_date doc["issued-date"]
|
118
|
-
next unless idate.between? d, d.next_year.prev_day
|
119
|
-
end
|
103
|
+
next unless match_year?(doc, d)
|
104
|
+
|
120
105
|
if /PD/ =~ opts[:stage]
|
121
106
|
next unless %w[draft-public draft-prelim].include? doc["status"]
|
122
107
|
else
|
123
108
|
next unless doc["status"] == "final"
|
124
109
|
end
|
125
110
|
doc["docidentifier"] =~ Regexp.new(docid)
|
126
|
-
end.map do |h|
|
127
|
-
/(?<serie>(?<=-)\w+$)/ =~ h["series"]
|
128
|
-
title = [h["title-main"], h["title-sub"]].compact.join " - "
|
129
|
-
release_date = RelatonNist.parse_date h["published-date"]
|
130
|
-
Hit.new(
|
131
|
-
{
|
132
|
-
code: h["docidentifier"], serie: serie.upcase, title: title,
|
133
|
-
url: h["uri"], status: h["status"], release_date: release_date,
|
134
|
-
json: h
|
135
|
-
}, self
|
136
|
-
)
|
137
111
|
end
|
138
112
|
end
|
139
113
|
|
140
|
-
#
|
114
|
+
# @param doc [Hash]
|
115
|
+
# @param date [Date] first day of year
|
116
|
+
# @return [TrueClass, FalseClass]
|
117
|
+
def match_year?(doc, date)
|
118
|
+
return true unless year
|
119
|
+
|
120
|
+
idate = RelatonBib.parse_date doc["issued-date"]
|
121
|
+
idate.between? date, date.next_year.prev_day
|
122
|
+
end
|
123
|
+
|
124
|
+
# Fetches json data form server
|
141
125
|
# @return [Hash]
|
142
126
|
def data
|
143
127
|
ctime = File.ctime DATAFILE if File.exist? DATAFILE
|
144
128
|
if !ctime || ctime.to_date < Date.today
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
129
|
+
fetch_data(ctime)
|
130
|
+
end
|
131
|
+
unzip
|
132
|
+
end
|
133
|
+
|
134
|
+
# Fetch data form server and save it to file
|
135
|
+
#
|
136
|
+
# @prarm ctime [Time, NilClass]
|
137
|
+
def fetch_data(ctime)
|
138
|
+
resp = OpenURI.open_uri("https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.meta")
|
139
|
+
if !ctime || ctime < resp.last_modified
|
140
|
+
@data = nil
|
141
|
+
zip = OpenURI.open_uri "https://csrc.nist.gov/CSRC/media/feeds/metanorma/pubs-export.zip"
|
142
|
+
zip.close
|
143
|
+
FileUtils.mkdir_p DATAFILEDIR unless Dir.exist? DATAFILEDIR
|
144
|
+
FileUtils.mv zip.path, DATAFILE
|
152
145
|
end
|
153
|
-
|
146
|
+
end
|
147
|
+
|
148
|
+
# upack zip file
|
149
|
+
#
|
150
|
+
# @return [Hash]
|
151
|
+
def unzip
|
152
|
+
return @data if @data
|
154
153
|
|
155
154
|
Zip::File.open(DATAFILE) do |zf|
|
156
155
|
zf.each do |f|
|
@@ -160,6 +159,5 @@ module RelatonNist
|
|
160
159
|
end
|
161
160
|
@data
|
162
161
|
end
|
163
|
-
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
164
162
|
end
|
165
163
|
end
|
@@ -51,11 +51,7 @@ module RelatonNist
|
|
51
51
|
end
|
52
52
|
|
53
53
|
code += "-1" if opts[:all_parts]
|
54
|
-
|
55
|
-
# return nil if ret.nil?
|
56
|
-
# ret.to_most_recent_reference unless year || opts[:keep_year]
|
57
|
-
# ret.to_all_parts if opts[:all_parts]
|
58
|
-
ret
|
54
|
+
nistbib_get1(code, year, opts)
|
59
55
|
end
|
60
56
|
|
61
57
|
private
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require "relaton/processor"
|
2
|
+
|
3
|
+
module RelatonNist
|
4
|
+
class Processor < Relaton::Processor
|
5
|
+
def initialize
|
6
|
+
@short = :relaton_nist
|
7
|
+
@prefix = "NIST"
|
8
|
+
@defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC)[ /]}
|
9
|
+
@idtype = "NIST"
|
10
|
+
end
|
11
|
+
|
12
|
+
# @param code [String]
|
13
|
+
# @param date [String, NilClass] year
|
14
|
+
# @param opts [Hash]
|
15
|
+
# @return [RelatonNist::GbBibliographicItem]
|
16
|
+
def get(code, date = nil, opts = {})
|
17
|
+
::RelatonNist::NistBibliography.get(code, date, opts)
|
18
|
+
end
|
19
|
+
|
20
|
+
# @param xml [String]
|
21
|
+
# @return [RelatonNist::GbBibliographicItem]
|
22
|
+
def from_xml(xml)
|
23
|
+
::RelatonNist::XMLParser.from_xml xml
|
24
|
+
end
|
25
|
+
|
26
|
+
# @param hash [Hash]
|
27
|
+
# @return [RelatonNist::GbBibliographicItem]
|
28
|
+
def hash_to_bib(hash)
|
29
|
+
item_hash = ::RelatonNist::HashConverter.hash_to_bib(hash)
|
30
|
+
::RelatonNist::NistBibliographicItem.new item_hash
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -21,7 +21,7 @@ module RelatonNist
|
|
21
21
|
unless /^(SP|NISTIR|FIPS) / =~ item_data[:docid][0].id
|
22
22
|
doctype = id_cleanup(item_data[:docid][0].id)
|
23
23
|
item_data[:docid][0] = RelatonBib::DocumentIdentifier.new(
|
24
|
-
id: titles[0][:content], type: "NIST",
|
24
|
+
id: titles[0][:content].upcase, type: "NIST",
|
25
25
|
)
|
26
26
|
end
|
27
27
|
item_data[:fetched] = Date.today.to_s
|
@@ -44,11 +44,9 @@ module RelatonNist
|
|
44
44
|
edition: fetch_edition(json),
|
45
45
|
language: [json["language"]],
|
46
46
|
script: [json["script"]],
|
47
|
-
# abstract: fetch_abstract(doc),
|
48
47
|
docstatus: fetch_status(json, hit_data[:status]),
|
49
48
|
copyright: fetch_copyright(json["published-date"]),
|
50
49
|
relation: fetch_relations_json(json),
|
51
|
-
# series: fetch_series(json),
|
52
50
|
keyword: fetch_keywords(json),
|
53
51
|
commentperiod: fetch_commentperiod_json(json),
|
54
52
|
}
|
@@ -179,14 +177,14 @@ module RelatonNist
|
|
179
177
|
dates = [{ type: "published", on: release_date.to_s }]
|
180
178
|
|
181
179
|
if doc.is_a? Hash
|
182
|
-
issued =
|
183
|
-
updated =
|
180
|
+
issued = RelatonBib.parse_date doc["issued-date"]
|
181
|
+
updated = RelatonBib.parse_date doc["updated-date"]
|
184
182
|
dates << { type: "updated", on: updated.to_s } if updated
|
185
|
-
obsoleted =
|
183
|
+
obsoleted = RelatonBib.parse_date doc["obsoleted-date"]
|
186
184
|
dates << { type: "obsoleted", on: obsoleted.to_s } if obsoleted
|
187
185
|
else
|
188
186
|
d = doc.at("//span[@id='pub-release-date']").text.strip
|
189
|
-
issued =
|
187
|
+
issued = RelatonBib.parse_date d
|
190
188
|
end
|
191
189
|
dates << { type: "issued", on: issued.to_s }
|
192
190
|
dates
|
data/lib/relaton_nist/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: relaton-nist
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -211,10 +211,8 @@ files:
|
|
211
211
|
- appveyor.yml
|
212
212
|
- bin/console
|
213
213
|
- bin/setup
|
214
|
-
- lib/relaton/processor.rb
|
215
214
|
- lib/relaton_nist.rb
|
216
215
|
- lib/relaton_nist/comment_period.rb
|
217
|
-
- lib/relaton_nist/data/pubs-export.zip
|
218
216
|
- lib/relaton_nist/document_status.rb
|
219
217
|
- lib/relaton_nist/hash_converter.rb
|
220
218
|
- lib/relaton_nist/hit.rb
|
@@ -222,6 +220,7 @@ files:
|
|
222
220
|
- lib/relaton_nist/keyword.rb
|
223
221
|
- lib/relaton_nist/nist_bibliographic_item.rb
|
224
222
|
- lib/relaton_nist/nist_bibliography.rb
|
223
|
+
- lib/relaton_nist/processor.rb
|
225
224
|
- lib/relaton_nist/scrapper.rb
|
226
225
|
- lib/relaton_nist/version.rb
|
227
226
|
- lib/relaton_nist/xml_parser.rb
|
@@ -245,8 +244,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
245
244
|
- !ruby/object:Gem::Version
|
246
245
|
version: '0'
|
247
246
|
requirements: []
|
248
|
-
|
249
|
-
rubygems_version: 2.6.12
|
247
|
+
rubygems_version: 3.0.6
|
250
248
|
signing_key:
|
251
249
|
specification_version: 4
|
252
250
|
summary: 'RelatonNist: retrive NIST standards.'
|
data/lib/relaton/processor.rb
DELETED
@@ -1,23 +0,0 @@
|
|
1
|
-
require "relaton/processor"
|
2
|
-
|
3
|
-
module Relaton
|
4
|
-
module RelatonNist
|
5
|
-
class Processor < Relaton::Processor
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
@short = :relaton_nist
|
9
|
-
@prefix = "NIST"
|
10
|
-
@defaultprefix = %r{^(NIST|NISTGCR|ITL Bulletin|JPCRD|NISTIR|CSRC)[ /]}
|
11
|
-
@idtype = "NIST"
|
12
|
-
end
|
13
|
-
|
14
|
-
def get(code, date = nil, opts = {})
|
15
|
-
::RelatonNist::NistBibliography.get(code, date, opts)
|
16
|
-
end
|
17
|
-
|
18
|
-
def from_xml(xml)
|
19
|
-
::RelatonNist::XMLParser.from_xml xml
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
end
|
Binary file
|