relaton-nist 0.8.2 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/grammars/basicdoc.rng +12 -55
- data/grammars/biblio.rng +109 -44
- data/grammars/isodoc.rng +575 -28
- data/grammars/isostandard.rng +137 -472
- data/grammars/nist.rng +98 -139
- data/lib/relaton_nist/nist_bibliography.rb +18 -14
- data/lib/relaton_nist/scrapper.rb +8 -8
- data/lib/relaton_nist/version.rb +1 -1
- data/relaton_nist.gemspec +2 -2
- metadata +10 -10
data/grammars/nist.rng
CHANGED
@@ -5,104 +5,14 @@
|
|
5
5
|
we cannot have a new default namespace: we will end up with a grammar with two different
|
6
6
|
namespaces, one for isostandard and one for csand additions. And we do not want that.
|
7
7
|
-->
|
8
|
-
<include href="
|
8
|
+
<include href="isodoc.rng">
|
9
9
|
<start>
|
10
10
|
<ref name="nist-standard"/>
|
11
11
|
</start>
|
12
|
-
<define name="figure">
|
13
|
-
<element name="figure">
|
14
|
-
<attribute name="id">
|
15
|
-
<data type="ID"/>
|
16
|
-
</attribute>
|
17
|
-
<optional>
|
18
|
-
<ref name="tname"/>
|
19
|
-
</optional>
|
20
|
-
<choice>
|
21
|
-
<ref name="image"/>
|
22
|
-
<ref name="pre"/>
|
23
|
-
<oneOrMore>
|
24
|
-
<ref name="subfigure"/>
|
25
|
-
</oneOrMore>
|
26
|
-
</choice>
|
27
|
-
<zeroOrMore>
|
28
|
-
<ref name="fn"/>
|
29
|
-
</zeroOrMore>
|
30
|
-
<optional>
|
31
|
-
<ref name="dl"/>
|
32
|
-
</optional>
|
33
|
-
<zeroOrMore>
|
34
|
-
<ref name="note"/>
|
35
|
-
</zeroOrMore>
|
36
|
-
</element>
|
37
|
-
</define>
|
38
|
-
<define name="subfigure">
|
39
|
-
<element name="figure">
|
40
|
-
<attribute name="id">
|
41
|
-
<data type="ID"/>
|
42
|
-
</attribute>
|
43
|
-
<optional>
|
44
|
-
<ref name="tname"/>
|
45
|
-
</optional>
|
46
|
-
<choice>
|
47
|
-
<ref name="image"/>
|
48
|
-
<ref name="pre"/>
|
49
|
-
</choice>
|
50
|
-
</element>
|
51
|
-
</define>
|
52
12
|
<define name="DocumentType">
|
53
13
|
<value>standard</value>
|
54
14
|
</define>
|
55
|
-
|
56
|
-
<choice>
|
57
|
-
<value>alternative</value>
|
58
|
-
<value>original</value>
|
59
|
-
<value>unofficial</value>
|
60
|
-
<value>subtitle</value>
|
61
|
-
<value>main</value>
|
62
|
-
</choice>
|
63
|
-
</define>
|
64
|
-
<!-- DocRelationType |= "obsoletedBy" | "supersedes" | "supersededBy" -->
|
65
|
-
<define name="DocRelationType">
|
66
|
-
<choice>
|
67
|
-
<value>obsoletes</value>
|
68
|
-
<value>updates</value>
|
69
|
-
<value>updatedBy</value>
|
70
|
-
<value>complements</value>
|
71
|
-
<value>derivedFrom</value>
|
72
|
-
<value>translatedFrom</value>
|
73
|
-
<value>adoptedFrom</value>
|
74
|
-
<value>equivalent</value>
|
75
|
-
<value>identical</value>
|
76
|
-
<value>nonequivalent</value>
|
77
|
-
<value>includedIn</value>
|
78
|
-
<value>includes</value>
|
79
|
-
<value>instance</value>
|
80
|
-
<value>partOf</value>
|
81
|
-
<value>hasDraft</value>
|
82
|
-
<value>obsoletedBy</value>
|
83
|
-
<value>supersedes</value>
|
84
|
-
<value>supersededBy</value>
|
85
|
-
</choice>
|
86
|
-
</define>
|
87
|
-
<!-- BibliographicDateType |= "abandoned" | "superseded" -->
|
88
|
-
<define name="BibliographicDateType">
|
89
|
-
<choice>
|
90
|
-
<value>published</value>
|
91
|
-
<value>accessed</value>
|
92
|
-
<value>created</value>
|
93
|
-
<value>implemented</value>
|
94
|
-
<value>obsoleted</value>
|
95
|
-
<value>confirmed</value>
|
96
|
-
<value>updated</value>
|
97
|
-
<value>issued</value>
|
98
|
-
<value>transmitted</value>
|
99
|
-
<value>copied</value>
|
100
|
-
<value>unchanged</value>
|
101
|
-
<value>circulated</value>
|
102
|
-
<value>abandoned</value>
|
103
|
-
<value>superseded</value>
|
104
|
-
</choice>
|
105
|
-
</define>
|
15
|
+
<!-- TitleType = ( "alternative" | "original" | "unofficial" | "subtitle" | "main" ) -->
|
106
16
|
<define name="preface">
|
107
17
|
<element name="preface">
|
108
18
|
<optional>
|
@@ -112,7 +22,10 @@
|
|
112
22
|
<ref name="foreword"/>
|
113
23
|
</optional>
|
114
24
|
<zeroOrMore>
|
115
|
-
<
|
25
|
+
<choice>
|
26
|
+
<ref name="clause"/>
|
27
|
+
<ref name="errata_clause"/>
|
28
|
+
</choice>
|
116
29
|
</zeroOrMore>
|
117
30
|
<optional>
|
118
31
|
<ref name="reviewernote"/>
|
@@ -122,62 +35,41 @@
|
|
122
35
|
</optional>
|
123
36
|
</element>
|
124
37
|
</define>
|
125
|
-
<define name="
|
126
|
-
<element name="
|
127
|
-
<
|
128
|
-
<
|
129
|
-
|
130
|
-
</attribute>
|
131
|
-
</optional>
|
132
|
-
<optional>
|
133
|
-
<attribute name="language"/>
|
134
|
-
</optional>
|
135
|
-
<optional>
|
136
|
-
<attribute name="script"/>
|
137
|
-
</optional>
|
138
|
-
<optional>
|
139
|
-
<attribute name="inline-header">
|
140
|
-
<data type="boolean"/>
|
141
|
-
</attribute>
|
142
|
-
</optional>
|
143
|
-
<optional>
|
144
|
-
<attribute name="obligation">
|
145
|
-
<choice>
|
146
|
-
<value>normative</value>
|
147
|
-
<value>informative</value>
|
148
|
-
</choice>
|
149
|
-
</attribute>
|
150
|
-
</optional>
|
151
|
-
<optional>
|
152
|
-
<ref name="section-title"/>
|
153
|
-
</optional>
|
154
|
-
<zeroOrMore>
|
155
|
-
<!--
|
156
|
-
allow hanging paragraps in annexes: they introduce lists
|
157
|
-
( paragraph-with-footnote | table | note | formula | admonition | ol | ul | dl | figure | quote | sourcecode | review | example )*,
|
158
|
-
-->
|
159
|
-
<ref name="BasicBlock"/>
|
160
|
-
</zeroOrMore>
|
161
|
-
<zeroOrMore>
|
162
|
-
<ref name="note"/>
|
163
|
-
</zeroOrMore>
|
164
|
-
<choice>
|
165
|
-
<zeroOrMore>
|
166
|
-
<ref name="clause-hanging-paragraph-with-footnote"/>
|
167
|
-
</zeroOrMore>
|
168
|
-
<ref name="terms"/>
|
169
|
-
</choice>
|
38
|
+
<define name="editorialgroup">
|
39
|
+
<element name="editorialgroup">
|
40
|
+
<oneOrMore>
|
41
|
+
<ref name="committee"/>
|
42
|
+
</oneOrMore>
|
170
43
|
</element>
|
171
44
|
</define>
|
172
45
|
<define name="BibDataExtensionType">
|
173
46
|
<optional>
|
174
47
|
<ref name="doctype"/>
|
175
48
|
</optional>
|
49
|
+
<optional>
|
50
|
+
<ref name="editorialgroup"/>
|
51
|
+
</optional>
|
52
|
+
<zeroOrMore>
|
53
|
+
<ref name="ics"/>
|
54
|
+
</zeroOrMore>
|
176
55
|
<optional>
|
177
56
|
<ref name="commentperiod"/>
|
178
57
|
</optional>
|
179
58
|
</define>
|
180
59
|
</include>
|
60
|
+
<define name="DocRelationType" combine="choice">
|
61
|
+
<choice>
|
62
|
+
<value>obsoletedBy</value>
|
63
|
+
<value>supersedes</value>
|
64
|
+
<value>supersededBy</value>
|
65
|
+
</choice>
|
66
|
+
</define>
|
67
|
+
<define name="BibliographicDateType" combine="choice">
|
68
|
+
<choice>
|
69
|
+
<value>abandoned</value>
|
70
|
+
<value>superseded</value>
|
71
|
+
</choice>
|
72
|
+
</define>
|
181
73
|
<define name="commentperiod">
|
182
74
|
<element name="commentperiod">
|
183
75
|
<element name="from">
|
@@ -205,6 +97,71 @@
|
|
205
97
|
<ref name="Basic-Section"/>
|
206
98
|
</element>
|
207
99
|
</define>
|
100
|
+
<define name="committee">
|
101
|
+
<element name="committee">
|
102
|
+
<text/>
|
103
|
+
</element>
|
104
|
+
</define>
|
105
|
+
<define name="errata">
|
106
|
+
<element name="errata">
|
107
|
+
<oneOrMore>
|
108
|
+
<ref name="erratarow"/>
|
109
|
+
</oneOrMore>
|
110
|
+
</element>
|
111
|
+
</define>
|
112
|
+
<define name="erratarow">
|
113
|
+
<element name="row">
|
114
|
+
<element name="date">
|
115
|
+
<ref name="ISO8601Date"/>
|
116
|
+
</element>
|
117
|
+
<element name="type">
|
118
|
+
<text/>
|
119
|
+
</element>
|
120
|
+
<element name="change">
|
121
|
+
<oneOrMore>
|
122
|
+
<ref name="TextElement"/>
|
123
|
+
</oneOrMore>
|
124
|
+
</element>
|
125
|
+
<element name="pages">
|
126
|
+
<text/>
|
127
|
+
</element>
|
128
|
+
</element>
|
129
|
+
</define>
|
130
|
+
<define name="errata_clause">
|
131
|
+
<element name="clause">
|
132
|
+
<optional>
|
133
|
+
<attribute name="id">
|
134
|
+
<data type="ID"/>
|
135
|
+
</attribute>
|
136
|
+
</optional>
|
137
|
+
<optional>
|
138
|
+
<attribute name="language"/>
|
139
|
+
</optional>
|
140
|
+
<optional>
|
141
|
+
<attribute name="script"/>
|
142
|
+
</optional>
|
143
|
+
<optional>
|
144
|
+
<attribute name="obligation">
|
145
|
+
<choice>
|
146
|
+
<value>normative</value>
|
147
|
+
<value>informative</value>
|
148
|
+
</choice>
|
149
|
+
</attribute>
|
150
|
+
</optional>
|
151
|
+
<optional>
|
152
|
+
<ref name="section-title"/>
|
153
|
+
</optional>
|
154
|
+
<group>
|
155
|
+
<zeroOrMore>
|
156
|
+
<ref name="BasicBlock"/>
|
157
|
+
</zeroOrMore>
|
158
|
+
<zeroOrMore>
|
159
|
+
<ref name="note"/>
|
160
|
+
</zeroOrMore>
|
161
|
+
</group>
|
162
|
+
<ref name="errata"/>
|
163
|
+
</element>
|
164
|
+
</define>
|
208
165
|
<define name="nist-standard">
|
209
166
|
<element name="nist-standard">
|
210
167
|
<ref name="bibdata"/>
|
@@ -221,7 +178,9 @@
|
|
221
178
|
<zeroOrMore>
|
222
179
|
<ref name="annex"/>
|
223
180
|
</zeroOrMore>
|
224
|
-
<
|
181
|
+
<optional>
|
182
|
+
<ref name="bibliography"/>
|
183
|
+
</optional>
|
225
184
|
</element>
|
226
185
|
</define>
|
227
186
|
</grammar>
|
@@ -60,9 +60,12 @@ module RelatonNist
|
|
60
60
|
def nistbib_get1(code, year, opts)
|
61
61
|
result = nistbib_search_filter(code, year, opts) || (return nil)
|
62
62
|
ret = nistbib_results_filter(result, year, opts)
|
63
|
-
|
64
|
-
|
65
|
-
|
63
|
+
if ret[:ret]
|
64
|
+
warn "[relaton-nist] (\"#{code}\") found #{ret[:ret].docidentifier.first.id}"
|
65
|
+
ret[:ret]
|
66
|
+
else
|
67
|
+
fetch_ref_err(code, year, ret[:years])
|
68
|
+
end
|
66
69
|
end
|
67
70
|
|
68
71
|
# Sort through the results from RelatonNist, fetching them three at a time,
|
@@ -108,33 +111,34 @@ module RelatonNist
|
|
108
111
|
{ years: missed_years }
|
109
112
|
end
|
110
113
|
|
111
|
-
def fetch_pages(
|
112
|
-
workers = RelatonBib::WorkersPool.new
|
114
|
+
def fetch_pages(hits, threads)
|
115
|
+
workers = RelatonBib::WorkersPool.new threads
|
113
116
|
workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
|
114
|
-
|
117
|
+
hits.each_with_index { |hit, i| workers << { i: i, hit: hit } }
|
115
118
|
workers.end
|
116
|
-
workers.result.
|
119
|
+
workers.result.sort_by { |a| a[:i] }.map { |x| x[:hit] }
|
117
120
|
end
|
118
121
|
|
119
122
|
def nistbib_search_filter(code, year, opts)
|
120
|
-
|
123
|
+
idregex = %r{[0-9-]{3,}}
|
124
|
+
docid = code.match(idregex).to_s
|
121
125
|
serie = code.match(%r{(FISP|SP|NISTIR)(?=\s)})
|
122
|
-
warn "
|
126
|
+
warn "[relaton-nist] (\"#{code}\") fetching..."
|
123
127
|
result = search(code, year, opts)
|
124
128
|
result.select do |i|
|
125
|
-
i.hit[:code]&.
|
129
|
+
i.hit[:code]&.match(idregex).to_s == docid && (!serie || i.hit[:serie] == serie.to_s)
|
126
130
|
end
|
127
131
|
end
|
128
132
|
|
129
133
|
def fetch_ref_err(code, year, missed_years)
|
130
134
|
id = year ? "#{code}:#{year}" : code
|
131
|
-
warn "WARNING: no match found online for #{id}. "\
|
135
|
+
warn "[relaton-nist] WARNING: no match found online for #{id}. "\
|
132
136
|
"The code must be exactly like it is on the standards website."
|
133
|
-
warn "(There was no match for #{year}, though there were matches "\
|
137
|
+
warn "[relaton-nist] (There was no match for #{year}, though there were matches "\
|
134
138
|
"found for #{missed_years.join(', ')}.)" unless missed_years.empty?
|
135
139
|
if /\d-\d/ =~ code
|
136
|
-
warn "The provided document part may not exist,
|
137
|
-
"may no longer be published in parts."
|
140
|
+
warn "[relaton-nist] The provided document part may not exist, "\
|
141
|
+
"or the document may no longer be published in parts."
|
138
142
|
end
|
139
143
|
nil
|
140
144
|
end
|
@@ -16,10 +16,10 @@ module RelatonNist
|
|
16
16
|
else
|
17
17
|
from_csrs hit_data
|
18
18
|
end
|
19
|
-
doctype = "standard"
|
19
|
+
# doctype = "standard"
|
20
20
|
titles = fetch_titles(hit_data)
|
21
21
|
unless /^(SP|NISTIR|FIPS) / =~ item_data[:docid][0].id
|
22
|
-
doctype = id_cleanup(item_data[:docid][0].id)
|
22
|
+
# doctype = id_cleanup(item_data[:docid][0].id)
|
23
23
|
item_data[:docid][0] = RelatonBib::DocumentIdentifier.new(
|
24
24
|
id: titles[0][:content].upcase, type: "NIST",
|
25
25
|
)
|
@@ -78,9 +78,9 @@ module RelatonNist
|
|
78
78
|
# Strip status from doc id
|
79
79
|
# @param id String
|
80
80
|
# @return String
|
81
|
-
def id_cleanup(id)
|
82
|
-
|
83
|
-
end
|
81
|
+
# def id_cleanup(id)
|
82
|
+
# id.sub(/ \(WITHDRAWN\)/, "").sub(/ \(([^) ]+ )?DRAFT\)/i, "")
|
83
|
+
# end
|
84
84
|
|
85
85
|
# Get page.
|
86
86
|
# @param path [String] page's path
|
@@ -235,7 +235,7 @@ module RelatonNist
|
|
235
235
|
)
|
236
236
|
end
|
237
237
|
if contr["surname"]
|
238
|
-
affiliation = RelatonBib::
|
238
|
+
affiliation = RelatonBib::Affiliation.new organization: org
|
239
239
|
entity = RelatonBib::Person.new(
|
240
240
|
name: full_name(contr, lang, script), affiliation: [affiliation],
|
241
241
|
)
|
@@ -271,7 +271,7 @@ module RelatonNist
|
|
271
271
|
url = nil
|
272
272
|
end
|
273
273
|
org = RelatonBib::Organization.new name: org_name, url: url, abbreviation: abbrev
|
274
|
-
affiliation = RelatonBib::
|
274
|
+
affiliation = RelatonBib::Affiliation.new organization: org
|
275
275
|
entity = RelatonBib::Person.new(
|
276
276
|
name: fullname, affiliation: [affiliation],
|
277
277
|
)
|
@@ -304,7 +304,7 @@ module RelatonNist
|
|
304
304
|
def name_parts(part, lang, script)
|
305
305
|
return [] unless part
|
306
306
|
|
307
|
-
[RelatonBib::LocalizedString.new(
|
307
|
+
[RelatonBib::LocalizedString.new(part, lang, script)]
|
308
308
|
end
|
309
309
|
|
310
310
|
# @param doc [String, Hash]
|
data/lib/relaton_nist/version.rb
CHANGED
data/relaton_nist.gemspec
CHANGED
@@ -29,11 +29,11 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_development_dependency "rake", "~> 10.0"
|
30
30
|
spec.add_development_dependency "rspec", "~> 3.0"
|
31
31
|
spec.add_development_dependency "ruby-debug-ide"
|
32
|
+
spec.add_development_dependency "ruby-jing"
|
32
33
|
spec.add_development_dependency "simplecov"
|
33
34
|
spec.add_development_dependency "vcr"
|
34
35
|
spec.add_development_dependency "webmock"
|
35
|
-
spec.add_development_dependency "ruby-jing"
|
36
36
|
|
37
|
-
spec.add_dependency "relaton-bib", "
|
37
|
+
spec.add_dependency "relaton-bib", ">= 1.0.4"
|
38
38
|
spec.add_dependency "rubyzip"
|
39
39
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: relaton-nist
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: debase
|
@@ -95,7 +95,7 @@ dependencies:
|
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
98
|
+
name: ruby-jing
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
101
|
- - ">="
|
@@ -109,7 +109,7 @@ dependencies:
|
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
112
|
+
name: simplecov
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - ">="
|
@@ -123,7 +123,7 @@ dependencies:
|
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
|
-
name:
|
126
|
+
name: vcr
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
129
|
- - ">="
|
@@ -137,7 +137,7 @@ dependencies:
|
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
|
-
name:
|
140
|
+
name: webmock
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
143
|
- - ">="
|
@@ -154,16 +154,16 @@ dependencies:
|
|
154
154
|
name: relaton-bib
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- - "
|
157
|
+
- - ">="
|
158
158
|
- !ruby/object:Gem::Version
|
159
|
-
version: 0.
|
159
|
+
version: 1.0.4
|
160
160
|
type: :runtime
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
|
-
- - "
|
164
|
+
- - ">="
|
165
165
|
- !ruby/object:Gem::Version
|
166
|
-
version: 0.
|
166
|
+
version: 1.0.4
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: rubyzip
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|