fsp_harvester 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +11 -0
- data/Gemfile.lock +319 -0
- data/LICENSE +21 -0
- data/README.md +34 -0
- data/Rakefile +12 -0
- data/bin/console +15 -0
- data/bin/setup +8 -0
- data/example_test.rb +24 -0
- data/lib/constants.rb +106 -0
- data/lib/fsp_harvester/version.rb +5 -0
- data/lib/fsp_harvester.rb +137 -0
- data/lib/metadata_object.rb +28 -0
- data/lib/swagger.rb +224 -0
- data/lib/warnings.json +33 -0
- data/lib/web_utils.rb +50 -0
- metadata +217 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: af33b0b18a3881064266522e20c5df538ed4ab67a7aa35bee6af212876f032a8
|
|
4
|
+
data.tar.gz: 11421f955b831e5ca726f310f12cc307f050cff154dbb6970812872bab685bdc
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 8834e22e1adbfab672c2f38408e633ac8fb94f3617709f2576c7d2cf653e22500ee95a45b951f153fc07241356a8e983840743ce9849986a32bcb35178df726b
|
|
7
|
+
data.tar.gz: 0c60642499bc9711d8f4c3b8e4818a950ff7718afb02b745b47abcf6b03dfd8abed20136f98ff1ccaf79bac8178a3f138ca431c1b0bb9a3edae4344202af61af
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
fsp_harvester (0.1.2)
|
|
5
|
+
json (~> 2.0)
|
|
6
|
+
linkeddata (~> 3.2)
|
|
7
|
+
linkheaders-processor (~> 0.1.9)
|
|
8
|
+
metainspector (~> 5.11.2)
|
|
9
|
+
parseconfig (~> 1.1)
|
|
10
|
+
rake (~> 13.0)
|
|
11
|
+
rest-client (~> 2.1)
|
|
12
|
+
rspec (~> 3.11)
|
|
13
|
+
rubocop (~> 1.7)
|
|
14
|
+
securerandom (~> 0.1.0)
|
|
15
|
+
xml-simple (~> 1.1)
|
|
16
|
+
|
|
17
|
+
GEM
|
|
18
|
+
remote: https://rubygems.org/
|
|
19
|
+
specs:
|
|
20
|
+
addressable (2.8.0)
|
|
21
|
+
public_suffix (>= 2.0.2, < 5.0)
|
|
22
|
+
amazing_print (1.4.0)
|
|
23
|
+
ast (2.4.2)
|
|
24
|
+
bcp47 (0.3.3)
|
|
25
|
+
i18n
|
|
26
|
+
builder (3.2.4)
|
|
27
|
+
concurrent-ruby (1.1.10)
|
|
28
|
+
connection_pool (2.2.5)
|
|
29
|
+
diff-lcs (1.5.0)
|
|
30
|
+
domain_name (0.5.20190701)
|
|
31
|
+
unf (>= 0.0.5, < 1.0.0)
|
|
32
|
+
ebnf (2.3.1)
|
|
33
|
+
amazing_print (~> 1.4)
|
|
34
|
+
htmlentities (~> 4.3)
|
|
35
|
+
rdf (~> 3.2)
|
|
36
|
+
scanf (~> 1.0)
|
|
37
|
+
sxp (~> 1.2)
|
|
38
|
+
unicode-types (~> 1.7)
|
|
39
|
+
faraday (1.10.0)
|
|
40
|
+
faraday-em_http (~> 1.0)
|
|
41
|
+
faraday-em_synchrony (~> 1.0)
|
|
42
|
+
faraday-excon (~> 1.1)
|
|
43
|
+
faraday-httpclient (~> 1.0)
|
|
44
|
+
faraday-multipart (~> 1.0)
|
|
45
|
+
faraday-net_http (~> 1.0)
|
|
46
|
+
faraday-net_http_persistent (~> 1.0)
|
|
47
|
+
faraday-patron (~> 1.0)
|
|
48
|
+
faraday-rack (~> 1.0)
|
|
49
|
+
faraday-retry (~> 1.0)
|
|
50
|
+
ruby2_keywords (>= 0.0.4)
|
|
51
|
+
faraday-cookie_jar (0.0.7)
|
|
52
|
+
faraday (>= 0.8.0)
|
|
53
|
+
http-cookie (~> 1.0.0)
|
|
54
|
+
faraday-em_http (1.0.0)
|
|
55
|
+
faraday-em_synchrony (1.0.0)
|
|
56
|
+
faraday-encoding (0.0.5)
|
|
57
|
+
faraday
|
|
58
|
+
faraday-excon (1.1.0)
|
|
59
|
+
faraday-http-cache (2.4.0)
|
|
60
|
+
faraday (>= 0.8)
|
|
61
|
+
faraday-httpclient (1.0.1)
|
|
62
|
+
faraday-multipart (1.0.4)
|
|
63
|
+
multipart-post (~> 2)
|
|
64
|
+
faraday-net_http (1.0.1)
|
|
65
|
+
faraday-net_http_persistent (1.2.0)
|
|
66
|
+
faraday-patron (1.0.0)
|
|
67
|
+
faraday-rack (1.0.0)
|
|
68
|
+
faraday-retry (1.0.3)
|
|
69
|
+
faraday_middleware (1.2.0)
|
|
70
|
+
faraday (~> 1.0)
|
|
71
|
+
fastimage (2.2.6)
|
|
72
|
+
haml (5.2.2)
|
|
73
|
+
temple (>= 0.8.0)
|
|
74
|
+
tilt
|
|
75
|
+
hamster (3.0.0)
|
|
76
|
+
concurrent-ruby (~> 1.0)
|
|
77
|
+
htmlentities (4.3.4)
|
|
78
|
+
http-accept (1.7.0)
|
|
79
|
+
http-cookie (1.0.5)
|
|
80
|
+
domain_name (~> 0.5)
|
|
81
|
+
i18n (1.12.0)
|
|
82
|
+
concurrent-ruby (~> 1.0)
|
|
83
|
+
json (2.6.2)
|
|
84
|
+
json-canonicalization (0.3.0)
|
|
85
|
+
json-ld (3.2.1)
|
|
86
|
+
htmlentities (~> 4.3)
|
|
87
|
+
json-canonicalization (~> 0.3)
|
|
88
|
+
link_header (~> 0.0, >= 0.0.8)
|
|
89
|
+
multi_json (~> 1.15)
|
|
90
|
+
rack (~> 2.2)
|
|
91
|
+
rdf (~> 3.2)
|
|
92
|
+
json-ld-preloaded (3.2.0)
|
|
93
|
+
json-ld (~> 3.2)
|
|
94
|
+
rdf (~> 3.2)
|
|
95
|
+
ld-patch (3.2.0)
|
|
96
|
+
ebnf (~> 2.2)
|
|
97
|
+
rdf (~> 3.2)
|
|
98
|
+
rdf-xsd (~> 3.2)
|
|
99
|
+
sparql (~> 3.2)
|
|
100
|
+
sxp (~> 1.2)
|
|
101
|
+
link_header (0.0.8)
|
|
102
|
+
linkeddata (3.2.0)
|
|
103
|
+
json-ld (~> 3.2)
|
|
104
|
+
json-ld-preloaded (~> 3.2)
|
|
105
|
+
ld-patch (~> 3.2)
|
|
106
|
+
nokogiri (~> 1.12, >= 1.12.5)
|
|
107
|
+
rdf (~> 3.2)
|
|
108
|
+
rdf-aggregate-repo (~> 3.2)
|
|
109
|
+
rdf-hamster-repo (~> 3.2)
|
|
110
|
+
rdf-isomorphic (~> 3.2)
|
|
111
|
+
rdf-json (~> 3.2)
|
|
112
|
+
rdf-microdata (~> 3.2)
|
|
113
|
+
rdf-n3 (~> 3.2)
|
|
114
|
+
rdf-normalize (~> 0.5)
|
|
115
|
+
rdf-ordered-repo (~> 3.2)
|
|
116
|
+
rdf-rdfa (~> 3.2)
|
|
117
|
+
rdf-rdfxml (~> 3.2)
|
|
118
|
+
rdf-reasoner (~> 0.8)
|
|
119
|
+
rdf-tabular (~> 3.2)
|
|
120
|
+
rdf-trig (~> 3.2)
|
|
121
|
+
rdf-trix (~> 3.2)
|
|
122
|
+
rdf-turtle (~> 3.2)
|
|
123
|
+
rdf-vocab (~> 3.2)
|
|
124
|
+
rdf-xsd (~> 3.2)
|
|
125
|
+
shacl (~> 0.2)
|
|
126
|
+
shex (~> 0.7)
|
|
127
|
+
sparql (~> 3.2)
|
|
128
|
+
sparql-client (~> 3.2)
|
|
129
|
+
linkheaders-processor (0.1.9)
|
|
130
|
+
json (~> 2.0)
|
|
131
|
+
json-ld (~> 3.2)
|
|
132
|
+
json-ld-preloaded (~> 3.2)
|
|
133
|
+
metainspector (~> 5.11.2)
|
|
134
|
+
rest-client (~> 2.1)
|
|
135
|
+
securerandom (~> 0.1.0)
|
|
136
|
+
logger (1.5.1)
|
|
137
|
+
matrix (0.4.2)
|
|
138
|
+
metainspector (5.11.2)
|
|
139
|
+
addressable (~> 2.7)
|
|
140
|
+
faraday (~> 1.4)
|
|
141
|
+
faraday-cookie_jar (~> 0.0)
|
|
142
|
+
faraday-encoding (~> 0.0)
|
|
143
|
+
faraday-http-cache (~> 2.2)
|
|
144
|
+
faraday_middleware (~> 1.0)
|
|
145
|
+
fastimage (~> 2.2)
|
|
146
|
+
nesty (~> 1.0)
|
|
147
|
+
nokogiri (~> 1.11)
|
|
148
|
+
mime-types (3.4.1)
|
|
149
|
+
mime-types-data (~> 3.2015)
|
|
150
|
+
mime-types-data (3.2022.0105)
|
|
151
|
+
multi_json (1.15.0)
|
|
152
|
+
multipart-post (2.2.3)
|
|
153
|
+
nesty (1.0.2)
|
|
154
|
+
net-http-persistent (4.0.1)
|
|
155
|
+
connection_pool (~> 2.2)
|
|
156
|
+
netrc (0.11.0)
|
|
157
|
+
nokogiri (1.13.8-x86_64-linux)
|
|
158
|
+
racc (~> 1.4)
|
|
159
|
+
parallel (1.22.1)
|
|
160
|
+
parseconfig (1.1.2)
|
|
161
|
+
parser (3.1.2.0)
|
|
162
|
+
ast (~> 2.4.1)
|
|
163
|
+
public_suffix (4.0.7)
|
|
164
|
+
racc (1.6.0)
|
|
165
|
+
rack (2.2.4)
|
|
166
|
+
rainbow (3.1.1)
|
|
167
|
+
rake (13.0.6)
|
|
168
|
+
rdf (3.2.8)
|
|
169
|
+
link_header (~> 0.0, >= 0.0.8)
|
|
170
|
+
rdf-aggregate-repo (3.2.1)
|
|
171
|
+
rdf (~> 3.2)
|
|
172
|
+
rdf-hamster-repo (3.2.1)
|
|
173
|
+
hamster (~> 3.0)
|
|
174
|
+
rdf (~> 3.2, >= 3.2.1)
|
|
175
|
+
rdf-isomorphic (3.2.1)
|
|
176
|
+
rdf (~> 3.2)
|
|
177
|
+
rdf-json (3.2.0)
|
|
178
|
+
rdf (~> 3.2)
|
|
179
|
+
rdf-microdata (3.2.1)
|
|
180
|
+
htmlentities (~> 4.3)
|
|
181
|
+
nokogiri (~> 1.13)
|
|
182
|
+
rdf (~> 3.2)
|
|
183
|
+
rdf-rdfa (~> 3.2)
|
|
184
|
+
rdf-xsd (~> 3.2)
|
|
185
|
+
rdf-n3 (3.2.1)
|
|
186
|
+
ebnf (~> 2.2)
|
|
187
|
+
rdf (~> 3.2)
|
|
188
|
+
sparql (~> 3.2)
|
|
189
|
+
sxp (~> 1.2)
|
|
190
|
+
rdf-normalize (0.5.0)
|
|
191
|
+
rdf (~> 3.2)
|
|
192
|
+
rdf-ordered-repo (3.2.1)
|
|
193
|
+
rdf (~> 3.2, >= 3.2.1)
|
|
194
|
+
rdf-rdfa (3.2.0)
|
|
195
|
+
haml (~> 5.2)
|
|
196
|
+
htmlentities (~> 4.3)
|
|
197
|
+
rdf (~> 3.2)
|
|
198
|
+
rdf-aggregate-repo (~> 3.2)
|
|
199
|
+
rdf-vocab (~> 3.2)
|
|
200
|
+
rdf-xsd (~> 3.2)
|
|
201
|
+
rdf-rdfxml (3.2.0)
|
|
202
|
+
htmlentities (~> 4.3)
|
|
203
|
+
rdf (~> 3.2)
|
|
204
|
+
rdf-rdfa (~> 3.2)
|
|
205
|
+
rdf-xsd (~> 3.2)
|
|
206
|
+
rdf-reasoner (0.8.0)
|
|
207
|
+
rdf (~> 3.2)
|
|
208
|
+
rdf-xsd (~> 3.2)
|
|
209
|
+
rdf-tabular (3.2.1)
|
|
210
|
+
addressable (~> 2.8)
|
|
211
|
+
bcp47 (~> 0.3, >= 0.3.3)
|
|
212
|
+
json-ld (~> 3.2)
|
|
213
|
+
rdf (~> 3.2, >= 3.2.7)
|
|
214
|
+
rdf-vocab (~> 3.2)
|
|
215
|
+
rdf-xsd (~> 3.2)
|
|
216
|
+
rdf-trig (3.2.0)
|
|
217
|
+
ebnf (~> 2.2)
|
|
218
|
+
rdf (~> 3.2)
|
|
219
|
+
rdf-turtle (~> 3.2)
|
|
220
|
+
rdf-trix (3.2.0)
|
|
221
|
+
rdf (~> 3.2)
|
|
222
|
+
rdf-xsd (~> 3.2)
|
|
223
|
+
rdf-turtle (3.2.1)
|
|
224
|
+
ebnf (~> 2.3)
|
|
225
|
+
rdf (~> 3.2)
|
|
226
|
+
rdf-vocab (3.2.1)
|
|
227
|
+
rdf (~> 3.2, >= 3.2.4)
|
|
228
|
+
rdf-xsd (3.2.1)
|
|
229
|
+
rdf (~> 3.2)
|
|
230
|
+
rexml (~> 3.2)
|
|
231
|
+
regexp_parser (2.5.0)
|
|
232
|
+
rest-client (2.1.0)
|
|
233
|
+
http-accept (>= 1.7.0, < 2.0)
|
|
234
|
+
http-cookie (>= 1.0.2, < 2.0)
|
|
235
|
+
mime-types (>= 1.16, < 4.0)
|
|
236
|
+
netrc (~> 0.8)
|
|
237
|
+
rexml (3.2.5)
|
|
238
|
+
rspec (3.11.0)
|
|
239
|
+
rspec-core (~> 3.11.0)
|
|
240
|
+
rspec-expectations (~> 3.11.0)
|
|
241
|
+
rspec-mocks (~> 3.11.0)
|
|
242
|
+
rspec-core (3.11.0)
|
|
243
|
+
rspec-support (~> 3.11.0)
|
|
244
|
+
rspec-expectations (3.11.0)
|
|
245
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
246
|
+
rspec-support (~> 3.11.0)
|
|
247
|
+
rspec-mocks (3.11.1)
|
|
248
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
249
|
+
rspec-support (~> 3.11.0)
|
|
250
|
+
rspec-support (3.11.0)
|
|
251
|
+
rubocop (1.32.0)
|
|
252
|
+
json (~> 2.3)
|
|
253
|
+
parallel (~> 1.10)
|
|
254
|
+
parser (>= 3.1.0.0)
|
|
255
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
256
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
257
|
+
rexml (>= 3.2.5, < 4.0)
|
|
258
|
+
rubocop-ast (>= 1.19.1, < 2.0)
|
|
259
|
+
ruby-progressbar (~> 1.7)
|
|
260
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
|
261
|
+
rubocop-ast (1.19.1)
|
|
262
|
+
parser (>= 3.1.1.0)
|
|
263
|
+
ruby-progressbar (1.11.0)
|
|
264
|
+
ruby2_keywords (0.0.5)
|
|
265
|
+
scanf (1.0.0)
|
|
266
|
+
securerandom (0.1.1)
|
|
267
|
+
shacl (0.3.0)
|
|
268
|
+
json-ld (~> 3.2)
|
|
269
|
+
rdf (~> 3.2, >= 3.2.8)
|
|
270
|
+
sparql (~> 3.2, >= 3.2.4)
|
|
271
|
+
sxp (~> 1.2)
|
|
272
|
+
shex (0.7.1)
|
|
273
|
+
ebnf (~> 2.2)
|
|
274
|
+
htmlentities (~> 4.3)
|
|
275
|
+
json-ld (~> 3.2)
|
|
276
|
+
json-ld-preloaded (~> 3.2)
|
|
277
|
+
rdf (~> 3.2)
|
|
278
|
+
rdf-xsd (~> 3.2)
|
|
279
|
+
sparql (~> 3.2)
|
|
280
|
+
sxp (~> 1.2)
|
|
281
|
+
sparql (3.2.4)
|
|
282
|
+
builder (~> 3.2)
|
|
283
|
+
ebnf (~> 2.2, >= 2.3.1)
|
|
284
|
+
logger (~> 1.5)
|
|
285
|
+
rdf (~> 3.2, >= 3.2.8)
|
|
286
|
+
rdf-aggregate-repo (~> 3.2)
|
|
287
|
+
rdf-xsd (~> 3.2)
|
|
288
|
+
sparql-client (~> 3.2, >= 3.2.1)
|
|
289
|
+
sxp (~> 1.2, >= 1.2.2)
|
|
290
|
+
sparql-client (3.2.1)
|
|
291
|
+
net-http-persistent (~> 4.0, >= 4.0.1)
|
|
292
|
+
rdf (~> 3.2, >= 3.2.6)
|
|
293
|
+
sxp (1.2.2)
|
|
294
|
+
matrix
|
|
295
|
+
rdf (~> 3.2)
|
|
296
|
+
temple (0.8.2)
|
|
297
|
+
tilt (2.0.11)
|
|
298
|
+
unf (0.1.4)
|
|
299
|
+
unf_ext
|
|
300
|
+
unf_ext (0.0.8.2)
|
|
301
|
+
unicode-display_width (2.2.0)
|
|
302
|
+
unicode-types (1.7.0)
|
|
303
|
+
xml-simple (1.1.9)
|
|
304
|
+
rexml
|
|
305
|
+
|
|
306
|
+
PLATFORMS
|
|
307
|
+
x86_64-linux
|
|
308
|
+
|
|
309
|
+
DEPENDENCIES
|
|
310
|
+
fsp_harvester!
|
|
311
|
+
rake (~> 13.0)
|
|
312
|
+
rspec (~> 3.11)
|
|
313
|
+
rubocop (~> 1.7)
|
|
314
|
+
|
|
315
|
+
RUBY VERSION
|
|
316
|
+
ruby 3.0.0p0
|
|
317
|
+
|
|
318
|
+
BUNDLED WITH
|
|
319
|
+
2.3.12
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Mark Wilkinson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# FspHarvester
|
|
2
|
+
|
|
3
|
+
EXPERIMENTAL: DO NOT USE
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem 'fsp_harvester'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle install
|
|
16
|
+
|
|
17
|
+
Or install it yourself as:
|
|
18
|
+
|
|
19
|
+
$ gem install fsp_harvester
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
## Development
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
## Contributing
|
|
29
|
+
|
|
30
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/markwilkinson/fsp_harvester.
|
|
31
|
+
|
|
32
|
+
## License
|
|
33
|
+
|
|
34
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "bundler/setup"
|
|
5
|
+
require "fsp_harvester"
|
|
6
|
+
|
|
7
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
8
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
9
|
+
|
|
10
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
11
|
+
# require "pry"
|
|
12
|
+
# Pry.start
|
|
13
|
+
|
|
14
|
+
require "irb"
|
|
15
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/example_test.rb
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen string literal = false
|
|
2
|
+
require 'cgi'
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'uri'
|
|
5
|
+
require 'rdf'
|
|
6
|
+
require 'rdf/turtle'
|
|
7
|
+
require 'sparql'
|
|
8
|
+
require 'fsp_harvester'
|
|
9
|
+
|
|
10
|
+
def test_guid(guid:)
|
|
11
|
+
_links, metadata = FspHarvester::Utils.resolve_guid(guid: guid) # [LinkHeader::Link], FspHarvester::MetadataObject
|
|
12
|
+
|
|
13
|
+
metadata.comments << if metadata.guidtype == 'unknown'
|
|
14
|
+
"FAILURE: The identifier #{guid} did not match any known identification system.\n"
|
|
15
|
+
else
|
|
16
|
+
"SUCCESS: The identifier #{guid} matched known GUID type system #{metadata.guidtype}.\n"
|
|
17
|
+
end
|
|
18
|
+
metadata.comments
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
guid = ARGV[0] || 'https://s11.no/2022/a2a-fair-metrics/07-http-describedby-citeas-linkset-json/'
|
|
22
|
+
response = test_guid(guid: guid)
|
|
23
|
+
|
|
24
|
+
puts response
|
data/lib/constants.rb
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
ACCEPT_ALL_HEADER = { 'Accept' => 'text/turtle, application/ld+json, application/rdf+xml, text/xhtml+xml, application/n3, application/rdf+n3, application/turtle, application/x-turtle, text/n3, text/turtle, text/rdf+n3, text/rdf+turtle, application/n-triples' }
|
|
2
|
+
|
|
3
|
+
TEXT_FORMATS = {
|
|
4
|
+
'text' => ['text/plain']
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
RDF_FORMATS = {
|
|
8
|
+
'jsonld' => ['application/ld+json', 'application/vnd.schemaorg.ld+json'], # NEW FOR DATACITE
|
|
9
|
+
'turtle' => ['text/turtle', 'application/n3', 'application/rdf+n3',
|
|
10
|
+
'application/turtle', 'application/x-turtle', 'text/n3', 'text/turtle',
|
|
11
|
+
'text/rdf+n3', 'text/rdf+turtle'],
|
|
12
|
+
# 'rdfa' => ['text/xhtml+xml', 'application/xhtml+xml'],
|
|
13
|
+
'rdfxml' => ['application/rdf+xml'],
|
|
14
|
+
'triples' => ['application/n-triples', 'application/n-quads', 'application/trig']
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
XML_FORMATS = {
|
|
18
|
+
'xml' => ['text/xhtml', 'text/xml']
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
HTML_FORMATS = {
|
|
22
|
+
'html' => ['text/html', 'text/xhtml+xml', 'application/xhtml+xml']
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
JSON_FORMATS = {
|
|
26
|
+
'json' => ['application/json']
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
DATA_PREDICATES = [
|
|
30
|
+
'http://www.w3.org/ns/ldp#contains',
|
|
31
|
+
'http://xmlns.com/foaf/0.1/primaryTopic',
|
|
32
|
+
'http://purl.obolibrary.org/obo/IAO_0000136', # is about
|
|
33
|
+
'http://purl.obolibrary.org/obo/IAO:0000136', # is about (not the valid URL...)
|
|
34
|
+
'https://www.w3.org/ns/ldp#contains',
|
|
35
|
+
'https://xmlns.com/foaf/0.1/primaryTopic',
|
|
36
|
+
|
|
37
|
+
'http://schema.org/mainEntity',
|
|
38
|
+
'http://schema.org/codeRepository',
|
|
39
|
+
'http://schema.org/distribution',
|
|
40
|
+
'https://schema.org/mainEntity',
|
|
41
|
+
'https://schema.org/codeRepository',
|
|
42
|
+
'https://schema.org/distribution',
|
|
43
|
+
|
|
44
|
+
'http://www.w3.org/ns/dcat#distribution',
|
|
45
|
+
'https://www.w3.org/ns/dcat#distribution',
|
|
46
|
+
'http://www.w3.org/ns/dcat#dataset',
|
|
47
|
+
'https://www.w3.org/ns/dcat#dataset',
|
|
48
|
+
'http://www.w3.org/ns/dcat#downloadURL',
|
|
49
|
+
'https://www.w3.org/ns/dcat#downloadURL',
|
|
50
|
+
'http://www.w3.org/ns/dcat#accessURL',
|
|
51
|
+
'https://www.w3.org/ns/dcat#accessURL',
|
|
52
|
+
|
|
53
|
+
'http://semanticscience.org/resource/SIO_000332', # is about
|
|
54
|
+
'http://semanticscience.org/resource/is-about', # is about
|
|
55
|
+
'https://semanticscience.org/resource/SIO_000332', # is about
|
|
56
|
+
'https://semanticscience.org/resource/is-about', # is about
|
|
57
|
+
'https://purl.obolibrary.org/obo/IAO_0000136' # is about
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
SELF_IDENTIFIER_PREDICATES = [
|
|
61
|
+
'http://purl.org/dc/elements/1.1/identifier',
|
|
62
|
+
'https://purl.org/dc/elements/1.1/identifier',
|
|
63
|
+
'http://purl.org/dc/terms/identifier',
|
|
64
|
+
'http://schema.org/identifier',
|
|
65
|
+
'https://purl.org/dc/terms/identifier',
|
|
66
|
+
'https://schema.org/identifier'
|
|
67
|
+
]
|
|
68
|
+
|
|
69
|
+
GUID_TYPES = { 'inchi' => Regexp.new(/^\w{14}-\w{10}-\w$/),
|
|
70
|
+
'doi' => Regexp.new(%r{^10.\d{4,9}/[-._;()/:A-Z0-9]+$}i),
|
|
71
|
+
'handle1' => Regexp.new(%r{^[^/]+/[^/]+$}i),
|
|
72
|
+
'handle2' => Regexp.new(%r{^\d{4,5}/[-._;()/:A-Z0-9]+$}i), # legacy style 12345/AGB47A
|
|
73
|
+
'uri' => Regexp.new(%r{^\w+:/?/?[^\s]+$}) }
|
|
74
|
+
|
|
75
|
+
CONFIG = File.exist?('config.conf') ? ParseConfig.new('config.conf') : {}
|
|
76
|
+
if CONFIG['extruct'] && CONFIG['extruct']['command'] && !CONFIG['extruct']['command'].empty?
|
|
77
|
+
extruct = config['extruct']['command']
|
|
78
|
+
end
|
|
79
|
+
extruct = 'extruct' unless @extruct_command
|
|
80
|
+
extruct.strip!
|
|
81
|
+
case @extruct
|
|
82
|
+
when /[&|;`$\s]/
|
|
83
|
+
abort 'The Extruct command in the config file appears to be subject to command injection. I will not continue'
|
|
84
|
+
when /echo/i
|
|
85
|
+
abort 'The Extruct command in the config file appears to be subject to command injection. I will not continue'
|
|
86
|
+
end
|
|
87
|
+
EXTRUCT_COMMAND = extruct
|
|
88
|
+
|
|
89
|
+
rdf_command = CONFIG['rdf']['command'] if CONFIG['rdf'] && CONFIG['rdf']['command'] && !CONFIG['rdf']['command'].empty?
|
|
90
|
+
rdf_command = 'rdf' unless @rdf_command
|
|
91
|
+
rdf_command.strip
|
|
92
|
+
case rdf_command
|
|
93
|
+
when /[&|;`$\s]/
|
|
94
|
+
abort 'The RDF command in the config file appears to be subject to command injection. I will not continue'
|
|
95
|
+
when /echo/i
|
|
96
|
+
abort 'The RDF command in the config file appears to be subject to command injection. I will not continue'
|
|
97
|
+
when !(/rdf$/ =~ $_)
|
|
98
|
+
abort "this software requires that Kelloggs Distiller tool is used. The distiller command must end in 'rdf'"
|
|
99
|
+
end
|
|
100
|
+
RDF_COMMAND = rdf_command
|
|
101
|
+
|
|
102
|
+
if CONFIG['tika'] && CONFIG['tika']['command'] && !CONFIG['tika']['command'].empty?
|
|
103
|
+
tika_command = CONFIG['tika']['command']
|
|
104
|
+
end
|
|
105
|
+
tika_command = 'http://localhost:9998/meta' unless @tika_command
|
|
106
|
+
TIKA_COMMAND = tika_command
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "fsp_harvester/version"
|
|
4
|
+
require "json/ld"
|
|
5
|
+
require "json/ld/preloaded"
|
|
6
|
+
require "json"
|
|
7
|
+
require "linkheaders/processor"
|
|
8
|
+
require "addressable"
|
|
9
|
+
require "tempfile"
|
|
10
|
+
require "xmlsimple"
|
|
11
|
+
require "nokogiri"
|
|
12
|
+
require "parseconfig"
|
|
13
|
+
require "rest-client"
|
|
14
|
+
require "cgi"
|
|
15
|
+
require "digest"
|
|
16
|
+
require "open3"
|
|
17
|
+
require "metainspector"
|
|
18
|
+
require "rdf/xsd"
|
|
19
|
+
require_relative "./metadata_object"
|
|
20
|
+
require_relative "./constants"
|
|
21
|
+
require_relative "./web_utils"
|
|
22
|
+
|
|
23
|
+
module FspHarvester
|
|
24
|
+
class Error < StandardError
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
class Utils
|
|
28
|
+
# @@distillerknown = {} # global, hash of sha256 keys of message bodies - have they been seen before t/f
|
|
29
|
+
# @warnings = JSON.parse(File.read("warnings.json"))
|
|
30
|
+
@meta = FspHarvester::MetadataObject.new
|
|
31
|
+
|
|
32
|
+
def self.resolve_guid(guid:)
|
|
33
|
+
@meta.finalURI = [guid]
|
|
34
|
+
type, url = convertToURL(guid: guid)
|
|
35
|
+
links = Array.new
|
|
36
|
+
unless type
|
|
37
|
+
@meta.warnings << ["006", guid, ""]
|
|
38
|
+
@meta.comments << "FATAL: GUID type not recognized.\n"
|
|
39
|
+
else
|
|
40
|
+
links, @meta = resolve_url(url: url)
|
|
41
|
+
end
|
|
42
|
+
[links, @meta]
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.convertToURL(guid:)
|
|
46
|
+
GUID_TYPES.each do |k, regex|
|
|
47
|
+
if k == "inchi" and regex.match(guid)
|
|
48
|
+
return "inchi", "https://pubchem.ncbi.nlm.nih.gov/rest/rdf/inchikey/#{guid}"
|
|
49
|
+
elsif k == "handle1" and regex.match(guid)
|
|
50
|
+
return "handle", "http://hdl.handle.net/#{guid}"
|
|
51
|
+
elsif k == "handle2" and regex.match(guid)
|
|
52
|
+
return "handle", "http://hdl.handle.net/#{guid}"
|
|
53
|
+
elsif k == "uri" and regex.match(guid)
|
|
54
|
+
return "uri", guid
|
|
55
|
+
elsif k == "doi" and regex.match(guid)
|
|
56
|
+
return "doi", "https://doi.org/#{guid}"
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
[nil, nil]
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def self.typeit(guid:)
|
|
63
|
+
Utils::GUID_TYPES.each do |type, regex|
|
|
64
|
+
return type if regex.match(guid)
|
|
65
|
+
end
|
|
66
|
+
false
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def self.resolve_url(url:, nolinkheaders: false, header: ACCEPT_ALL_HEADER)
|
|
70
|
+
@meta.guidtype = "uri" if @meta.guidtype.nil?
|
|
71
|
+
warn "\n\n FETCHING #{url} #{header}\n\n"
|
|
72
|
+
response = FspHarvester::WebUtils.fspfetch(url: url, headers: header)
|
|
73
|
+
warn "\n\n head #{response.headers.inspect}\n\n"
|
|
74
|
+
|
|
75
|
+
unless response
|
|
76
|
+
@meta.warnings << ["001", url, header]
|
|
77
|
+
@meta.comments << "WARN: Unable to resolve #{url} using HTTP Accept header #{header}.\n"
|
|
78
|
+
return [[], @meta]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
@meta.comments << "INFO: following redirection using this header led to the following URL: #{@meta.finalURI.last}. Using the output from this URL for the next few tests..."
|
|
82
|
+
@meta.full_response << response.body
|
|
83
|
+
|
|
84
|
+
links = process_link_headers(response: response) unless nolinkheaders
|
|
85
|
+
[links, @meta]
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.process_link_headers(response:)
|
|
89
|
+
warn "\n\n parsing #{response.headers}\n\n"
|
|
90
|
+
|
|
91
|
+
parser = LinkHeaders::Processor.new(default_anchor: @meta.finalURI.last)
|
|
92
|
+
parser.extract_and_parse(response: response)
|
|
93
|
+
factory = parser.factory # LinkHeaders::LinkFactory
|
|
94
|
+
|
|
95
|
+
citeas = 0
|
|
96
|
+
describedby = 0
|
|
97
|
+
warn "\n\n length #{factory.all_links.length}\n\n"
|
|
98
|
+
|
|
99
|
+
factory.all_links.each do |l|
|
|
100
|
+
case l.relation
|
|
101
|
+
when "cite-as"
|
|
102
|
+
citeas += 1
|
|
103
|
+
when "describedby"
|
|
104
|
+
describedby += 1
|
|
105
|
+
unless l.respond_to? "type"
|
|
106
|
+
@meta.warnings << ["005", l.url, ""]
|
|
107
|
+
@meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute\n"
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
if citeas > 1
|
|
112
|
+
self.check_for_conflicts(factory: factory) # this merelty adsds to the metadata objects if there are conflicts
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
unless citeas == 1 && describedby > 0
|
|
116
|
+
@meta.warnings << ["004", "", ""]
|
|
117
|
+
@meta.comments << "WARN: The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header\n"
|
|
118
|
+
end
|
|
119
|
+
factory.all_links
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def self.check_for_conflicts(factory:) # incoming: {"link1" => {"sectiontype1" => value, "sectiontype2" => value2}}
|
|
123
|
+
@meta.comments << "INFO: checking for conflicting cite-as links"
|
|
124
|
+
citeas = Array.new
|
|
125
|
+
factory.all_links.each do |link|
|
|
126
|
+
next unless link.relation == 'cite-as'
|
|
127
|
+
citeas << link.href
|
|
128
|
+
end
|
|
129
|
+
unless citeas == citeas.uniq
|
|
130
|
+
@meta.warnings << ["007", url, header]
|
|
131
|
+
@meta.comments << "WARN: The resource does not follow the FAIR Signposting standard: Found conflicting cite-as link headers\n"
|
|
132
|
+
else
|
|
133
|
+
@meta.comments << "INFO: No conflicting cite-as links found."
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module FspHarvester
|
|
2
|
+
class MetadataObject
|
|
3
|
+
attr_accessor :hash, :graph, :comments, :warnings, :guidtype, :full_response, :finalURI # a hash of metadata # a RDF.rb graph of metadata # an array of comments # the type of GUID that was detected # will be an array of Net::HTTP::Response
|
|
4
|
+
|
|
5
|
+
def initialize(_params = {}) # get a name from the "new" call, or set a default
|
|
6
|
+
@hash = {}
|
|
7
|
+
@graph = RDF::Graph.new
|
|
8
|
+
@comments = []
|
|
9
|
+
@warnings = []
|
|
10
|
+
@full_response = []
|
|
11
|
+
@finalURI = []
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def merge_hash(hash)
|
|
15
|
+
# $stderr.puts "\n\n\nIncoming Hash #{hash.inspect}"
|
|
16
|
+
self.hash = self.hash.merge(hash)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def merge_rdf(triples) # incoming list of triples
|
|
20
|
+
graph << triples
|
|
21
|
+
graph
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def rdf
|
|
25
|
+
graph
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
data/lib/swagger.rb
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
class Swagger
|
|
2
|
+
attr_accessor :debug
|
|
3
|
+
attr_accessor :title
|
|
4
|
+
attr_accessor :tests_metric
|
|
5
|
+
attr_accessor :description
|
|
6
|
+
attr_accessor :applies_to_principle
|
|
7
|
+
attr_accessor :organization
|
|
8
|
+
attr_accessor :org_url
|
|
9
|
+
attr_accessor :responsible_developer
|
|
10
|
+
attr_accessor :email
|
|
11
|
+
attr_accessor :developer_ORCiD
|
|
12
|
+
attr_accessor :protocol
|
|
13
|
+
attr_accessor :host
|
|
14
|
+
attr_accessor :basePath
|
|
15
|
+
attr_accessor :path
|
|
16
|
+
attr_accessor :response_description
|
|
17
|
+
attr_accessor :schemas
|
|
18
|
+
attr_accessor :comments
|
|
19
|
+
attr_accessor :fairsharing_key_location
|
|
20
|
+
attr_accessor :score
|
|
21
|
+
attr_accessor :testedGUID
|
|
22
|
+
|
|
23
|
+
def initialize(params = {})
|
|
24
|
+
@debug = params.fetch(:debug, false)
|
|
25
|
+
|
|
26
|
+
@title = params.fetch(:title, 'unnamed')
|
|
27
|
+
@tests_metric = params.fetch(:tests_metric)
|
|
28
|
+
@description = params.fetch(:description, 'default_description')
|
|
29
|
+
@applies_to_principle = params.fetch(:applies_to_principle, 'some principle')
|
|
30
|
+
@version = params.fetch(:version, "0.1")
|
|
31
|
+
@organization = params.fetch(:organization, 'Some Organization')
|
|
32
|
+
@org_url = params.fetch(:org_url)
|
|
33
|
+
@responsible_develper = params.fetch(:responsible_developer, 'Some Person')
|
|
34
|
+
@email = params.fetch(:email)
|
|
35
|
+
@developer_ORCiD = params.fetch(:developer_ORCiD)
|
|
36
|
+
@host = params.fetch(:host)
|
|
37
|
+
@protocol = params.fetch(:protocol, "https")
|
|
38
|
+
@basePath = params.fetch(:basePath)
|
|
39
|
+
@path = params.fetch(:path)
|
|
40
|
+
@response_description = params.fetch(:response_description)
|
|
41
|
+
@schemas = params.fetch(:schemas, [])
|
|
42
|
+
@comments = params.fetch(:comments, [])
|
|
43
|
+
@fairsharing_key_location = params.fetch(:fairsharing_key_location)
|
|
44
|
+
@score = params.fetch(:score, 0)
|
|
45
|
+
@testedGUID = params.fetch(:testedGUID, "")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def fairsharing_key
|
|
54
|
+
return @fairsharing_key_location
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def getSwagger
|
|
60
|
+
|
|
61
|
+
message = <<"EOF_EOF"
|
|
62
|
+
swagger: '2.0'
|
|
63
|
+
info:
|
|
64
|
+
version: '#{@version}'
|
|
65
|
+
title: "#{@title}"
|
|
66
|
+
x-tests_metric: '#{@tests_metric}'
|
|
67
|
+
description: >-
|
|
68
|
+
#{@description}
|
|
69
|
+
x-applies_to_principle: "#{@applies_to_principle}"
|
|
70
|
+
contact:
|
|
71
|
+
x-organization: "#{@organization}"
|
|
72
|
+
url: "#{@org_url}"
|
|
73
|
+
name: '#{@responsible_develper}'
|
|
74
|
+
x-role: "responsible developer"
|
|
75
|
+
email: #{@email}
|
|
76
|
+
x-id: '#{developer_ORCiD}'
|
|
77
|
+
host: #{@host}
|
|
78
|
+
basePath: #{@basePath}
|
|
79
|
+
schemes:
|
|
80
|
+
- #{@protocol}
|
|
81
|
+
paths:
|
|
82
|
+
#{@path}:
|
|
83
|
+
post:
|
|
84
|
+
parameters:
|
|
85
|
+
- name: content
|
|
86
|
+
in: body
|
|
87
|
+
required: true
|
|
88
|
+
schema:
|
|
89
|
+
$ref: '#/definitions/schemas'
|
|
90
|
+
consumes:
|
|
91
|
+
- application/json
|
|
92
|
+
produces:
|
|
93
|
+
- application/json
|
|
94
|
+
responses:
|
|
95
|
+
"200":
|
|
96
|
+
description: >-
|
|
97
|
+
#{@response_description}
|
|
98
|
+
definitions:
|
|
99
|
+
schemas:
|
|
100
|
+
required:
|
|
101
|
+
EOF_EOF
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
self.schemas.keys.each do |key|
|
|
106
|
+
message += " - #{key}\n"
|
|
107
|
+
end
|
|
108
|
+
message += " properties:\n"
|
|
109
|
+
self.schemas.keys.each do |key|
|
|
110
|
+
message += " #{key}:\n"
|
|
111
|
+
message += " type: #{self.schemas[key][0]}\n"
|
|
112
|
+
message += " description: >-\n"
|
|
113
|
+
message += " #{self.schemas[key][1]}\n"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
return message
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# A utility function that SHOULD NOT BE CALLED EXTERNALLY
|
|
122
|
+
#
|
|
123
|
+
# @param s - subject node
|
|
124
|
+
# @param p - predicate node
|
|
125
|
+
# @param o - object node
|
|
126
|
+
# @param repo - an RDF::Graph object
|
|
127
|
+
def triplify(s, p, o, repo)
|
|
128
|
+
|
|
129
|
+
if s.class == String
|
|
130
|
+
s = s.strip
|
|
131
|
+
end
|
|
132
|
+
if p.class == String
|
|
133
|
+
p = p.strip
|
|
134
|
+
end
|
|
135
|
+
if o.class == String
|
|
136
|
+
o = o.strip
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
unless s.respond_to?('uri')
|
|
140
|
+
|
|
141
|
+
if s.to_s =~ /^\w+:\/?\/?[^\s]+/
|
|
142
|
+
s = RDF::URI.new(s.to_s)
|
|
143
|
+
else
|
|
144
|
+
self.debug and $stderr.puts "Subject #{s.to_s} must be a URI-compatible thingy"
|
|
145
|
+
abort "Subject #{s.to_s} must be a URI-compatible thingy"
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
unless p.respond_to?('uri')
|
|
150
|
+
|
|
151
|
+
if p.to_s =~ /^\w+:\/?\/?[^\s]+/
|
|
152
|
+
p = RDF::URI.new(p.to_s)
|
|
153
|
+
else
|
|
154
|
+
self.debug and $stderr.puts "Predicate #{p.to_s} must be a URI-compatible thingy"
|
|
155
|
+
abort "Predicate #{p.to_s} must be a URI-compatible thingy"
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
unless o.respond_to?('uri')
|
|
160
|
+
if o.to_s =~ /\A\w+:\/?\/?\w[^\s]+/
|
|
161
|
+
o = RDF::URI.new(o.to_s)
|
|
162
|
+
elsif o.to_s =~ /^\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d/
|
|
163
|
+
o = RDF::Literal.new(o.to_s, :datatype => RDF::XSD.date)
|
|
164
|
+
elsif o.to_s =~ /^[+-]?\d+\.\d+/
|
|
165
|
+
o = RDF::Literal.new(o.to_s, :datatype => RDF::XSD.float)
|
|
166
|
+
elsif o.to_s =~ /^[+-]?[0-9]+$/
|
|
167
|
+
o = RDF::Literal.new(o.to_s, :datatype => RDF::XSD.int)
|
|
168
|
+
else
|
|
169
|
+
o = RDF::Literal.new(o.to_s, :language => :en)
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
self.debug and $stderr.puts("\n\ninserting #{s.to_s} #{p.to_s} #{o.to_s}\n\n")
|
|
174
|
+
triple = RDF::Statement(s, p, o)
|
|
175
|
+
repo.insert(triple)
|
|
176
|
+
|
|
177
|
+
return true
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
# A utility function that SHOULD NOT BE CALLED EXTERNALLY
|
|
182
|
+
#
|
|
183
|
+
# @param s - subject node
|
|
184
|
+
# @param p - predicate node
|
|
185
|
+
# @param o - object node
|
|
186
|
+
# @param repo - an RDF::Graph object
|
|
187
|
+
def Swagger.triplify(s, p, o, repo)
|
|
188
|
+
return triplify(s,p,o,repo)
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def addComment(newcomment)
|
|
192
|
+
self.comments << newcomment.to_s
|
|
193
|
+
#return self.comments
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def createEvaluationResponse
|
|
197
|
+
|
|
198
|
+
g = RDF::Graph.new
|
|
199
|
+
|
|
200
|
+
dt = Time.now.iso8601
|
|
201
|
+
uri = self.testedGUID
|
|
202
|
+
|
|
203
|
+
me = self.protocol + "://" + self.host + "/" + self.basePath + self.path
|
|
204
|
+
|
|
205
|
+
meURI ="#{me}##{uri}/result-#{dt}"
|
|
206
|
+
meURI =Addressable::URI.escape(meURI)
|
|
207
|
+
|
|
208
|
+
triplify(meURI, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "http://fairmetrics.org/resources/metric_evaluation_result", g );
|
|
209
|
+
triplify(meURI, "http://semanticscience.org/resource/SIO_000300", self.score, g )
|
|
210
|
+
triplify(meURI, "http://purl.obolibrary.org/obo/date", dt, g )
|
|
211
|
+
triplify(meURI, "http://schema.org/softwareVersion", VERSION, g )
|
|
212
|
+
triplify(meURI,"http://semanticscience.org/resource/SIO_000332", uri, g)
|
|
213
|
+
|
|
214
|
+
comments = "no comments received. "
|
|
215
|
+
|
|
216
|
+
comments = self.comments.join("\n") if self.comments.size > 0
|
|
217
|
+
triplify(meURI, "http://schema.org/comment", comments, g)
|
|
218
|
+
|
|
219
|
+
return g.dump(:jsonld)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
|
data/lib/warnings.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"001": {
|
|
3
|
+
"message": "Unable to resolve guid using Accept headers for Linked Data",
|
|
4
|
+
"linkout": "",
|
|
5
|
+
"severity": "WARN"
|
|
6
|
+
},
|
|
7
|
+
"002": {
|
|
8
|
+
"message": "HTTP Response (203) is non-authoritative",
|
|
9
|
+
"linkout": "",
|
|
10
|
+
"severity": "WARN"
|
|
11
|
+
},
|
|
12
|
+
"003": {
|
|
13
|
+
"message": "HTTP Response indicates failure (500-range)",
|
|
14
|
+
"linkout": "",
|
|
15
|
+
"severity": "WARN"
|
|
16
|
+
},
|
|
17
|
+
"004": {
|
|
18
|
+
"message": "The resource does not follow the FAIR Signposting standard, which requires exactly one cite-as header, and at least one describedby header",
|
|
19
|
+
"linkout": "",
|
|
20
|
+
"severity": "WARN"
|
|
21
|
+
},
|
|
22
|
+
"005": {
|
|
23
|
+
"message": "The resource does not follow the FAIR Signposting standard, which requires any describedby links to also have a 'type' attribute indicating the Accept headers that should be sent with the request",
|
|
24
|
+
"linkout": "",
|
|
25
|
+
"severity": "WARN"
|
|
26
|
+
},
|
|
27
|
+
"006": {
|
|
28
|
+
"message": "GUID type not recognized",
|
|
29
|
+
"linkout": "",
|
|
30
|
+
"severity": "WARN"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
}
|
data/lib/web_utils.rb
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
module FspHarvester
|
|
2
|
+
|
|
3
|
+
class WebUtils
|
|
4
|
+
def self.fspfetch(url:, headers: ACCEPT_ALL_HEADER) # we will try to retrieve turtle whenever possible
|
|
5
|
+
warn 'In fetch routine now. '
|
|
6
|
+
|
|
7
|
+
begin
|
|
8
|
+
warn "executing call over the Web to #{url}"
|
|
9
|
+
response = RestClient::Request.execute({
|
|
10
|
+
method: :get,
|
|
11
|
+
url: url.to_s,
|
|
12
|
+
# user: user,
|
|
13
|
+
# password: pass,
|
|
14
|
+
headers: headers
|
|
15
|
+
})
|
|
16
|
+
@meta.finalURI |= [response.request.url] if @meta # it's possible to call this method without affecting the metadata object being created by the harvester
|
|
17
|
+
warn "There was a response to the call #{url}"
|
|
18
|
+
warn "There was a response to the call #{response.request.url}"
|
|
19
|
+
warn "Response code #{response.code}"
|
|
20
|
+
if response.code == 203 && @meta
|
|
21
|
+
@meta.warnings << ["002", url, headers]
|
|
22
|
+
@meta.comments << "WARN: Response is non-authoritative (HTTP response code: #{response.code}). Headers may have been manipulated encountered when trying to resolve #{url}\n"
|
|
23
|
+
end
|
|
24
|
+
response
|
|
25
|
+
rescue RestClient::ExceptionWithResponse => e
|
|
26
|
+
warn "EXCEPTION WITH RESPONSE! #{e.response}\n#{e.response.headers}"
|
|
27
|
+
@meta.warnings << ["003", url, headers] if @meta
|
|
28
|
+
@meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n" if @meta
|
|
29
|
+
if e.response.code == 500
|
|
30
|
+
return false
|
|
31
|
+
else
|
|
32
|
+
e.response
|
|
33
|
+
end
|
|
34
|
+
# now we are returning the headers and body that were returned
|
|
35
|
+
rescue RestClient::Exception => e
|
|
36
|
+
warn "EXCEPTION WITH NO RESPONSE! #{e}"
|
|
37
|
+
@meta.warnings << ["003", url, headers] if @meta
|
|
38
|
+
@meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n" if @meta
|
|
39
|
+
false
|
|
40
|
+
# now we are returning 'False', and we will check that with an \"if\" statement in our main code
|
|
41
|
+
rescue Exception => e
|
|
42
|
+
warn "EXCEPTION UNKNOWN! #{e}"
|
|
43
|
+
@meta.warnings << ["003", url, headers] if @meta
|
|
44
|
+
@meta.comments << "WARN: HTTP error #{e} encountered when trying to resolve #{url}\n" if @meta
|
|
45
|
+
false
|
|
46
|
+
# now we are returning 'False', and we will check that with an \"if\" statement in our main code
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: fsp_harvester
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.2
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Mark Wilkinson
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2022-07-27 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: json
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '2.0'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '2.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: linkeddata
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '3.2'
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '3.2'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: linkheaders-processor
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 0.1.9
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 0.1.9
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: metainspector
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 5.11.2
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 5.11.2
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: parseconfig
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '1.1'
|
|
76
|
+
type: :runtime
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '1.1'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: rake
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '13.0'
|
|
90
|
+
type: :runtime
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - "~>"
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '13.0'
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: rest-client
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '2.1'
|
|
104
|
+
type: :runtime
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '2.1'
|
|
111
|
+
- !ruby/object:Gem::Dependency
|
|
112
|
+
name: rspec
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - "~>"
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '3.11'
|
|
118
|
+
type: :runtime
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - "~>"
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '3.11'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: rubocop
|
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
|
128
|
+
requirements:
|
|
129
|
+
- - "~>"
|
|
130
|
+
- !ruby/object:Gem::Version
|
|
131
|
+
version: '1.7'
|
|
132
|
+
type: :runtime
|
|
133
|
+
prerelease: false
|
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
135
|
+
requirements:
|
|
136
|
+
- - "~>"
|
|
137
|
+
- !ruby/object:Gem::Version
|
|
138
|
+
version: '1.7'
|
|
139
|
+
- !ruby/object:Gem::Dependency
|
|
140
|
+
name: securerandom
|
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
|
142
|
+
requirements:
|
|
143
|
+
- - "~>"
|
|
144
|
+
- !ruby/object:Gem::Version
|
|
145
|
+
version: 0.1.0
|
|
146
|
+
type: :runtime
|
|
147
|
+
prerelease: false
|
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
149
|
+
requirements:
|
|
150
|
+
- - "~>"
|
|
151
|
+
- !ruby/object:Gem::Version
|
|
152
|
+
version: 0.1.0
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: xml-simple
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - "~>"
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: '1.1'
|
|
160
|
+
type: :runtime
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - "~>"
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: '1.1'
|
|
167
|
+
description: Metadata harvester that follows the FAIR Signposting specification.
|
|
168
|
+
email:
|
|
169
|
+
- markw@illuminae.com
|
|
170
|
+
executables: []
|
|
171
|
+
extensions: []
|
|
172
|
+
extra_rdoc_files: []
|
|
173
|
+
files:
|
|
174
|
+
- CHANGELOG.md
|
|
175
|
+
- Gemfile
|
|
176
|
+
- Gemfile.lock
|
|
177
|
+
- LICENSE
|
|
178
|
+
- README.md
|
|
179
|
+
- Rakefile
|
|
180
|
+
- bin/console
|
|
181
|
+
- bin/setup
|
|
182
|
+
- example_test.rb
|
|
183
|
+
- lib/constants.rb
|
|
184
|
+
- lib/fsp_harvester.rb
|
|
185
|
+
- lib/fsp_harvester/version.rb
|
|
186
|
+
- lib/metadata_object.rb
|
|
187
|
+
- lib/swagger.rb
|
|
188
|
+
- lib/warnings.json
|
|
189
|
+
- lib/web_utils.rb
|
|
190
|
+
homepage: https://github.com/markwilkinson/FAIR-Signposting-Harvester
|
|
191
|
+
licenses:
|
|
192
|
+
- MIT
|
|
193
|
+
metadata:
|
|
194
|
+
allowed_push_host: https://rubygems.org
|
|
195
|
+
homepage_uri: https://github.com/markwilkinson/FAIR-Signposting-Harvester
|
|
196
|
+
source_code_uri: https://github.com/markwilkinson/FAIR-Signposting-Harvester
|
|
197
|
+
changelog_uri: https://github.com/markwilkinson/FAIR-Signposting-Harvester
|
|
198
|
+
post_install_message:
|
|
199
|
+
rdoc_options: []
|
|
200
|
+
require_paths:
|
|
201
|
+
- lib
|
|
202
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
203
|
+
requirements:
|
|
204
|
+
- - ">="
|
|
205
|
+
- !ruby/object:Gem::Version
|
|
206
|
+
version: 3.0.0
|
|
207
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
208
|
+
requirements:
|
|
209
|
+
- - ">="
|
|
210
|
+
- !ruby/object:Gem::Version
|
|
211
|
+
version: '0'
|
|
212
|
+
requirements: []
|
|
213
|
+
rubygems_version: 3.2.28
|
|
214
|
+
signing_key:
|
|
215
|
+
specification_version: 4
|
|
216
|
+
summary: Metadata harvester that follows the FAIR Signposting specification.
|
|
217
|
+
test_files: []
|