qa 0.10.1 → 0.10.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +12 -2
- data/app/controllers/qa/terms_controller.rb +14 -16
- data/app/models/qa/mesh_tree.rb +3 -5
- data/app/models/qa/subject_mesh_term.rb +15 -15
- data/config/routes.rb +1 -1
- data/lib/generators/qa/install/install_generator.rb +2 -3
- data/lib/generators/qa/local/tables/mysql/mysql_generator.rb +3 -5
- data/lib/generators/qa/local/tables/tables_generator.rb +4 -6
- data/lib/qa/authorities/assign_fast/generic_authority.rb +16 -19
- data/lib/qa/authorities/assign_fast_subauthority.rb +3 -5
- data/lib/qa/authorities/authority_with_sub_authority.rb +1 -2
- data/lib/qa/authorities/base.rb +2 -3
- data/lib/qa/authorities/geonames.rb +11 -12
- data/lib/qa/authorities/getty.rb +7 -8
- data/lib/qa/authorities/getty/aat.rb +11 -12
- data/lib/qa/authorities/getty/tgn.rb +15 -15
- data/lib/qa/authorities/getty/ulan.rb +15 -16
- data/lib/qa/authorities/loc/generic_authority.rb +32 -33
- data/lib/qa/authorities/loc_subauthority.rb +17 -20
- data/lib/qa/authorities/local.rb +41 -40
- data/lib/qa/authorities/local/file_based_authority.rb +18 -19
- data/lib/qa/authorities/local/mysql_table_based_authority.rb +8 -6
- data/lib/qa/authorities/local/registry.rb +5 -5
- data/lib/qa/authorities/local/table_based_authority.rb +26 -19
- data/lib/qa/authorities/mesh.rb +8 -16
- data/lib/qa/authorities/mesh_tools.rb +5 -5
- data/lib/qa/authorities/mesh_tools/mesh_data_parser.rb +4 -6
- data/lib/qa/authorities/mesh_tools/mesh_importer.rb +11 -13
- data/lib/qa/authorities/oclcts.rb +0 -1
- data/lib/qa/authorities/oclcts/generic_oclc_authority.rb +16 -16
- data/lib/qa/authorities/tgnlang.rb +7 -12
- data/lib/qa/version.rb +1 -1
- data/spec/controllers/terms_controller_spec.rb +44 -54
- data/spec/lib/authorities/assign_fast_spec.rb +25 -27
- data/spec/lib/authorities/file_based_authority_spec.rb +25 -26
- data/spec/lib/authorities/geonames_spec.rb +5 -6
- data/spec/lib/authorities/getty/aat_spec.rb +6 -10
- data/spec/lib/authorities/getty/tgn_spec.rb +6 -10
- data/spec/lib/authorities/getty/ulan_spec.rb +6 -10
- data/spec/lib/authorities/getty_spec.rb +4 -5
- data/spec/lib/authorities/loc_spec.rb +30 -36
- data/spec/lib/authorities/local_spec.rb +5 -7
- data/spec/lib/authorities/mesh_spec.rb +9 -9
- data/spec/lib/authorities/mysql_table_based_authority_spec.rb +13 -5
- data/spec/lib/authorities/oclcts_spec.rb +17 -21
- data/spec/lib/authorities/table_based_authority_spec.rb +21 -12
- data/spec/lib/authorities/tgnlang_spec.rb +4 -6
- data/spec/lib/authorities_loc_subauthorities.rb +50 -54
- data/spec/lib/mesh_data_parser_spec.rb +73 -79
- data/spec/lib/services/rdf_authority_parser_spec.rb +2 -7
- data/spec/lib/tasks/mesh.rake_spec.rb +16 -12
- data/spec/models/subject_mesh_term_spec.rb +4 -4
- data/spec/routing/route_spec.rb +13 -15
- data/spec/spec_helper.rb +3 -4
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +0 -1
- metadata +45 -17
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::AAT < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,8 +11,7 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
def build_query_url
|
15
|
-
query = URI.escape(sparql(untaint(q)))
|
14
|
+
def build_query_url(q)
|
16
15
|
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q))}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
17
16
|
end
|
18
17
|
|
@@ -25,32 +24,32 @@ module Qa::Authorities
|
|
25
24
|
gvp:prefLabelGVP [skosxl:literalForm ?name].
|
26
25
|
FILTER regex(?name, \"#{search}\", \"i\") .
|
27
26
|
} ORDER BY ?name"
|
27
|
+
sparql
|
28
28
|
end
|
29
29
|
|
30
30
|
def untaint(q)
|
31
31
|
q.gsub(/[^\w\s-]/, '')
|
32
32
|
end
|
33
33
|
|
34
|
-
def find
|
34
|
+
def find(id)
|
35
35
|
json(find_url(id))
|
36
36
|
end
|
37
37
|
|
38
|
-
def find_url
|
38
|
+
def find_url(id)
|
39
39
|
"http://vocab.getty.edu/aat/#{id}.json"
|
40
40
|
end
|
41
41
|
|
42
42
|
def request_options
|
43
|
-
{ accept: 'application/sparql-results+json'}
|
43
|
+
{ accept: 'application/sparql-results+json' }
|
44
44
|
end
|
45
45
|
|
46
46
|
private
|
47
47
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
# Reformats the data received from the Getty service
|
49
|
+
def parse_authority_response(response)
|
50
|
+
response['results']['bindings'].map do |result|
|
51
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] }
|
52
|
+
end
|
52
53
|
end
|
53
|
-
end
|
54
|
-
|
55
54
|
end
|
56
55
|
end
|
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::TGN < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,11 +11,11 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
def build_query_url
|
14
|
+
def build_query_url(q)
|
15
15
|
query = URI.escape(sparql(untaint(q)))
|
16
16
|
# Replace ampersands, otherwise the query will fail
|
17
17
|
# Gsub hack to convert the encoded regex in the REPLACE into a form Getty understands
|
18
|
-
"http://vocab.getty.edu/sparql.json?query=#{query.gsub('&','%26').gsub(',[%5E,]+,[%5E,]+$','%2C[^%2C]%2B%2C[^%2C]%2B%24')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
18
|
+
"http://vocab.getty.edu/sparql.json?query=#{query.gsub('&', '%26').gsub(',[%5E,]+,[%5E,]+$', '%2C[^%2C]%2B%2C[^%2C]%2B%24')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
19
19
|
end
|
20
20
|
|
21
21
|
# Use a regex to exclude the continent and 'world' from the query
|
@@ -25,12 +25,12 @@ module Qa::Authorities
|
|
25
25
|
search = untaint(q)
|
26
26
|
if search.include?(' ')
|
27
27
|
ex = "(("
|
28
|
-
search.split(' ').each do |
|
28
|
+
search.split(' ').each do |i|
|
29
29
|
ex += "regex(CONCAT(?name, ', ', REPLACE(str(?par), \",[^,]+,[^,]+$\", \"\")), \"#{i}\",\"i\" ) && "
|
30
30
|
end
|
31
31
|
ex = ex[0..ex.length - 4]
|
32
32
|
ex += ') && ('
|
33
|
-
search.split(' ').each do |
|
33
|
+
search.split(' ').each do |i|
|
34
34
|
ex += "regex(?name, \"#{i}\",\"i\" ) || "
|
35
35
|
end
|
36
36
|
ex = ex[0..ex.length - 4]
|
@@ -48,33 +48,33 @@ module Qa::Authorities
|
|
48
48
|
gvp:parentString ?par .
|
49
49
|
FILTER #{ex} .
|
50
50
|
} ORDER BY ?name ASC(?par)"
|
51
|
+
sparql
|
51
52
|
end
|
52
53
|
|
53
54
|
def untaint(q)
|
54
55
|
q.gsub(/[^\w\s-]/, '')
|
55
56
|
end
|
56
57
|
|
57
|
-
def find
|
58
|
+
def find(id)
|
58
59
|
json(find_url(id))
|
59
60
|
end
|
60
61
|
|
61
|
-
def find_url
|
62
|
+
def find_url(id)
|
62
63
|
"http://vocab.getty.edu/tgn/#{id}.json"
|
63
64
|
end
|
64
65
|
|
65
66
|
def request_options
|
66
|
-
{ accept: 'application/sparql-results+json'}
|
67
|
+
{ accept: 'application/sparql-results+json' }
|
67
68
|
end
|
68
69
|
|
69
70
|
private
|
70
71
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
72
|
+
# Reformats the data received from the service
|
73
|
+
# Adds the parentString, minus the contintent and 'World' for disambiguation
|
74
|
+
def parse_authority_response(response)
|
75
|
+
response['results']['bindings'].map do |result|
|
76
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] + ' (' + result['par']['value'].gsub(/\,[^\,]+\,[^\,]+$/, '') + ')' }
|
77
|
+
end
|
76
78
|
end
|
77
|
-
end
|
78
|
-
|
79
79
|
end
|
80
80
|
end
|
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::Ulan < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,10 +11,9 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q)).gsub('&','%26')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
14
|
+
# Replace ampersands, otherwise the query will fail
|
15
|
+
def build_query_url(q)
|
16
|
+
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q)).gsub('&', '%26')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
18
17
|
end
|
19
18
|
|
20
19
|
def sparql(q)
|
@@ -22,7 +21,7 @@ module Qa::Authorities
|
|
22
21
|
# if more than one term is supplied, check both preferred and alt labels
|
23
22
|
if search.include?(' ')
|
24
23
|
ex = "("
|
25
|
-
search.split(' ').each do |
|
24
|
+
search.split(' ').each do |i|
|
26
25
|
ex += "regex(CONCAT(?name, ' ', ?alt), \"#{i}\",\"i\" ) && "
|
27
26
|
end
|
28
27
|
ex = ex[0..ex.length - 4]
|
@@ -39,33 +38,33 @@ module Qa::Authorities
|
|
39
38
|
skos:altLabel ?alt .
|
40
39
|
FILTER #{ex} .
|
41
40
|
} ORDER BY ?name"
|
41
|
+
sparql
|
42
42
|
end
|
43
43
|
|
44
44
|
def untaint(q)
|
45
45
|
q.gsub(/[^\w\s-]/, '')
|
46
46
|
end
|
47
47
|
|
48
|
-
def find
|
48
|
+
def find(id)
|
49
49
|
json(find_url(id))
|
50
50
|
end
|
51
51
|
|
52
|
-
def find_url
|
52
|
+
def find_url(id)
|
53
53
|
"http://vocab.getty.edu/ulan/#{id}.json"
|
54
54
|
end
|
55
55
|
|
56
56
|
def request_options
|
57
|
-
{ accept: 'application/sparql-results+json'}
|
57
|
+
{ accept: 'application/sparql-results+json' }
|
58
58
|
end
|
59
59
|
|
60
60
|
private
|
61
61
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
# Reformats the data received from the Getty service
|
63
|
+
# Add the bio for disambiguation
|
64
|
+
def parse_authority_response(response)
|
65
|
+
response['results']['bindings'].map do |result|
|
66
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] + ' (' + result['bio']['value'] + ')' }
|
67
|
+
end
|
67
68
|
end
|
68
|
-
end
|
69
|
-
|
70
69
|
end
|
71
70
|
end
|
@@ -7,65 +7,64 @@ module Qa::Authorities
|
|
7
7
|
|
8
8
|
include WebServiceBase
|
9
9
|
|
10
|
-
def search
|
10
|
+
def search(q)
|
11
11
|
@raw_response = get_json(build_query_url(q))
|
12
12
|
parse_authority_response
|
13
13
|
end
|
14
14
|
|
15
|
-
def build_query_url
|
15
|
+
def build_query_url(q)
|
16
16
|
escaped_query = URI.escape(q)
|
17
17
|
authority_fragment = Loc.get_url_for_authority(subauthority) + URI.escape(subauthority)
|
18
|
-
|
18
|
+
"http://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
|
19
19
|
end
|
20
20
|
|
21
|
-
def find
|
21
|
+
def find(id)
|
22
22
|
get_json(find_url(id))
|
23
23
|
end
|
24
24
|
|
25
|
-
def find_url
|
25
|
+
def find_url(id)
|
26
26
|
"http://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
|
27
27
|
end
|
28
28
|
|
29
29
|
private
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
# Reformats the data received from the LOC service
|
32
|
+
def parse_authority_response
|
33
|
+
@raw_response.select { |response| response[0] == "atom:entry" }.map do |response|
|
34
|
+
loc_response_to_qa(response_to_struct(response))
|
35
|
+
end
|
35
36
|
end
|
36
|
-
end
|
37
37
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
38
|
+
# Converts most of the atom data into an OpenStruct object.
|
39
|
+
#
|
40
|
+
# Note that this is a pretty naive conversion. There should probably just
|
41
|
+
# be a class that properly translates and stores the various pieces of
|
42
|
+
# data, especially if this logic could be useful in other auth lookups.
|
43
|
+
def response_to_struct(response)
|
44
|
+
contents = response.each_with_object({}) do |result_parts, result|
|
45
|
+
next unless result_parts[0]
|
46
|
+
key = result_parts[0].sub('atom:', '').sub('dcterms:', '')
|
47
|
+
info = result_parts[1]
|
48
|
+
val = result_parts[2]
|
49
49
|
|
50
|
-
|
50
|
+
case key
|
51
51
|
when 'title', 'id', 'name', 'updated', 'created'
|
52
52
|
result[key] = val
|
53
53
|
when 'link'
|
54
54
|
result["links"] ||= []
|
55
55
|
result["links"] << [info["type"], info["href"]]
|
56
|
+
end
|
56
57
|
end
|
57
|
-
end
|
58
|
-
|
59
|
-
OpenStruct.new(result)
|
60
|
-
end
|
61
58
|
|
62
|
-
|
63
|
-
|
64
|
-
{
|
65
|
-
"id" => data.id || data.title,
|
66
|
-
"label" => data.title
|
67
|
-
}
|
68
|
-
end
|
59
|
+
OpenStruct.new(contents)
|
60
|
+
end
|
69
61
|
|
62
|
+
# Simple conversion from LoC-based struct to QA hash
|
63
|
+
def loc_response_to_qa(data)
|
64
|
+
{
|
65
|
+
"id" => data.id || data.title,
|
66
|
+
"label" => data.title
|
67
|
+
}
|
68
|
+
end
|
70
69
|
end
|
71
70
|
end
|
@@ -1,11 +1,9 @@
|
|
1
1
|
module Qa::Authorities::LocSubauthority
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
when datatypes.include?(authority) then datatype_base_url
|
8
|
-
when preservation.include?(authority) then vocab_preservation_base_url
|
2
|
+
def get_url_for_authority(authority)
|
3
|
+
if authorities.include?(authority) then authority_base_url
|
4
|
+
elsif vocabularies.include?(authority) then vocab_base_url
|
5
|
+
elsif datatypes.include?(authority) then datatype_base_url
|
6
|
+
elsif preservation.include?(authority) then vocab_preservation_base_url
|
9
7
|
end
|
10
8
|
end
|
11
9
|
|
@@ -71,20 +69,19 @@ module Qa::Authorities::LocSubauthority
|
|
71
69
|
|
72
70
|
private
|
73
71
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
def authority_base_url
|
79
|
-
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F"
|
80
|
-
end
|
72
|
+
def vocab_base_url
|
73
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2F"
|
74
|
+
end
|
81
75
|
|
82
|
-
|
83
|
-
|
84
|
-
|
76
|
+
def authority_base_url
|
77
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F"
|
78
|
+
end
|
85
79
|
|
86
|
-
|
87
|
-
|
88
|
-
|
80
|
+
def datatype_base_url
|
81
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fdatatypes%2F"
|
82
|
+
end
|
89
83
|
|
84
|
+
def vocab_preservation_base_url
|
85
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2Fpreservation%2F"
|
86
|
+
end
|
90
87
|
end
|
data/lib/qa/authorities/local.rb
CHANGED
@@ -7,58 +7,59 @@ module Qa::Authorities
|
|
7
7
|
autoload :TableBasedAuthority
|
8
8
|
autoload :MysqlTableBasedAuthority
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
end
|
10
|
+
class << self
|
11
|
+
attr_reader :config
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
def load_config(file)
|
14
|
+
@config = YAML.load_file(file)
|
15
|
+
end
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
# Path to sub-authority files is either the full path to a directory or
|
18
|
+
# the path to a directory relative to the Rails application
|
19
|
+
def subauthorities_path
|
20
|
+
if config[:local_path].starts_with?(File::Separator)
|
21
|
+
config[:local_path]
|
22
|
+
else
|
23
|
+
File.join(Rails.root, config[:local_path])
|
24
|
+
end
|
25
25
|
end
|
26
|
-
end
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
# Local sub-authorities are any YAML files in the subauthorities_path
|
28
|
+
def names
|
29
|
+
unless Dir.exist? subauthorities_path
|
30
|
+
raise Qa::ConfigDirectoryNotFound, "There's no directory at #{subauthorities_path}. You must create it in order to use local authorities"
|
31
|
+
end
|
32
|
+
Dir.entries(subauthorities_path).map { |f| File.basename(f, ".yml") if f =~ /yml$/ }.compact
|
32
33
|
end
|
33
|
-
Dir.entries(subauthorities_path).map { |f| File.basename(f, ".yml") if f.match(/yml$/) }.compact
|
34
|
-
end
|
35
34
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
35
|
+
def subauthority_for(subauthority)
|
36
|
+
validate_subauthority!(subauthority)
|
37
|
+
registry.instance_for(subauthority)
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
def registry
|
41
|
+
@registry ||= begin
|
42
|
+
Registry.new do |reg|
|
43
|
+
register_defaults(reg)
|
44
|
+
end
|
45
45
|
end
|
46
46
|
end
|
47
|
-
end
|
48
47
|
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
def register_subauthority(subauthority, class_name)
|
49
|
+
registry.add(subauthority, class_name)
|
50
|
+
end
|
52
51
|
|
53
|
-
|
54
|
-
|
55
|
-
|
52
|
+
def subauthorities
|
53
|
+
registry.keys
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
56
57
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
58
|
+
def register_defaults(reg)
|
59
|
+
names.each do |name|
|
60
|
+
reg.add(name, 'Qa::Authorities::Local::FileBasedAuthority')
|
61
|
+
end
|
61
62
|
end
|
62
|
-
|
63
|
+
end
|
63
64
|
end
|
64
65
|
end
|