qa 0.10.1 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +12 -2
- data/app/controllers/qa/terms_controller.rb +14 -16
- data/app/models/qa/mesh_tree.rb +3 -5
- data/app/models/qa/subject_mesh_term.rb +15 -15
- data/config/routes.rb +1 -1
- data/lib/generators/qa/install/install_generator.rb +2 -3
- data/lib/generators/qa/local/tables/mysql/mysql_generator.rb +3 -5
- data/lib/generators/qa/local/tables/tables_generator.rb +4 -6
- data/lib/qa/authorities/assign_fast/generic_authority.rb +16 -19
- data/lib/qa/authorities/assign_fast_subauthority.rb +3 -5
- data/lib/qa/authorities/authority_with_sub_authority.rb +1 -2
- data/lib/qa/authorities/base.rb +2 -3
- data/lib/qa/authorities/geonames.rb +11 -12
- data/lib/qa/authorities/getty.rb +7 -8
- data/lib/qa/authorities/getty/aat.rb +11 -12
- data/lib/qa/authorities/getty/tgn.rb +15 -15
- data/lib/qa/authorities/getty/ulan.rb +15 -16
- data/lib/qa/authorities/loc/generic_authority.rb +32 -33
- data/lib/qa/authorities/loc_subauthority.rb +17 -20
- data/lib/qa/authorities/local.rb +41 -40
- data/lib/qa/authorities/local/file_based_authority.rb +18 -19
- data/lib/qa/authorities/local/mysql_table_based_authority.rb +8 -6
- data/lib/qa/authorities/local/registry.rb +5 -5
- data/lib/qa/authorities/local/table_based_authority.rb +26 -19
- data/lib/qa/authorities/mesh.rb +8 -16
- data/lib/qa/authorities/mesh_tools.rb +5 -5
- data/lib/qa/authorities/mesh_tools/mesh_data_parser.rb +4 -6
- data/lib/qa/authorities/mesh_tools/mesh_importer.rb +11 -13
- data/lib/qa/authorities/oclcts.rb +0 -1
- data/lib/qa/authorities/oclcts/generic_oclc_authority.rb +16 -16
- data/lib/qa/authorities/tgnlang.rb +7 -12
- data/lib/qa/version.rb +1 -1
- data/spec/controllers/terms_controller_spec.rb +44 -54
- data/spec/lib/authorities/assign_fast_spec.rb +25 -27
- data/spec/lib/authorities/file_based_authority_spec.rb +25 -26
- data/spec/lib/authorities/geonames_spec.rb +5 -6
- data/spec/lib/authorities/getty/aat_spec.rb +6 -10
- data/spec/lib/authorities/getty/tgn_spec.rb +6 -10
- data/spec/lib/authorities/getty/ulan_spec.rb +6 -10
- data/spec/lib/authorities/getty_spec.rb +4 -5
- data/spec/lib/authorities/loc_spec.rb +30 -36
- data/spec/lib/authorities/local_spec.rb +5 -7
- data/spec/lib/authorities/mesh_spec.rb +9 -9
- data/spec/lib/authorities/mysql_table_based_authority_spec.rb +13 -5
- data/spec/lib/authorities/oclcts_spec.rb +17 -21
- data/spec/lib/authorities/table_based_authority_spec.rb +21 -12
- data/spec/lib/authorities/tgnlang_spec.rb +4 -6
- data/spec/lib/authorities_loc_subauthorities.rb +50 -54
- data/spec/lib/mesh_data_parser_spec.rb +73 -79
- data/spec/lib/services/rdf_authority_parser_spec.rb +2 -7
- data/spec/lib/tasks/mesh.rake_spec.rb +16 -12
- data/spec/models/subject_mesh_term_spec.rb +4 -4
- data/spec/routing/route_spec.rb +13 -15
- data/spec/spec_helper.rb +3 -4
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +0 -1
- metadata +45 -17
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::AAT < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,8 +11,7 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
def build_query_url
|
15
|
-
query = URI.escape(sparql(untaint(q)))
|
14
|
+
def build_query_url(q)
|
16
15
|
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q))}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
17
16
|
end
|
18
17
|
|
@@ -25,32 +24,32 @@ module Qa::Authorities
|
|
25
24
|
gvp:prefLabelGVP [skosxl:literalForm ?name].
|
26
25
|
FILTER regex(?name, \"#{search}\", \"i\") .
|
27
26
|
} ORDER BY ?name"
|
27
|
+
sparql
|
28
28
|
end
|
29
29
|
|
30
30
|
def untaint(q)
|
31
31
|
q.gsub(/[^\w\s-]/, '')
|
32
32
|
end
|
33
33
|
|
34
|
-
def find
|
34
|
+
def find(id)
|
35
35
|
json(find_url(id))
|
36
36
|
end
|
37
37
|
|
38
|
-
def find_url
|
38
|
+
def find_url(id)
|
39
39
|
"http://vocab.getty.edu/aat/#{id}.json"
|
40
40
|
end
|
41
41
|
|
42
42
|
def request_options
|
43
|
-
{ accept: 'application/sparql-results+json'}
|
43
|
+
{ accept: 'application/sparql-results+json' }
|
44
44
|
end
|
45
45
|
|
46
46
|
private
|
47
47
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
# Reformats the data received from the Getty service
|
49
|
+
def parse_authority_response(response)
|
50
|
+
response['results']['bindings'].map do |result|
|
51
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] }
|
52
|
+
end
|
52
53
|
end
|
53
|
-
end
|
54
|
-
|
55
54
|
end
|
56
55
|
end
|
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::TGN < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,11 +11,11 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
def build_query_url
|
14
|
+
def build_query_url(q)
|
15
15
|
query = URI.escape(sparql(untaint(q)))
|
16
16
|
# Replace ampersands, otherwise the query will fail
|
17
17
|
# Gsub hack to convert the encoded regex in the REPLACE into a form Getty understands
|
18
|
-
"http://vocab.getty.edu/sparql.json?query=#{query.gsub('&','%26').gsub(',[%5E,]+,[%5E,]+$','%2C[^%2C]%2B%2C[^%2C]%2B%24')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
18
|
+
"http://vocab.getty.edu/sparql.json?query=#{query.gsub('&', '%26').gsub(',[%5E,]+,[%5E,]+$', '%2C[^%2C]%2B%2C[^%2C]%2B%24')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
19
19
|
end
|
20
20
|
|
21
21
|
# Use a regex to exclude the continent and 'world' from the query
|
@@ -25,12 +25,12 @@ module Qa::Authorities
|
|
25
25
|
search = untaint(q)
|
26
26
|
if search.include?(' ')
|
27
27
|
ex = "(("
|
28
|
-
search.split(' ').each do |
|
28
|
+
search.split(' ').each do |i|
|
29
29
|
ex += "regex(CONCAT(?name, ', ', REPLACE(str(?par), \",[^,]+,[^,]+$\", \"\")), \"#{i}\",\"i\" ) && "
|
30
30
|
end
|
31
31
|
ex = ex[0..ex.length - 4]
|
32
32
|
ex += ') && ('
|
33
|
-
search.split(' ').each do |
|
33
|
+
search.split(' ').each do |i|
|
34
34
|
ex += "regex(?name, \"#{i}\",\"i\" ) || "
|
35
35
|
end
|
36
36
|
ex = ex[0..ex.length - 4]
|
@@ -48,33 +48,33 @@ module Qa::Authorities
|
|
48
48
|
gvp:parentString ?par .
|
49
49
|
FILTER #{ex} .
|
50
50
|
} ORDER BY ?name ASC(?par)"
|
51
|
+
sparql
|
51
52
|
end
|
52
53
|
|
53
54
|
def untaint(q)
|
54
55
|
q.gsub(/[^\w\s-]/, '')
|
55
56
|
end
|
56
57
|
|
57
|
-
def find
|
58
|
+
def find(id)
|
58
59
|
json(find_url(id))
|
59
60
|
end
|
60
61
|
|
61
|
-
def find_url
|
62
|
+
def find_url(id)
|
62
63
|
"http://vocab.getty.edu/tgn/#{id}.json"
|
63
64
|
end
|
64
65
|
|
65
66
|
def request_options
|
66
|
-
{ accept: 'application/sparql-results+json'}
|
67
|
+
{ accept: 'application/sparql-results+json' }
|
67
68
|
end
|
68
69
|
|
69
70
|
private
|
70
71
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
72
|
+
# Reformats the data received from the service
|
73
|
+
# Adds the parentString, minus the contintent and 'World' for disambiguation
|
74
|
+
def parse_authority_response(response)
|
75
|
+
response['results']['bindings'].map do |result|
|
76
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] + ' (' + result['par']['value'].gsub(/\,[^\,]+\,[^\,]+$/, '') + ')' }
|
77
|
+
end
|
76
78
|
end
|
77
|
-
end
|
78
|
-
|
79
79
|
end
|
80
80
|
end
|
@@ -2,7 +2,7 @@ module Qa::Authorities
|
|
2
2
|
class Getty::Ulan < Base
|
3
3
|
include WebServiceBase
|
4
4
|
|
5
|
-
def search
|
5
|
+
def search(q)
|
6
6
|
parse_authority_response(json(build_query_url(q)))
|
7
7
|
end
|
8
8
|
|
@@ -11,10 +11,9 @@ module Qa::Authorities
|
|
11
11
|
get_json(*args)
|
12
12
|
end
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q)).gsub('&','%26')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
14
|
+
# Replace ampersands, otherwise the query will fail
|
15
|
+
def build_query_url(q)
|
16
|
+
"http://vocab.getty.edu/sparql.json?query=#{URI.escape(sparql(q)).gsub('&', '%26')}&_implicit=false&implicit=true&_equivalent=false&_form=%2Fsparql"
|
18
17
|
end
|
19
18
|
|
20
19
|
def sparql(q)
|
@@ -22,7 +21,7 @@ module Qa::Authorities
|
|
22
21
|
# if more than one term is supplied, check both preferred and alt labels
|
23
22
|
if search.include?(' ')
|
24
23
|
ex = "("
|
25
|
-
search.split(' ').each do |
|
24
|
+
search.split(' ').each do |i|
|
26
25
|
ex += "regex(CONCAT(?name, ' ', ?alt), \"#{i}\",\"i\" ) && "
|
27
26
|
end
|
28
27
|
ex = ex[0..ex.length - 4]
|
@@ -39,33 +38,33 @@ module Qa::Authorities
|
|
39
38
|
skos:altLabel ?alt .
|
40
39
|
FILTER #{ex} .
|
41
40
|
} ORDER BY ?name"
|
41
|
+
sparql
|
42
42
|
end
|
43
43
|
|
44
44
|
def untaint(q)
|
45
45
|
q.gsub(/[^\w\s-]/, '')
|
46
46
|
end
|
47
47
|
|
48
|
-
def find
|
48
|
+
def find(id)
|
49
49
|
json(find_url(id))
|
50
50
|
end
|
51
51
|
|
52
|
-
def find_url
|
52
|
+
def find_url(id)
|
53
53
|
"http://vocab.getty.edu/ulan/#{id}.json"
|
54
54
|
end
|
55
55
|
|
56
56
|
def request_options
|
57
|
-
{ accept: 'application/sparql-results+json'}
|
57
|
+
{ accept: 'application/sparql-results+json' }
|
58
58
|
end
|
59
59
|
|
60
60
|
private
|
61
61
|
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
62
|
+
# Reformats the data received from the Getty service
|
63
|
+
# Add the bio for disambiguation
|
64
|
+
def parse_authority_response(response)
|
65
|
+
response['results']['bindings'].map do |result|
|
66
|
+
{ 'id' => result['s']['value'], 'label' => result['name']['value'] + ' (' + result['bio']['value'] + ')' }
|
67
|
+
end
|
67
68
|
end
|
68
|
-
end
|
69
|
-
|
70
69
|
end
|
71
70
|
end
|
@@ -7,65 +7,64 @@ module Qa::Authorities
|
|
7
7
|
|
8
8
|
include WebServiceBase
|
9
9
|
|
10
|
-
def search
|
10
|
+
def search(q)
|
11
11
|
@raw_response = get_json(build_query_url(q))
|
12
12
|
parse_authority_response
|
13
13
|
end
|
14
14
|
|
15
|
-
def build_query_url
|
15
|
+
def build_query_url(q)
|
16
16
|
escaped_query = URI.escape(q)
|
17
17
|
authority_fragment = Loc.get_url_for_authority(subauthority) + URI.escape(subauthority)
|
18
|
-
|
18
|
+
"http://id.loc.gov/search/?q=#{escaped_query}&q=#{authority_fragment}&format=json"
|
19
19
|
end
|
20
20
|
|
21
|
-
def find
|
21
|
+
def find(id)
|
22
22
|
get_json(find_url(id))
|
23
23
|
end
|
24
24
|
|
25
|
-
def find_url
|
25
|
+
def find_url(id)
|
26
26
|
"http://id.loc.gov/authorities/#{@subauthority}/#{id}.json"
|
27
27
|
end
|
28
28
|
|
29
29
|
private
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
31
|
+
# Reformats the data received from the LOC service
|
32
|
+
def parse_authority_response
|
33
|
+
@raw_response.select { |response| response[0] == "atom:entry" }.map do |response|
|
34
|
+
loc_response_to_qa(response_to_struct(response))
|
35
|
+
end
|
35
36
|
end
|
36
|
-
end
|
37
37
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
38
|
+
# Converts most of the atom data into an OpenStruct object.
|
39
|
+
#
|
40
|
+
# Note that this is a pretty naive conversion. There should probably just
|
41
|
+
# be a class that properly translates and stores the various pieces of
|
42
|
+
# data, especially if this logic could be useful in other auth lookups.
|
43
|
+
def response_to_struct(response)
|
44
|
+
contents = response.each_with_object({}) do |result_parts, result|
|
45
|
+
next unless result_parts[0]
|
46
|
+
key = result_parts[0].sub('atom:', '').sub('dcterms:', '')
|
47
|
+
info = result_parts[1]
|
48
|
+
val = result_parts[2]
|
49
49
|
|
50
|
-
|
50
|
+
case key
|
51
51
|
when 'title', 'id', 'name', 'updated', 'created'
|
52
52
|
result[key] = val
|
53
53
|
when 'link'
|
54
54
|
result["links"] ||= []
|
55
55
|
result["links"] << [info["type"], info["href"]]
|
56
|
+
end
|
56
57
|
end
|
57
|
-
end
|
58
|
-
|
59
|
-
OpenStruct.new(result)
|
60
|
-
end
|
61
58
|
|
62
|
-
|
63
|
-
|
64
|
-
{
|
65
|
-
"id" => data.id || data.title,
|
66
|
-
"label" => data.title
|
67
|
-
}
|
68
|
-
end
|
59
|
+
OpenStruct.new(contents)
|
60
|
+
end
|
69
61
|
|
62
|
+
# Simple conversion from LoC-based struct to QA hash
|
63
|
+
def loc_response_to_qa(data)
|
64
|
+
{
|
65
|
+
"id" => data.id || data.title,
|
66
|
+
"label" => data.title
|
67
|
+
}
|
68
|
+
end
|
70
69
|
end
|
71
70
|
end
|
@@ -1,11 +1,9 @@
|
|
1
1
|
module Qa::Authorities::LocSubauthority
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
when datatypes.include?(authority) then datatype_base_url
|
8
|
-
when preservation.include?(authority) then vocab_preservation_base_url
|
2
|
+
def get_url_for_authority(authority)
|
3
|
+
if authorities.include?(authority) then authority_base_url
|
4
|
+
elsif vocabularies.include?(authority) then vocab_base_url
|
5
|
+
elsif datatypes.include?(authority) then datatype_base_url
|
6
|
+
elsif preservation.include?(authority) then vocab_preservation_base_url
|
9
7
|
end
|
10
8
|
end
|
11
9
|
|
@@ -71,20 +69,19 @@ module Qa::Authorities::LocSubauthority
|
|
71
69
|
|
72
70
|
private
|
73
71
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
def authority_base_url
|
79
|
-
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F"
|
80
|
-
end
|
72
|
+
def vocab_base_url
|
73
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2F"
|
74
|
+
end
|
81
75
|
|
82
|
-
|
83
|
-
|
84
|
-
|
76
|
+
def authority_base_url
|
77
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fauthorities%2F"
|
78
|
+
end
|
85
79
|
|
86
|
-
|
87
|
-
|
88
|
-
|
80
|
+
def datatype_base_url
|
81
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fdatatypes%2F"
|
82
|
+
end
|
89
83
|
|
84
|
+
def vocab_preservation_base_url
|
85
|
+
"cs%3Ahttp%3A%2F%2Fid.loc.gov%2Fvocabulary%2Fpreservation%2F"
|
86
|
+
end
|
90
87
|
end
|
data/lib/qa/authorities/local.rb
CHANGED
@@ -7,58 +7,59 @@ module Qa::Authorities
|
|
7
7
|
autoload :TableBasedAuthority
|
8
8
|
autoload :MysqlTableBasedAuthority
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
end
|
10
|
+
class << self
|
11
|
+
attr_reader :config
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
def load_config(file)
|
14
|
+
@config = YAML.load_file(file)
|
15
|
+
end
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
17
|
+
# Path to sub-authority files is either the full path to a directory or
|
18
|
+
# the path to a directory relative to the Rails application
|
19
|
+
def subauthorities_path
|
20
|
+
if config[:local_path].starts_with?(File::Separator)
|
21
|
+
config[:local_path]
|
22
|
+
else
|
23
|
+
File.join(Rails.root, config[:local_path])
|
24
|
+
end
|
25
25
|
end
|
26
|
-
end
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
# Local sub-authorities are any YAML files in the subauthorities_path
|
28
|
+
def names
|
29
|
+
unless Dir.exist? subauthorities_path
|
30
|
+
raise Qa::ConfigDirectoryNotFound, "There's no directory at #{subauthorities_path}. You must create it in order to use local authorities"
|
31
|
+
end
|
32
|
+
Dir.entries(subauthorities_path).map { |f| File.basename(f, ".yml") if f =~ /yml$/ }.compact
|
32
33
|
end
|
33
|
-
Dir.entries(subauthorities_path).map { |f| File.basename(f, ".yml") if f.match(/yml$/) }.compact
|
34
|
-
end
|
35
34
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
35
|
+
def subauthority_for(subauthority)
|
36
|
+
validate_subauthority!(subauthority)
|
37
|
+
registry.instance_for(subauthority)
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
def registry
|
41
|
+
@registry ||= begin
|
42
|
+
Registry.new do |reg|
|
43
|
+
register_defaults(reg)
|
44
|
+
end
|
45
45
|
end
|
46
46
|
end
|
47
|
-
end
|
48
47
|
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
def register_subauthority(subauthority, class_name)
|
49
|
+
registry.add(subauthority, class_name)
|
50
|
+
end
|
52
51
|
|
53
|
-
|
54
|
-
|
55
|
-
|
52
|
+
def subauthorities
|
53
|
+
registry.keys
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
56
57
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
58
|
+
def register_defaults(reg)
|
59
|
+
names.each do |name|
|
60
|
+
reg.add(name, 'Qa::Authorities::Local::FileBasedAuthority')
|
61
|
+
end
|
61
62
|
end
|
62
|
-
|
63
|
+
end
|
63
64
|
end
|
64
65
|
end
|