qa 0.9.0 → 0.10.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: debb1457464aa8c952a8bc130618d1edcf86e7df
4
- data.tar.gz: 93d2a32f54288ce095f2efd3ea322fe190c9f453
3
+ metadata.gz: e5ce81b7b841779705a850c38259249eb3d87911
4
+ data.tar.gz: 57de65366200cca987411cf3a0ee8fe9c65320b1
5
5
  SHA512:
6
- metadata.gz: 3f0664726f8224d72fa6b75f53d410c3f760dda40983fb7743c8dcd2e750fba8791042adbd883978b9b90ce12a9b8fafd4e2138f64efafc4632b672b02c93dfa
7
- data.tar.gz: f00430c377ba7ad6e886ae5a6a4960d6f306ebb6c6222427350f7899ef3ad702bb1b2a9dcdd88022e1dc857930946afe502eb8b342d3411f692b1b09b8edc783
6
+ metadata.gz: 4cba3e3e087eeaa9a5b754d53bfa55f6b89033b1b108ea750e999aec6b3c68fb7c4c1ac1a82024c6a77d1e1c5046ea57fc2f4bc818e7d386071d7480c5ba2e4a
7
+ data.tar.gz: b5d1f9021e03bf74c8f7a9a576f95483fa212a51f70a13ac7666de3fb59f4ece0a5f9da57c93f533bbcff9321577c504ebd83e01b17c539372e769b63019e4ce
data/README.md CHANGED
@@ -278,7 +278,12 @@ Results are in JSON.
278
278
  The entire list (up to the first 1000 terms) can also be returned using:
279
279
 
280
280
  /qa/terms/local/languages/
281
-
281
+
282
+ #### Loading RDF data into database tables
283
+
284
+ You can use the Qa::Services::RDFAuthorityParser to import rdf files into yopur database tables. See the class file, lib/qa/services/rdf_authority_parser.rb, for examples and more information.
285
+ To run the class in your local project you must include `gem 'linkeddata'` into your Gemfile and `require 'linkeddata'` into an initializer or your application.rb
286
+
282
287
  ### Medical Subject Headings (MeSH)
283
288
 
284
289
  Provides autocompletion of [MeSH terms](http://www.nlm.nih.gov/mesh/introduction.html). This
data/lib/qa.rb CHANGED
@@ -6,6 +6,7 @@ module Qa
6
6
  extend ActiveSupport::Autoload
7
7
 
8
8
  autoload :Authorities
9
+ autoload :Services
9
10
 
10
11
  # Raised when the configuration directory for local authorities doesn't exist
11
12
  class ConfigDirectoryNotFound < StandardError; end
@@ -5,7 +5,7 @@ module Qa
5
5
  class MysqlTableBasedAuthority < Local::TableBasedAuthority
6
6
  def self.check_for_index
7
7
  conn = ActiveRecord::Base.connection
8
- if conn.table_exists?('qa_local_authority_entries') && conn.index_name_exists?(:qa_local_authority_entries, 'index_qa_local_authority_entries_on_lower_label_and_authority', :default).blank?
8
+ if table_or_view_exists? && conn.index_name_exists?(:qa_local_authority_entries, 'index_qa_local_authority_entries_on_lower_label_and_authority', :default).blank?
9
9
  Rails.logger.error "You've installed mysql local authority tables, but you haven't indexed the lower label. Rails doesn't support functional indexes in migrations, so we tried to execute it for you but something went wrong...\n" \
10
10
  'Make sure your table has a lower_label column which is virtuall created and that column is indexed' \
11
11
  ' ALTER TABLE qa_local_authority_entries ADD lower_label VARCHAR(256) GENERATED ALWAYS AS (lower(label)) VIRTUAL' \
@@ -2,7 +2,7 @@ module Qa::Authorities
2
2
  class Local::TableBasedAuthority < Base
3
3
  def self.check_for_index
4
4
  conn = ActiveRecord::Base.connection
5
- if conn.table_exists?('local_authority_entries') && !conn.indexes('local_authority_entries').find { |i| i.name == 'index_local_authority_entries_on_lower_label' }
5
+ if table_or_view_exists? && !conn.indexes('local_authority_entries').find { |i| i.name == 'index_local_authority_entries_on_lower_label' }
6
6
  Rails.logger.error "You've installed local authority tables, but you haven't indexed the label. Rails doesn't support functional indexes in migrations, so you'll have to add this manually:\n" \
7
7
  "CREATE INDEX \"index_qa_local_authority_entries_on_lower_label\" ON \"qa_local_authority_entries\" (local_authority_id, lower(label))\n" \
8
8
  " OR on Sqlite: \n" \
@@ -34,6 +34,15 @@ module Qa::Authorities
34
34
 
35
35
  private
36
36
 
37
+ def self.table_or_view_exists?
38
+ conn = ActiveRecord::Base.connection
39
+ if conn.respond_to?(:data_source_exists?)
40
+ conn.data_source_exists?('qa_local_authority_entries')
41
+ else
42
+ conn.table_exists?('qa_local_authority_entries')
43
+ end
44
+ end
45
+
37
46
  def base_relation
38
47
  Qa::LocalAuthorityEntry.where(local_authority: local_authority)
39
48
  end
@@ -0,0 +1,5 @@
1
+ module Qa::Services
2
+ extend ActiveSupport::Autoload
3
+
4
+ autoload :RDFAuthorityParser
5
+ end
@@ -0,0 +1,59 @@
1
+ # frozen_string_literal: true
2
+ module Qa
3
+ module Services
4
+ # Class to parse downloaded content from rdf sources such as
5
+ # Library of Congress and Lexvo to create LocalAuthorityEntries
6
+ # for a named LocalAuthority.
7
+ #
8
+ # Example calls:
9
+ # RDFAuthorityParser.import_rdf('subjects',['/tmp/authoritiessubjects.nt'])
10
+ # RDFAuthorityParser.import_rdf('languages',['/tmp/lexvo_2013-02-09.rdf'],
11
+ # format: 'rdfxml',
12
+ # predicate: RDF::URI("http://www.w3.org/2008/05/skos#prefLabel"))
13
+ #
14
+ class RDFAuthorityParser
15
+ class << self
16
+ # import the rdf from the sources and store them as entries for the
17
+ # named LocalAuthority
18
+ #
19
+ # @param [String] name The name of the authority to update
20
+ # @param [Array] sources An array of file names to process
21
+ # @param [Hash] opts options for processing
22
+ # @option [opts] :format format of the rdf to parse
23
+ # see RDF::Reader for valid options
24
+ # defaults to :ntripples
25
+ # @option [opts] :predicate label predicate to use for
26
+ # LocalAuthorityEntry.label, which
27
+ # must match a field in the source file(s)
28
+ # defaults to http://www.w3.org/2004/02/skos/core#prefLabel
29
+ def import_rdf(name, sources, opts = {})
30
+ authority = Qa::LocalAuthority.find_or_create_by(name: name)
31
+ format = opts.fetch(:format, :ntriples)
32
+ predicate = opts.fetch(:predicate, ::RDF::Vocab::SKOS.prefLabel)
33
+ sources.each do |uri|
34
+ import_source(authority, format, predicate, uri)
35
+ end
36
+ end
37
+
38
+ private
39
+
40
+ def import_source(authority, format, predicate, uri)
41
+ ::RDF::Reader.open(uri, format: format) do |reader|
42
+ reader.each_statement do |statement|
43
+ parse_statement(statement, predicate, authority)
44
+ end
45
+ end
46
+ end
47
+
48
+ def parse_statement(statement, predicate, authority)
49
+ return unless statement.predicate == predicate
50
+ Qa::LocalAuthorityEntry.create(local_authority: authority,
51
+ label: statement.object.to_s,
52
+ uri: statement.subject.to_s)
53
+ rescue ActiveRecord::RecordNotUnique
54
+ Rails.logger.warn("Duplicate record: #{statement.subject}")
55
+ end
56
+ end
57
+ end
58
+ end
59
+ end
@@ -1,3 +1,3 @@
1
1
  module Qa
2
- VERSION = "0.9.0"
2
+ VERSION = "0.10.0"
3
3
  end
@@ -0,0 +1,91 @@
1
+ <?xml version="1.0" encoding="UTF-8" ?>
2
+ <rdf:RDF
3
+ xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
4
+ xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
5
+ xmlns:lvont="http://lexvo.org/ontology#"
6
+ xmlns:lexvo="http://lexvo.org/id/"
7
+ xmlns:owl="http://www.w3.org/2002/07/owl#"
8
+ xmlns:skos="http://www.w3.org/2008/05/skos#"
9
+ xmlns:skosxl="http://www.w3.org/2008/05/skos-xl#"
10
+ xmlns:foaf="http://xmlns.com/foaf/0.1/"
11
+ xmlns:xsd="http://www.w3.org/2001/XMLSchema#" >
12
+
13
+ <!--
14
+ This data file is a part of
15
+
16
+ Lexvo
17
+ http://www.lexvo.org/
18
+ Gerard de Melo, 2008-2013
19
+
20
+ License: Creative Commons Attribution-Share Alike 3.0 Unported
21
+ http://creativecommons.org/licenses/by-sa/3.0/
22
+
23
+ Portions copyright various contributors (e.g. article authors on Wikipedia).
24
+ For information about the data sources and the respective copyrights,
25
+ please see:
26
+ http://www.lexvo.org/linkeddata/sources.html
27
+ -->
28
+ <rdf:Description rdf:about="http://lexvo.org/id/iso639-3/aab">
29
+ <rdf:type rdf:resource="lvont:Language" />
30
+ <rdfs:comment xml:lang="en">Alumu (Alumu–Akpondu, Alumu–Tesu) is a Niger–Congo language spoken by approxi
31
+ mately 7000 people in Nassarawa State, Nigeria. It has lost the nominal affix system characteristic of the
32
+ Niger–Congo family.</rdfs:comment>
33
+ <rdfs:label xml:lang="de">Alumu-Tesu</rdfs:label>
34
+ <rdfs:label xml:lang="en">Alumu language</rdfs:label>
35
+ <rdfs:label xml:lang="en">Alumu-Tesu</rdfs:label>
36
+ <rdfs:label xml:lang="hr">Alumu-Tesu jezik</rdfs:label>
37
+ <rdfs:label xml:lang="nl">Alumu-Tesu</rdfs:label>
38
+ <rdfs:label xml:lang="pms">Lenga Alumu-Tesu</rdfs:label>
39
+ <owl:sameAs rdf:resource="http://dbpedia.org/resource/Alumu_language" />
40
+ <owl:sameAs rdf:resource="http://www.mpii.de/yago/resource/Alumu_language" />
41
+ <lvont:iso639P3PCode rdf:datatype="http://www.w3.org/2001/XMLSchema#string">aab</lvont:iso639P3PCode>
42
+ <lvont:label rdf:resource="http://lexvo.org/id/term/deu/Alumu-Tesu" /> <!-- Alumu-Tesu -->
43
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Alumu%20language" /> <!-- Alumu language -->
44
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Alumu-Tesu" /> <!-- Alumu-Tesu -->
45
+ <lvont:label rdf:resource="http://lexvo.org/id/term/hrv/Alumu-Tesu%20jezik" /> <!-- Alumu-Tesu jezik -->
46
+ <lvont:label rdf:resource="http://lexvo.org/id/term/nld/Alumu-Tesu" /> <!-- Alumu-Tesu -->
47
+ <lvont:label rdf:resource="http://lexvo.org/id/term/pms/Lenga%20Alumu-Tesu" /> <!-- Lenga Alumu-Tesu -->
48
+ <skos:prefLabel xml:lang="en">Alumu-Tesu</skos:prefLabel>
49
+ </rdf:Description>
50
+ <rdf:Description rdf:about="http://lexvo.org/id/iso639-3/aac">
51
+ <rdf:type rdf:resource="lvont:Language" />
52
+ <rdfs:comment xml:lang="en">The Ari language is a Papuan language of the Trans–New Guinea family. As of the 2000 census there were only 50 Ari speakers, living in two villages.</rdfs:comment>
53
+ <rdfs:label xml:lang="en">Ari language</rdfs:label>
54
+ <rdfs:label xml:lang="en">Ari</rdfs:label>
55
+ <rdfs:label xml:lang="fr">Ari</rdfs:label>
56
+ <rdfs:label xml:lang="hr">Ari jezik</rdfs:label>
57
+ <rdfs:label xml:lang="nl">Ari</rdfs:label>
58
+ <rdfs:label xml:lang="pms">Lenga Ari</rdfs:label>
59
+ <owl:sameAs rdf:resource="http://dbpedia.org/resource/Ari_language" />
60
+ <owl:sameAs rdf:resource="http://www.lingvoj.org/lang/aac" />
61
+ <owl:sameAs rdf:resource="http://www.mpii.de/yago/resource/Ari_language" />
62
+ <lvont:iso639P3PCode rdf:datatype="http://www.w3.org/2001/XMLSchema#string">aac</lvont:iso639P3PCode>
63
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Ari%20language" /> <!-- Ari language -->
64
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Ari" /> <!-- Ari -->
65
+ <lvont:label rdf:resource="http://lexvo.org/id/term/fra/Ari" /> <!-- Ari -->
66
+ <lvont:label rdf:resource="http://lexvo.org/id/term/hrv/Ari%20jezik" /> <!-- Ari jezik -->
67
+ <lvont:label rdf:resource="http://lexvo.org/id/term/nld/Ari" /> <!-- Ari -->
68
+ <lvont:label rdf:resource="http://lexvo.org/id/term/pms/Lenga%20Ari" /> <!-- Lenga Ari -->
69
+ <skos:prefLabel xml:lang="en">Ari</skos:prefLabel>
70
+ </rdf:Description>
71
+ <rdf:Description rdf:about="http://lexvo.org/id/iso639-3/aac">
72
+ <rdf:type rdf:resource="lvont:Language" />
73
+ <rdfs:comment xml:lang="en">The Ari language is a Papuan language of the Trans–New Guinea family. As of the 2000 census there were only 50 Ari speakers, living in two villages.</rdfs:comment>
74
+ <rdfs:label xml:lang="en">Ari language</rdfs:label>
75
+ <rdfs:label xml:lang="en">Ari</rdfs:label>
76
+ <rdfs:label xml:lang="fr">Ari</rdfs:label>
77
+ <rdfs:label xml:lang="hr">Ari jezik</rdfs:label>
78
+ <rdfs:label xml:lang="nl">Ari</rdfs:label>
79
+ <rdfs:label xml:lang="pms">Lenga Ari</rdfs:label>
80
+ <owl:sameAs rdf:resource="http://dbpedia.org/resource/Ari_language" />
81
+ <owl:sameAs rdf:resource="http://www.lingvoj.org/lang/aac" />
82
+ <owl:sameAs rdf:resource="http://www.mpii.de/yago/resource/Ari_language" />
83
+ <lvont:iso639P3PCode rdf:datatype="http://www.w3.org/2001/XMLSchema#string">aac</lvont:iso639P3PCode>
84
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Ari%20language" /> <!-- Ari language -->
85
+ <lvont:label rdf:resource="http://lexvo.org/id/term/eng/Ari" /> <!-- Ari -->
86
+ <lvont:label rdf:resource="http://lexvo.org/id/term/fra/Ari" /> <!-- Ari -->
87
+ <lvont:label rdf:resource="http://lexvo.org/id/term/hrv/Ari%20jezik" /> <!-- Ari jezik -->
88
+ <lvont:label rdf:resource="http://lexvo.org/id/term/nld/Ari" /> <!-- Ari -->
89
+ <lvont:label rdf:resource="http://lexvo.org/id/term/pms/Lenga%20Ari" /> <!-- Lenga Ari -->
90
+ <skos:prefLabel xml:lang="en">Ari</skos:prefLabel>
91
+ </rdf:Description>
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+ require 'spec_helper'
3
+
4
+ describe Qa::Services::RDFAuthorityParser do
5
+ let(:source) {[File.join(fixture_path,'lexvo_snippet.rdf')] }
6
+ let(:format) { 'rdfxml' }
7
+ let(:predicate) { RDF::URI("http://www.w3.org/2008/05/skos#prefLabel") }
8
+ let(:name) { 'language' }
9
+ let(:alum_entry) { Qa::LocalAuthorityEntry.find_by(label: 'Alumu-Tesu') }
10
+ let(:ari_entry) { Qa::LocalAuthorityEntry.find_by(label: 'Ari') }
11
+
12
+ describe "#import_rdf" do
13
+ before do
14
+ blah = Qa::LocalAuthority.find_by_name(name)
15
+ described_class.import_rdf(name, source, format: format, predicate: predicate)
16
+ end
17
+ it "creates the authority and authority entries" do
18
+ expect(Qa::LocalAuthority.count).to eq(1)
19
+ expect(Qa::LocalAuthority.find_by(name: name)).not_to be_nil
20
+ expect(Qa::LocalAuthorityEntry.count).to eq(2)
21
+ expect(alum_entry).not_to be_nil
22
+ expect(alum_entry.uri).to eq('http://lexvo.org/id/iso639-3/aab')
23
+ expect(ari_entry).not_to be_nil
24
+ expect(ari_entry.uri).to eq('http://lexvo.org/id/iso639-3/aac')
25
+ end
26
+ end
27
+ end
@@ -1,4 +1,6 @@
1
1
 
2
+ require 'linkeddata'
3
+
2
4
  require 'engine_cart'
3
5
  require 'simplecov'
4
6
  require 'byebug' unless ENV['TRAVIS']
@@ -18,7 +20,7 @@ Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
18
20
 
19
21
  RSpec.configure do |config|
20
22
 
21
- config.fixture_path = "../spec/fixtures"
23
+ config.fixture_path = File.expand_path("../fixtures", __FILE__)
22
24
 
23
25
  config.use_transactional_fixtures = true
24
26
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.0
4
+ version: 0.10.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stephen Anderson
@@ -15,7 +15,7 @@ authors:
15
15
  autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
- date: 2016-08-08 00:00:00.000000000 Z
18
+ date: 2016-08-16 00:00:00.000000000 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  name: rails
@@ -177,6 +177,20 @@ dependencies:
177
177
  - - ">="
178
178
  - !ruby/object:Gem::Version
179
179
  version: '0'
180
+ - !ruby/object:Gem::Dependency
181
+ name: linkeddata
182
+ requirement: !ruby/object:Gem::Requirement
183
+ requirements:
184
+ - - ">="
185
+ - !ruby/object:Gem::Version
186
+ version: '0'
187
+ type: :development
188
+ prerelease: false
189
+ version_requirements: !ruby/object:Gem::Requirement
190
+ requirements:
191
+ - - ">="
192
+ - !ruby/object:Gem::Version
193
+ version: '0'
180
194
  description: Provides a set of uniform RESTful routes to query any controlled vocabulary
181
195
  or set of authority terms.
182
196
  email:
@@ -244,6 +258,8 @@ files:
244
258
  - lib/qa/authorities/web_service_base.rb
245
259
  - lib/qa/data/TGN_LANGUAGES.xml
246
260
  - lib/qa/engine.rb
261
+ - lib/qa/services.rb
262
+ - lib/qa/services/rdf_authority_parser.rb
247
263
  - lib/qa/version.rb
248
264
  - lib/tasks/mesh.rake
249
265
  - lib/tasks/qa_tasks.rake
@@ -262,6 +278,7 @@ files:
262
278
  - spec/fixtures/getty-aat-find-response.json
263
279
  - spec/fixtures/getty-tgn-find-response.json
264
280
  - spec/fixtures/getty-ulan-find-response.json
281
+ - spec/fixtures/lexvo_snippet.rdf
265
282
  - spec/fixtures/loc-names-response.txt
266
283
  - spec/fixtures/loc-response.txt
267
284
  - spec/fixtures/loc-subject-find-response.txt
@@ -288,6 +305,7 @@ files:
288
305
  - spec/lib/authorities/tgnlang_spec.rb
289
306
  - spec/lib/authorities_loc_subauthorities.rb
290
307
  - spec/lib/mesh_data_parser_spec.rb
308
+ - spec/lib/services/rdf_authority_parser_spec.rb
291
309
  - spec/lib/tasks/mesh.rake_spec.rb
292
310
  - spec/models/subject_mesh_term_spec.rb
293
311
  - spec/routing/route_spec.rb
@@ -333,6 +351,7 @@ test_files:
333
351
  - spec/fixtures/getty-aat-find-response.json
334
352
  - spec/fixtures/getty-tgn-find-response.json
335
353
  - spec/fixtures/getty-ulan-find-response.json
354
+ - spec/fixtures/lexvo_snippet.rdf
336
355
  - spec/fixtures/loc-names-response.txt
337
356
  - spec/fixtures/loc-response.txt
338
357
  - spec/fixtures/loc-subject-find-response.txt
@@ -359,6 +378,7 @@ test_files:
359
378
  - spec/lib/authorities/tgnlang_spec.rb
360
379
  - spec/lib/authorities_loc_subauthorities.rb
361
380
  - spec/lib/mesh_data_parser_spec.rb
381
+ - spec/lib/services/rdf_authority_parser_spec.rb
362
382
  - spec/lib/tasks/mesh.rake_spec.rb
363
383
  - spec/models/subject_mesh_term_spec.rb
364
384
  - spec/routing/route_spec.rb