iqvoc 4.7.0 → 4.8.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -0
  3. data/Gemfile +11 -11
  4. data/Gemfile.lock +178 -122
  5. data/README.md +39 -24
  6. data/{test/performance/browsing_test.rb → app/aides/inline_data_helper.rb} +23 -6
  7. data/app/aides/maker.rb +139 -0
  8. data/{lib → app/aides}/multi_logger.rb +0 -0
  9. data/app/aides/origin.rb +47 -0
  10. data/app/aides/rdfapi.rb +59 -0
  11. data/app/aides/skos_exporter.rb +151 -0
  12. data/app/aides/skos_importer.rb +348 -0
  13. data/app/assets/javascripts/iqvoc/entityselect.js.erb +7 -9
  14. data/app/controllers/application_controller.rb +1 -3
  15. data/app/controllers/collections/versions_controller.rb +1 -3
  16. data/app/controllers/concepts/versions_controller.rb +9 -3
  17. data/app/controllers/concerns/controller_extensions.rb +109 -0
  18. data/app/{concerns → controllers/concerns}/reverse_match_errors.rb +0 -0
  19. data/app/controllers/hierarchy_controller.rb +7 -3
  20. data/app/controllers/instance_configuration_controller.rb +1 -1
  21. data/app/controllers/pages_controller.rb +10 -0
  22. data/app/controllers/search_results_controller.rb +2 -2
  23. data/app/controllers/triplestore_sync_controller.rb +2 -4
  24. data/app/helpers/application_helper.rb +1 -1
  25. data/app/helpers/widget_helper.rb +3 -3
  26. data/app/jobs/export_job.rb +1 -3
  27. data/app/jobs/import_job.rb +1 -3
  28. data/app/models/ability.rb +59 -0
  29. data/app/models/abstract_user.rb +1 -1
  30. data/app/models/collection/base.rb +12 -3
  31. data/app/models/collection/member/skos/base.rb +1 -1
  32. data/app/models/concept/base.rb +15 -8
  33. data/app/models/concept/relation/base.rb +1 -1
  34. data/app/models/concept/relation/skos/base.rb +1 -1
  35. data/app/models/concept/skos/scheme.rb +1 -1
  36. data/app/models/concept/validations.rb +1 -1
  37. data/app/models/concerns/deep_cloning.rb +92 -0
  38. data/app/models/concerns/first_level_object_scopes.rb +9 -0
  39. data/app/{concerns → models/concerns}/first_level_object_validations.rb +9 -2
  40. data/app/models/concerns/rankable.rb +31 -0
  41. data/app/models/{search_extension.rb → concerns/search_extension.rb} +0 -0
  42. data/app/{concerns → models/concerns}/versioning.rb +0 -6
  43. data/app/models/configuration_setting.rb +1 -1
  44. data/app/models/labeling/skos/base.rb +2 -2
  45. data/app/models/match/skos/base.rb +2 -2
  46. data/app/models/note/skos/base.rb +7 -6
  47. data/app/models/note/skos/change_note.rb +1 -1
  48. data/{lib/iqvoc/rdf_sync.rb → app/services/rdf_sync_service.rb} +3 -3
  49. data/app/view_models/concept_view.rb +1 -1
  50. data/app/views/collections/_form.html.erb +2 -2
  51. data/app/views/concepts/scheme/edit.html.erb +1 -1
  52. data/app/views/pages/components.html.erb +45 -0
  53. data/app/views/pages/version.html.erb +6 -0
  54. data/app/views/partials/concept/_reverse_match_notice.html.erb +0 -1
  55. data/app/views/search_results/_sidebar.html.erb +3 -3
  56. data/config/application.rb +4 -1
  57. data/config/boot.rb +1 -2
  58. data/config/database.yml.postgresql +23 -0
  59. data/config/engine.rb +0 -2
  60. data/config/environments/heroku.rb +1 -1
  61. data/config/initializers/inflections.rb +9 -3
  62. data/config/initializers/iqvoc.rb +1 -7
  63. data/config/initializers/mime_types.rb +0 -1
  64. data/config/locales/de.yml +2 -1
  65. data/config/locales/en.yml +11 -10
  66. data/config/routes.rb +2 -0
  67. data/config/travis/database.yml.mysql +9 -0
  68. data/config/travis/database.yml.postgresql +7 -0
  69. data/config/travis/database.yml.sqlite +5 -0
  70. data/db/migrate/20141204151558_add_foreign_key_constraints.rb +23 -0
  71. data/iqvoc.gemspec +2 -2
  72. data/lib/generators/app/template.rb +15 -7
  73. data/lib/iqvoc.rb +2 -1
  74. data/lib/iqvoc/configuration/core.rb +18 -4
  75. data/lib/iqvoc/configuration/instance_configuration.rb +125 -0
  76. data/lib/iqvoc/configuration/navigation.rb +63 -0
  77. data/lib/iqvoc/environments/development.rb +4 -0
  78. data/lib/iqvoc/environments/production.rb +11 -12
  79. data/lib/iqvoc/environments/test.rb +4 -1
  80. data/lib/iqvoc/version.rb +2 -2
  81. data/lib/tasks/exporter.rake +1 -4
  82. data/lib/tasks/importer.rake +1 -5
  83. data/lib/tasks/sync.rake +1 -2
  84. data/test/controllers/concept_movement_test.rb +11 -11
  85. data/test/controllers/hierarchy_test.rb +83 -79
  86. data/test/controllers/reverse_match_test.rb +2 -2
  87. data/test/integration/alphabetical_test.rb +2 -3
  88. data/test/integration/browse_concepts_and_labels_test.rb +2 -2
  89. data/test/integration/collection_circularity_test.rb +6 -6
  90. data/test/integration/concept_scheme_browsing_test.rb +2 -2
  91. data/test/integration/edit_concepts_test.rb +1 -1
  92. data/test/integration/export_test.rb +5 -3
  93. data/test/integration/import_test.rb +4 -1
  94. data/test/integration/instance_configuration_browsing_test.rb +2 -2
  95. data/test/integration/navigation_test.rb +2 -2
  96. data/test/integration/note_annotations_test.rb +12 -11
  97. data/test/integration/reverse_match_job_test.rb +19 -10
  98. data/test/integration/search_test.rb +6 -6
  99. data/test/integration/tree_test.rb +3 -3
  100. data/test/integration/untranslated_test.rb +1 -1
  101. data/test/models/concept_test.rb +13 -14
  102. data/test/models/inline_data_test.rb +9 -9
  103. data/test/models/instance_configuration_test.rb +7 -3
  104. data/test/models/origin_test.rb +9 -59
  105. data/test/models/rdf_sync_test.rb +2 -4
  106. data/test/models/rdfapi_test.rb +0 -2
  107. data/test/models/skos_collection_import_test.rb +3 -4
  108. data/test/models/skos_export_test.rb +3 -5
  109. data/test/models/skos_import_test.rb +12 -10
  110. data/test/test_helper.rb +0 -1
  111. data/vendor/assets/stylesheets/{jquery-ui.css.scss → jquery-ui.scss} +0 -0
  112. data/vendor/assets/stylesheets/{jquery-ui.structure.css.scss → jquery-ui.structure.scss} +0 -0
  113. data/vendor/assets/stylesheets/{jquery-ui.theme.css.scss → jquery-ui.theme.scss} +0 -0
  114. metadata +34 -28
  115. data/lib/iqvoc/ability.rb +0 -60
  116. data/lib/iqvoc/controller_extensions.rb +0 -111
  117. data/lib/iqvoc/deep_cloning.rb +0 -90
  118. data/lib/iqvoc/inline_data_helper.rb +0 -45
  119. data/lib/iqvoc/instance_configuration.rb +0 -123
  120. data/lib/iqvoc/maker.rb +0 -141
  121. data/lib/iqvoc/navigation.rb +0 -61
  122. data/lib/iqvoc/origin.rb +0 -111
  123. data/lib/iqvoc/rankable.rb +0 -33
  124. data/lib/iqvoc/rdfapi.rb +0 -60
  125. data/lib/iqvoc/skos_exporter.rb +0 -153
  126. data/lib/iqvoc/skos_importer.rb +0 -337
@@ -1,61 +0,0 @@
1
- # encoding: UTF-8
2
-
3
- # Copyright 2011-2013 innoQ Deutschland GmbH
4
- #
5
- # Licensed under the Apache License, Version 2.0 (the "License");
6
- # you may not use this file except in compliance with the License.
7
- # You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
-
17
- ##
18
- # Provides a basic API wrapping low-level iQvoc config operations
19
- # on Iqvoc.navigation_items
20
-
21
- module Iqvoc
22
- module Navigation
23
- def self.items
24
- Iqvoc.navigation_items
25
- end
26
-
27
- def self.add(item, position = nil)
28
- if position
29
- items.insert(position, item)
30
- else
31
- items << item
32
- end
33
- end
34
-
35
- def self.add_grouped(item, position = nil)
36
- index = setup_extension_group(position)
37
- items[index][:items] << item
38
- end
39
-
40
- private
41
- # Setup an empty navigation group for extensions
42
- # Returns index for the new (or existing) group, so add_grouped
43
- # can use the index to insert it's item under the group
44
- def self.setup_extension_group(position)
45
- group = {
46
- text: proc { t('txt.views.navigation.extensions') },
47
- items: []
48
- }
49
-
50
- if position && !items[position][:items]
51
- items.insert(position, group)
52
- elsif position && items[position][:items]
53
- return position
54
- else
55
- items << group
56
- end
57
-
58
- items.index(group)
59
- end
60
- end
61
- end
@@ -1,111 +0,0 @@
1
- # encoding: UTF-8
2
-
3
- # Copyright 2011-2013 innoQ Deutschland GmbH
4
- #
5
- # Licensed under the Apache License, Version 2.0 (the "License");
6
- # you may not use this file except in compliance with the License.
7
- # You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
-
17
- # Provides utilities to replace special chars etc in
18
- # texts to generate a valid turtle compatible id (an url slug):
19
- # Iqvoc::Origin.new("fübar").to_s # => "fuebar"
20
- #
21
- # Note that .to_s respects eventually previously executed method chains
22
- # Just calling "to_s" runs all registered filters.
23
- # Prepending "to_s" with a specific filter method only runs the given filter:
24
- # Iqvoc::Origin.new("fübar").replace_umlauts.to_s # => "fuebar"
25
- #
26
- # Adding your own filter classes is easy:
27
- # class FoobarStripper < Iqvoc::Origin::Filters::GenericFilter
28
- # def call(obj, str)
29
- # str = str.gsub("foobar", "")
30
- # run(obj, str)
31
- # end
32
- # end
33
- # Iqvoc::Origin::Filters.register(:strip_foobars, FoobarStripper)
34
- #
35
- module Iqvoc
36
- class Origin
37
- module Filters
38
- class GenericFilter
39
- def call(obj, str)
40
- # do what has to be done with str
41
- # afterwards: make sure to pass "obj" and your modified "str" to "run()"
42
- run(obj, str)
43
- end
44
-
45
- def run(obj, str)
46
- obj.tap do |obj|
47
- obj.value = str
48
- end
49
- end
50
- end
51
-
52
- class UriConformanceFilter < GenericFilter
53
- def call(obj, str)
54
- str = str.parameterize # basic url conformance
55
-
56
- # prefix with '_' if origin starts with digit
57
- str = str.gsub(/^[0-9].*$/) do |match|
58
- "_#{match}"
59
- end
60
- run(obj, str)
61
- end
62
- end
63
-
64
- @filters = {}
65
- @filters[:uri_conformance_filter] = UriConformanceFilter
66
-
67
- def self.register(name, klass)
68
- @filters[name.to_sym] = klass
69
- end
70
-
71
- def self.registered
72
- @filters
73
- end
74
- end
75
-
76
- attr_accessor :initial_value, :value, :filters
77
-
78
- def initialize(value)
79
- self.initial_value = value.to_s
80
- self.value = initial_value
81
- end
82
-
83
- def touched?
84
- value != initial_value
85
- end
86
-
87
- def run_filters!
88
- Filters.registered.each do |key, filter_class|
89
- filter_class.new.call(self, value)
90
- end
91
- end
92
-
93
- def method_missing(meth, *args)
94
- if Filters.registered.keys.include?(meth.to_sym)
95
- Filters.registered[meth.to_sym].new.call(self, value)
96
- else
97
- super
98
- end
99
- end
100
-
101
- def to_s
102
- return value if touched?
103
- run_filters!
104
- value
105
- end
106
-
107
- def inspect
108
- '#<Iqvoc::Origin:0x%08x>' % object_id
109
- end
110
- end
111
- end
@@ -1,33 +0,0 @@
1
- require 'active_support/concern'
2
-
3
- module Iqvoc
4
- module Rankable
5
- extend ActiveSupport::Concern
6
-
7
- def build_rdf(document, subject, suppress_extra_labels = false)
8
- super
9
- if self.class.rankable?
10
- predicate = "ranked#{rdf_predicate.titleize}"
11
-
12
- subject.Schema.build_predicate(predicate) do |blank_node|
13
- blank_node.Schema.relationWeight(rank)
14
- blank_node.Schema.relationTarget(IqRdf.build_uri(target.origin))
15
- end
16
- end
17
- end
18
-
19
- module ClassMethods
20
- def rankable?
21
- true
22
- end
23
-
24
- def partial_name(obj)
25
- 'partials/concept/relation/ranked'
26
- end
27
-
28
- def edit_partial_name(obj)
29
- 'partials/concept/relation/edit_ranked'
30
- end
31
- end
32
- end
33
- end
@@ -1,60 +0,0 @@
1
- # encoding: UTF-8
2
-
3
- # Copyright 2011-2013 innoQ Deutschland GmbH
4
- #
5
- # Licensed under the Apache License, Version 2.0 (the "License");
6
- # you may not use this file except in compliance with the License.
7
- # You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing, software
12
- # distributed under the License is distributed on an "AS IS" BASIS,
13
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
- # See the License for the specific language governing permissions and
15
- # limitations under the License.
16
-
17
- module Iqvoc
18
- module RDFAPI
19
- FIRST_LEVEL_OBJECT_CLASSES = [Iqvoc::Concept.base_class, Iqvoc::Collection.base_class]
20
- SECOND_LEVEL_OBJECT_CLASSES = Iqvoc::Concept.labeling_classes.keys +
21
- Iqvoc::Concept.note_classes +
22
- Iqvoc::Concept.relation_classes +
23
- Iqvoc::Concept.match_classes +
24
- Iqvoc::Concept.notation_classes +
25
- Iqvoc::Concept.additional_association_classes.keys +
26
- [Iqvoc::Collection.member_class]
27
-
28
- OBJECT_DICTIONARY = FIRST_LEVEL_OBJECT_CLASSES.inject({}) do |hash, klass|
29
- hash["#{klass.rdf_namespace}:#{klass.rdf_class}"] = klass
30
- hash
31
- end
32
-
33
- PREDICATE_DICTIONARY = SECOND_LEVEL_OBJECT_CLASSES.inject({}) do |hash, klass|
34
- hash["#{klass.rdf_namespace}:#{klass.rdf_predicate}"] = klass
35
- hash
36
- end
37
-
38
- URI_REGEXP = /^https?:\/\/[^\s]+$/
39
- LITERAL_REGEXP = /^"(.+)"(@(.+))?$/
40
-
41
- def self.devour(rdf_subject, rdf_predicate, rdf_object)
42
- case rdf_predicate
43
- when 'a', 'rdf:type'
44
- case rdf_object
45
- when String
46
- target = OBJECT_DICTIONARY[rdf_object] || rdf_object.constantize
47
- else
48
- target = rdf_object
49
- end
50
- target.find_or_initialize_by(origin: rdf_subject)
51
- when String
52
- # dictionary lookup
53
- target = PREDICATE_DICTIONARY[rdf_predicate] || rdf_predicate.constantize
54
- target.build_from_rdf(rdf_subject, target, rdf_object)
55
- else # is a class
56
- rdf_predicate.build_from_rdf(rdf_subject, rdf_predicate, rdf_object)
57
- end
58
- end
59
- end
60
- end
@@ -1,153 +0,0 @@
1
- require 'iq_rdf'
2
- require 'uri'
3
- require 'fileutils'
4
-
5
- module Iqvoc
6
- class SkosExporter
7
- include RdfHelper # necessary to use render_concept helper
8
- include RdfNamespacesHelper
9
- include Rails.application.routes.url_helpers
10
-
11
- def initialize(file_path, type, default_namespace_url, logger = Rails.logger)
12
- default_url_options[:port] = URI.parse(default_namespace_url).port
13
- default_url_options[:host] = URI.parse(default_namespace_url).to_s.gsub(/\/$/, '')
14
-
15
- @file_path = file_path
16
- @type = type
17
- @logger = logger
18
- @document = IqRdf::Document.new
19
-
20
- unless ['ttl', 'nt', 'xml'].include? @type
21
- raise "Iqvoc::SkosExporter: Unknown rdf serialization. Parameter 'type' should be 'ttl' (Turtle), 'nt' (N-Triples) or 'xml' (RDF-XML)."
22
- end
23
-
24
- unless @file_path.is_a?(String)
25
- raise "Iqvoc::SkosExporter#export: Parameter 'file' should be a String."
26
- end
27
- end
28
-
29
- def run
30
- export
31
- end
32
-
33
- private
34
-
35
- def export
36
- ActiveSupport.run_load_hooks(:rdf_export_before, self)
37
-
38
- start = Time.now
39
- @logger.info 'Starting export...'
40
- @logger.info "file_path = #{@file_path}"
41
- @logger.info "type = #{@type}"
42
-
43
- # add export data
44
- add_namespaces(@document)
45
- add_collections(@document)
46
- add_concepts(@document)
47
-
48
- ActiveSupport.run_load_hooks(:rdf_export_before_save, self)
49
-
50
- # saving export to disk
51
- save_file(@file_path, @type, @document)
52
-
53
- done = Time.now
54
- @logger.info "Export Job finished in #{(done - start).to_i} seconds."
55
-
56
- ActiveSupport.run_load_hooks(:rdf_export_after, self)
57
- end
58
-
59
- def add_namespaces(document)
60
- @logger.info 'Exporting namespaces...'
61
-
62
- RdfNamespacesHelper.instance_methods.each do |meth|
63
- namespaces = send(meth)
64
- document.namespaces(namespaces) if namespaces.is_a?(Hash)
65
- end
66
-
67
- @logger.info 'Finished exporting namespaces.'
68
- end
69
-
70
- def add_collections(document)
71
- @logger.info 'Exporting collections...'
72
-
73
- offset = 0
74
- while true
75
- collections = Iqvoc::Collection.base_class.published.order('id').limit(100).offset(offset)
76
- limit = collections.size < 100 ? collections.size : 100
77
- break if collections.size == 0
78
-
79
- # Todo: Preloading???
80
- collections.each do |collection|
81
- render_collection(document, collection)
82
- end
83
-
84
- @logger.info "Collections #{offset+1}-#{offset+limit} exported."
85
- offset += collections.size # Size is important!
86
- end
87
-
88
- @logger.info "Finished exporting collections (#{offset} collections exported)."
89
- end
90
-
91
- def add_concepts(document)
92
- @logger.info 'Exporting concepts...'
93
-
94
- offset = 0
95
- while true
96
- concepts = Iqvoc::Concept.base_class.published.order('id').limit(100).offset(offset)
97
- limit = concepts.size < 100 ? concepts.size : 100
98
- break if concepts.size == 0
99
-
100
- # When in single query mode, AR handles ALL includes to be loaded by that
101
- # one query. We don't want that! So let's do it manually :-)
102
- ActiveRecord::Associations::Preloader.new.preload(concepts,
103
- Iqvoc::Concept.base_class.default_includes + [
104
- :matches,
105
- :collection_members,
106
- :notations,
107
- { relations: :target, labelings: :target, notes: :annotations }
108
- ])
109
-
110
- concepts.each do |concept|
111
- render_concept(document, concept, true)
112
- end
113
-
114
- @logger.info "Concepts #{offset+1}-#{offset+limit} exported."
115
- offset += concepts.size # Size is important!
116
- end
117
-
118
- @logger.info "Finished exporting concepts (#{offset} concepts exported)."
119
- end
120
-
121
- def save_file(file_path, type, content)
122
- begin
123
- @logger.info "Saving export to '#{@file_path}'"
124
- create_directory(@file_path)
125
- file = File.open(@file_path, 'w')
126
- content = serialize_rdf(content, type)
127
- file.write(content)
128
- rescue IOError => e
129
- # some error occur
130
- # e.g not writable
131
- ensure
132
- file.close unless file == nil
133
- end
134
- end
135
-
136
- def create_directory(file_path)
137
- dirname = File.dirname(file_path)
138
- unless File.directory?(dirname)
139
- FileUtils.mkdir_p(dirname)
140
- end
141
- end
142
-
143
- def serialize_rdf(document, type)
144
- if type == 'xml'
145
- document.to_xml
146
- elsif type == 'ttl'
147
- document.to_turtle
148
- else
149
- document.to_ntriples
150
- end
151
- end
152
- end
153
- end
@@ -1,337 +0,0 @@
1
- require 'iqvoc/rdfapi'
2
-
3
- module Iqvoc
4
- class SkosImporter
5
- class_attribute :first_level_object_classes, :second_level_object_classes
6
- self.first_level_object_classes = [
7
- Iqvoc::Concept.base_class,
8
- Iqvoc::Collection.base_class
9
- ]
10
- self.second_level_object_classes = Iqvoc::Concept.labeling_classes.keys +
11
- Iqvoc::Concept.note_classes +
12
- Iqvoc::Concept.relation_classes +
13
- Iqvoc::Concept.match_classes +
14
- Iqvoc::Concept.notation_classes +
15
- Iqvoc::Concept.additional_association_classes.keys +
16
- [Iqvoc::Concept.root_class] +
17
- [Iqvoc::Collection.member_class]
18
-
19
- def self.prepend_first_level_object_classes(args)
20
- self.first_level_object_classes.unshift(*args)
21
- end
22
-
23
- def initialize(object, default_namespace_url, logger = Rails.logger, publish = true, verbose = false)
24
- @file = case object
25
- when File
26
- File.open(object)
27
- when Array
28
- object
29
- else
30
- open(object)
31
- end
32
-
33
- @default_namespace_url = default_namespace_url
34
- @publish = publish
35
- @verbose = verbose
36
- @logger = logger
37
-
38
- unless @file.is_a?(File) || @file.is_a?(Array)
39
- raise "Iqvoc::SkosImporter#import: Parameter 'file' should be a File or an Array."
40
- end
41
-
42
- # Some general Namespaces to support in any case
43
- @prefixes = {
44
- 'http://www.w3.org/2004/02/skos/core#' => 'skos:',
45
- 'http://www.w3.org/2008/05/skos#' => 'skos:',
46
- 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' => 'rdf:',
47
- default_namespace_url => ':'
48
- }
49
- # Add the namespaces specified in the Iqvoc config
50
- Iqvoc.rdf_namespaces.each do |pref, uri|
51
- @prefixes[uri] = "#{pref.to_s}:"
52
- end
53
-
54
- @seen_first_level_objects = {} # Concept cache (don't load any concept twice from db)
55
-
56
- # Assign the default concept scheme singleton instance as a seen first level object upfront
57
- # in order to handle a missing scheme definition in ntriple data
58
- @seen_first_level_objects[Iqvoc::Concept.root_class.instance.origin] = Iqvoc::Concept.root_class
59
-
60
- @new_subjects = {} # Concepts, collections, labels etc. to be published later
61
-
62
- # Triples the importer doesn't understand immediately. Example:
63
- #
64
- # :a skos:prefLabel "foo". # => What is :a? Remember this and try again later
65
- # ....
66
- # :a rdf:type skos:Concept # => Now I know :a, good I remembered it's prefLabel...
67
- @unknown_second_level_triples = Set.new
68
-
69
- # Hash of arrays of arrays: { "_:n123" => [["pred1", "obj1"], ["pred2", "obj2"]] }
70
- @blank_nodes = {}
71
-
72
- @existing_origins = {} # To prevent the creation of first level objects we already have
73
- first_level_object_classes.each do |klass|
74
- klass.select('origin').load.each do |thing|
75
- @existing_origins[thing.origin] = klass
76
- end
77
- end
78
- end
79
-
80
- def run
81
- print_known_namespaces
82
- print_known_import_classes
83
- import @file
84
- end
85
-
86
- private
87
-
88
- def import(file)
89
- ActiveSupport.run_load_hooks(:skos_importer_before_import, self)
90
-
91
- start = Time.now
92
-
93
- @logger.info "default namespace: '#{@default_namespace_url}'"
94
- @logger.info "publish: '#{@publish}'"
95
-
96
- first_level_types = {} # type identifier ("namespace:SomeClass") to Iqvoc class assignment hash
97
- first_level_object_classes.each do |klass|
98
- first_level_types["#{klass.rdf_namespace}:#{klass.rdf_class}"] = klass
99
- end
100
- second_level_types = {}
101
- second_level_object_classes.each do |klass|
102
- second_level_types["#{klass.rdf_namespace}:#{klass.rdf_predicate}"] = klass
103
- end
104
-
105
- @logger.info 'Iqvoc::SkosImporter: Importing triples...'
106
- file.each_with_index do |line, index|
107
- extracted_triple = *extract_triple(line)
108
-
109
- if @verbose && has_unknown_namespaces?(extracted_triple)
110
- @logger.warn "Iqvoc::SkosImporter: Unknown namespaces. Skipping #{extracted_triple.join(' ')}"
111
- end
112
-
113
- identify_blank_nodes(*extracted_triple) ||
114
- import_first_level_objects(first_level_types, *extracted_triple) ||
115
- import_second_level_objects(second_level_types, false, line)
116
- end
117
-
118
- # treeify blank nodes hash
119
- @blank_nodes.each do |origin, bnode_struct|
120
- tranform_blank_node(origin, bnode_struct)
121
- end
122
-
123
- @logger.info "Computing 'forward' defined triples..."
124
- @unknown_second_level_triples.each do |line|
125
- import_second_level_objects(second_level_types, true, line)
126
- end
127
-
128
- first_import_step_done = Time.now
129
- @logger.info "Basic import done (took #{(first_import_step_done - start).to_i} seconds)."
130
-
131
- published = publish
132
-
133
- done = Time.now
134
- @logger.info "Publishing of #{published} subjects done (took #{(done - first_import_step_done).to_i} seconds). #{@new_subjects.count - published} are in draft state."
135
- @logger.info "Imported #{published} published and #{@new_subjects.count - published} draft subjects in #{(done - start).to_i} seconds."
136
- @logger.info "First step took #{(first_import_step_done - start).to_i} seconds, publishing took #{(done - first_import_step_done).to_i} seconds."
137
-
138
- ActiveSupport.run_load_hooks(:skos_importer_after_import, self)
139
- end
140
-
141
- def publish
142
- published = 0
143
- # Respect order of first level classes configured in FIRST_LEVEL_OBJECTS
144
- # Example: XL labels have to be published before referencing concepts
145
- sorted_new_subjects = @new_subjects.sort_by do |origin, klass|
146
- first_level_object_classes.index(klass)
147
- end
148
-
149
- if @publish
150
- @logger.info "Publishing #{@new_subjects.count} new subjects..."
151
-
152
- sorted_new_subjects.each do |origin, klass|
153
- subject = klass.find_by(origin: origin)
154
- if subject.publishable?
155
- subject.publish!
156
- published += 1
157
- else
158
- @logger.warn "WARNING: Publishing failed! Subject ('#{subject.origin}') invalid: #{subject.errors.to_hash.inspect}"
159
- end
160
- end
161
- end
162
- published
163
- end
164
-
165
- def identify_blank_nodes(subject, predicate, object)
166
- if blank_node?(subject)
167
- @blank_nodes[subject] ||= []
168
- @blank_nodes[subject] << [predicate, object]
169
- true
170
- else
171
- false
172
- end
173
- end
174
-
175
- def import_first_level_objects(types, subject, predicate, object)
176
- if (predicate == 'rdf:type' && types[object] && subject =~ /^:(.+)$/)
177
- # We've found a subject definition with a class we know and which is in our responsibility (":")
178
- origin = $1
179
-
180
- if (@existing_origins[origin])
181
- if (types[object] == @existing_origins[origin])
182
- @logger.info "Iqvoc::SkosImporter: Subject with origin '#{origin}' already exists. Skipping duplicate creation (should be no problem)."
183
- else
184
- @logger.warn "Iqvoc::SkosImporter: Subject with origin '#{origin} already exists but has another class (#{@existing_origins[origin]}) then the one I wanted to create (#{types[object]}). You seem to have a problem with your configuration!"
185
- end
186
- else
187
- @logger.info "Iqvoc::SkosImporter: Creating Subject: #{subject} #{predicate} #{object}" if @verbose
188
- # FIXME
189
-
190
- types[object].create do |klass|
191
- klass.origin = origin
192
- end
193
- @seen_first_level_objects[origin] = types[object]
194
- @new_subjects[origin] = types[object]
195
- end
196
- true
197
- else
198
- false
199
- end
200
- end
201
-
202
- def import_second_level_objects(types, final, line)
203
- subject, predicate, object = *extract_triple(line)
204
-
205
- return unless (subject =~ /^:(.*)$/ && types[predicate]) # We're not responsible for this
206
-
207
- # Load the subject and replace the string by the respective data object
208
- subject_origin = $1
209
- subject = load_first_level_object(subject_origin)
210
- unless subject
211
- if final
212
- @logger.warn "Iqvoc::SkosImporter: Couldn't find Subject with origin '#{subject_origin}. Skipping entry '#{subject} #{predicate} #{object}.'"
213
- else
214
- @unknown_second_level_triples << line
215
- end
216
- return false
217
- end
218
-
219
- # Load the data object for the object string if this is representing a thing in our domain
220
- if (object =~ /^:(.*)$/ && types[predicate])
221
- object_origin = $1
222
- object = load_first_level_object(object_origin)
223
- unless object
224
- if final
225
- @logger.warn "Iqvoc::SkosImporter: Couldn't find Object with origin '#{object_origin}'. Skipping entry ':#{subject_origin} #{predicate} #{object_origin}.'"
226
- else
227
- @unknown_second_level_triples << line
228
- end
229
- return false
230
- end
231
- end
232
-
233
- # If not in final mode every :my_concept :bla _:blank_node. triple should
234
- # be saved for final mode. Why? Example:
235
- #
236
- # :a iqvoc:changeNote _:b01 # => I do not know know anything about the blank node now
237
- # _:b01 dc:author "DHH"...
238
- #
239
- if blank_node?(object)
240
- if final
241
- object = @blank_nodes[object]
242
- else
243
- @unknown_second_level_triples << line
244
- return false
245
- end
246
- end
247
- begin
248
- types[predicate].build_from_rdf(subject, predicate, object)
249
- rescue InvalidStringLiteralError => e
250
- @logger.warn e.message
251
- end
252
- end
253
-
254
- def load_first_level_object(origin)
255
- unless @seen_first_level_objects[origin]
256
- klass = @existing_origins[origin]
257
- if klass
258
- @seen_first_level_objects[origin] = klass
259
- end
260
- end
261
-
262
- # FIXME: bang
263
- # FIXME: return something?
264
- if klass = @seen_first_level_objects[origin]
265
- klass.find_by!(origin: origin)
266
- end
267
- end
268
-
269
- def blank_node?(str)
270
- str.dup.to_s =~ /^_:.+/
271
- end
272
-
273
- def extract_triple(line)
274
- raise "'#{line}' doesn't look like valid ntriples data." unless line =~ /^(.*)\.\s*$/
275
- line = $1.squish
276
-
277
- triple = line.split(' ', 3) # The first one are uris the last can be a literal too
278
-
279
- triple.each do |e| # Do some fun with the uris and literals
280
- @prefixes.keys.each do |uri_prefix| # Use prefixes instead of full uris
281
- e.gsub! /^<#{uri_prefix}([^>]*)>/ do |matches|
282
- @prefixes[uri_prefix] + $1.gsub('.', '_')
283
- end
284
- end
285
- e.squish!
286
- e.gsub!(/^:(.*)$/) do
287
- ":#{Iqvoc::Origin.new($1)}" # Force correct origins in the default namespace
288
- end
289
- end
290
- triple
291
- end
292
-
293
- def has_unknown_namespaces?(triple)
294
- triple.each do |obj|
295
- return true if obj =~ /^<.*>$/
296
- break
297
- end
298
- false
299
- end
300
-
301
- # if blank node contains another blank node,
302
- # move child blank node to his parent
303
- def tranform_blank_node(origin, bnode_struct)
304
- bnode_struct.each_index do |i|
305
- bnode_origin = bnode_struct[i][1] # only origin could contain another blank node
306
- if blank_node?(bnode_origin)
307
- bnode_child_struct = @blank_nodes[bnode_origin]
308
- bnode_struct[i][1] = bnode_child_struct # move to parent node
309
- tranform_blank_node(bnode_origin, bnode_child_struct)
310
-
311
- @blank_nodes.delete(bnode_origin) # remove old blank node
312
- end
313
- end
314
- end
315
-
316
- def print_known_namespaces
317
- @logger.info "Known namespaces:"
318
- @prefixes.each_with_index do |(uri, pref), i|
319
- @logger.info "\t #{i+1}: #{pref} => #{uri}"
320
- end
321
- end
322
-
323
- def print_known_import_classes
324
- @logger.info "Known first level classes:"
325
- first_level_object_classes.each_with_index do |floc, i|
326
- @logger.info "\t #{i+1}: #{floc.rdf_namespace}:#{floc.rdf_class} => #{floc.to_s}"
327
- end
328
-
329
- @logger.info "Known second level classes:"
330
- second_level_object_classes.each_with_index do |sloc, i|
331
- @logger.info "\t #{i+1}: #{sloc.rdf_namespace}:#{sloc.rdf_predicate} => #{sloc.to_s}"
332
- end
333
- end
334
-
335
- ActiveSupport.run_load_hooks(:skos_importer, self)
336
- end
337
- end