solis 0.69.0 → 0.71.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (9) hide show
  1. checksums.yaml +4 -4
  2. data/lib/solis/config_file.rb +6 -1
  3. data/lib/solis/graph.rb +8 -0
  4. data/lib/solis/query.rb +1 -0
  5. data/lib/solis/shape/reader/sheet.rb +198 -338
  6. data/lib/solis/shape.rb +3 -0
  7. data/lib/solis/version.rb +1 -1
  8. data/solis.gemspec +8 -8
  9. metadata +21 -21
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fb7b4c2f87c4dfaa321b9e986296cbcef41849230a8c29efcd3385e6168e7034
4
- data.tar.gz: 9992f2c9eef0d56c9ec0fc53fa68b45a8b5efb8e9065ddd637427c9f7a3b2087
3
+ metadata.gz: 16ed244eaf62cd747c1e2c2ea369066dcd6a03e529d3932916b4f7791a29b956
4
+ data.tar.gz: ef2a72fd541967ccf54ae75ca4f18abb21727cf9398c4da38b9aff3a8b9b5ca7
5
5
  SHA512:
6
- metadata.gz: '0284ecdce9377633bddef224daf54c25862a65e3eef78995db67b61380cae989958c088aba229c3ee100106008c948031679d61b845f4a8dbfacc5d540e0ebfb'
7
- data.tar.gz: d6cbf0d9e0bcf54c7e5a129eb56575bea5af2391ab9f64afdd268c420e747c3c2f5577b5ce84d05ca8f8de6acf094287d3732e31fb5e8ceb2bddd89fc408eb29
6
+ metadata.gz: b955562d76f6ca7942d634098b430c4e19c21c64323c8c676fcacbc494949b9d38aaf6d271c197fd3e8ed49bf200534c1ab473fd56aa883d3bbeda9136482427
7
+ data.tar.gz: f80157b099693522c7afa1a0ce607d174ab164a76ad94a69f359a8331588a858710b3a98ee2360cc6d10ef8e7c1299383eb7a129eac7c94773295bc95604fae1
data/lib/solis/config_file.rb CHANGED
@@ -51,12 +51,17 @@ class ConfigFile
51
51
  @config
52
52
  end
53
53
 
54
+ def self.keys
55
+ init
56
+ @config.keys
57
+ end
58
+
54
59
  private
55
60
 
56
61
  def self.init
57
62
  discover_config_file_path
58
63
  if @config.empty?
59
- config = YAML::load_file("#{path}/#{name}")
64
+ config = YAML::load_file("#{path}/#{name}", aliases: true)
60
65
  @config = process(config)
61
66
  end
62
67
  end
data/lib/solis/graph.rb CHANGED
@@ -79,6 +79,9 @@ module Solis
79
79
  unless @inflections.nil?
80
80
  raise "Inflection file not found #{File.absolute_path(@inflections)}" unless File.exist?(@inflections)
81
81
  JSON.parse(File.read(@inflections)).each do |s, p|
82
+ raise "No plural found" if s.nil? && p.nil?
83
+ raise "No plural found for #{p}" if s.nil?
84
+ raise "No plural found for #{s}" if p.nil?
82
85
  ActiveSupport::Inflector.inflections.irregular(s, p)
83
86
  end
84
87
  end
@@ -89,6 +92,7 @@ module Solis
89
92
  LOGGER.warn("Dangling entity found #{_[:target_class].to_s} removing")
90
93
  next
91
94
  end
95
+ #@shapes[shape_name][:attributes].select { |_, metadata| metadata.key?(:node_kind) && !metadata[:node_kind].nil? }.values.map { |m| m[:datatype].to_s.split('#').last }
92
96
  @shapes[shape_name][:attributes].select { |_, metadata| metadata.key?(:node_kind) && !metadata[:node_kind].nil? }.values.map { |m| m[:datatype].to_s }
93
97
  end
94
98
  shape_keys += @shapes.keys
@@ -221,6 +225,10 @@ module Solis
221
225
  relations.each_key do |k|
222
226
  next if relations[k][:node_kind].is_a?(RDF::URI) && relations[k][:class].value.gsub(@graph_name, '').gsub('Shape', '').eql?(shape_name)
223
227
  relation_shape = relations[k][:class].value.gsub(@graph_name, '').gsub('Shape', '')
228
+ if relation_shape =~ /\//
229
+ relation_shape = relations[k][:class].value.split('/').last.gsub('Shape','')
230
+ end
231
+
224
232
  shape_as_resource(relation_shape, stack_level << relation_shape) unless stack_level.include?(relation_shape)
225
233
  end
226
234
 
data/lib/solis/query.rb CHANGED
@@ -119,6 +119,7 @@ module Solis
119
119
  core_query = core_query(relationship)
120
120
  count_query = core_query.gsub(/SELECT .* WHERE/, 'SELECT (COUNT(distinct ?concept) as ?count) WHERE')
121
121
 
122
+ count_query = count_query.split('a ?type')[0]+'a ?type }'
122
123
  result = sparql_client.query(count_query)
123
124
  solution = result.first
124
125
  solution.nil? ? 0 : solution[:count].object || 0
data/lib/solis/shape/reader/sheet.rb CHANGED
@@ -11,33 +11,41 @@ module Solis
11
11
  def self.read(key, spreadsheet_id, options = {})
12
12
  class << self
13
13
  def validate(sheets)
14
- raise "Please make sure the sheet contains '_PREFIXES', '_METADATA', '_ENTITIES' tabs" unless (%w[
15
- _PREFIXES _METADATA _ENTITIES
16
- ] - sheets.keys).length == 0
14
+ #raise "Please make sure the sheet contains '_PREFIXES', '_METADATA', '_ENTITIES' tabs" unless (%w[_PREFIXES _METADATA _ENTITIES] - sheets.keys).length == 0
17
15
 
18
16
  prefixes = sheets['_PREFIXES']
19
17
  metadata = sheets['_METADATA']
20
- entities = sheets['_ENTITIES']
21
-
22
- raise "_PREFIXES tab must have ['base', 'prefix', 'uri'] as a header at row 1" unless (%w[base prefix
23
- uri] - prefixes.header).length == 0
24
- raise "_METADATA tab must have ['key', 'value'] as a header at row 1" unless (%w[key
25
- value] - metadata.header).length == 0
26
- raise "_ENTITIES tab must have ['name', 'nameplural', 'description', 'subclassof', 'sameas'] as a header at row 1" unless (%w[
27
- name nameplural description subclassof sameas
28
- ] - entities.header).length == 0
29
18
 
19
+ raise "_PREFIXES tab must have ['base', 'prefix', 'uri'] as a header at row 1" unless (%w[base prefix uri] - prefixes.header).length == 0
30
20
  raise '_PREFIXES.base can only have one base URI' if prefixes.map { |m| m['base'] }.grep(/\*/).count != 1
21
+
22
+ raise "_METADATA tab must have ['key', 'value'] as a header at row 1" unless (%w[key value] - metadata.header).length == 0
23
+
24
+ if sheets.key?('_ENTITIES')
25
+ entities = sheets['_ENTITIES']
26
+ raise "_ENTITIES tab must have ['name', 'nameplural', 'description', 'subclassof', 'sameas'] as a header at row 1" unless (%w[name nameplural description subclassof sameas] - entities.header).length == 0
27
+
28
+ entities.each do |entity|
29
+ raise "Plural not found for #{entity['name']}" if entity['nameplural'].nil? || entity['nameplural'].empty?
30
+ end
31
+ end
32
+
33
+ if sheets.key?('_REFERENCES')
34
+ references = sheets['_REFERENCES']
35
+ raise "_REFERENCES tab must have ['sheeturl', 'description', 'entityrange'] as a header at row 1" unless (%w[sheeturl description entityrange] - references.header).length == 0
36
+ end
37
+
31
38
  end
32
39
 
33
40
  def read_sheets(key, spreadsheet_id, options)
34
41
  data = nil
35
42
 
36
- cache_dir = ConfigFile.include?(:solis) && ConfigFile[:solis].include?(:cache) ? ConfigFile[:solis][:cache] : '/tmp'
43
+ cache_dir = ConfigFile.include?(:cache) ? ConfigFile[:cache] : '/tmp'
37
44
 
38
45
  if ::File.exist?("#{cache_dir}/#{spreadsheet_id}.json") && (options.include?(:from_cache) && options[:from_cache])
39
46
  Solis::LOGGER.info("from cache #{cache_dir}/#{spreadsheet_id}.json")
40
47
  data = JSON.parse(::File.read("#{cache_dir}/#{spreadsheet_id}.json"), { symbolize_names: true })
48
+ return data
41
49
  else
42
50
  Solis::LOGGER.info("from source #{spreadsheet_id}")
43
51
  session = SimpleSheets.new(key, spreadsheet_id)
@@ -49,66 +57,69 @@ module Solis
49
57
  end
50
58
 
51
59
  validate(sheets)
60
+ end
61
+ sheets
62
+ end
52
63
 
53
- entities = {}
54
- prefixes = {}
55
- ontology_metadata = {}
64
+ def process_sheet(key, sheet_id, sheets, options = {follow: true})
65
+ entities = {}
66
+ prefixes = {}
67
+ ontology_metadata = {}
56
68
 
57
- sheets['_PREFIXES'].each do |e|
58
- prefixes.store(e['prefix'].to_sym, { uri: e['uri'], base: e['base'].eql?('*') })
69
+ sheets['_PREFIXES'].each do |e|
70
+ prefixes.store(e['prefix'].to_sym, { uri: e['uri'], base: e['base'].eql?('*') })
71
+ end
72
+ sheets['_METADATA'].each { |e| ontology_metadata.store(e['key'].to_sym, e['value']) }
73
+
74
+ base_uri = prefixes.select { |_k, v| v[:base] }.select { |s| !s.empty? }
75
+
76
+ graph_prefix = base_uri.keys.first
77
+ graph_name = base_uri.values.first[:uri]
78
+
79
+ sheets['_ENTITIES'].each do |e|
80
+
81
+ top_class = e['name'].to_s
82
+ #if prefixes[graph_prefix][:data].nil? || prefixes[graph_prefix][:data].empty?
83
+ entity_data = parse_entity_data(e['name'].to_s, graph_prefix, graph_name, sheets[top_class], { key: key, prefixes: prefixes, follow: options[:follow] })
84
+ # prefixes[graph_prefix][:data] = entity_data
85
+ #else
86
+ # entity_data = prefixes[graph_prefix][:data]
87
+ #end
88
+
89
+ if entity_data.empty?
90
+ entity_data[:id] = {
91
+ datatype: 'xsd:string',
92
+ path: "#{graph_prefix}:id",
93
+ cardinality: { min: '1', max: '1' },
94
+ same_as: '',
95
+ description: 'systeem UUID'
96
+ }
59
97
  end
60
- sheets['_METADATA'].each { |e| ontology_metadata.store(e['key'].to_sym, e['value']) }
61
-
62
- base_uri = prefixes.select { |_k, v| v[:base] }.select { |s| !s.empty? }
63
-
64
- graph_prefix = base_uri.keys.first
65
- graph_name = base_uri.values.first[:uri]
66
-
67
- sheets['_ENTITIES'].each do |e|
68
-
69
- top_class = e['name'].to_s
70
- # subclassof = e['subclassof'].empty? ? nil : e['subclassof'].split(':').last
71
- # while subclassof
72
- # candidate_sco = sheets['_ENTITIES'].select{|t| t['name'].eql?(subclassof)}.first
73
- # subclassof = candidate_sco['subclassof'].empty? ? nil : candidate_sco['subclassof'].split(':').last
74
- # top_class = candidate_sco['name'].to_s if candidate_sco['subclassof'].empty?
75
- # end
76
-
77
- entity_data = parse_entity_data(e['name'].to_s, graph_prefix, graph_name, sheets[top_class])
78
98
 
79
- if entity_data.empty?
80
- entity_data[:id] = {
81
- datatype: 'xsd:string',
82
- path: "#{graph_prefix}:id",
83
- cardinality: { min: '1', max: '1' },
84
- same_as: '',
85
- description: 'systeem UUID'
86
- }
87
- end
88
-
89
- entities.store(e['name'].to_sym, { description: e['description'],
90
- plural: e['nameplural'],
91
- label: e['name'].to_s.strip,
92
- sub_class_of: e['subclassof'].nil? || e['subclassof'].empty? ? [] : [e['subclassof']],
93
- same_as: e['sameas'],
94
- properties: entity_data })
95
- end
99
+ entities.store(e['name'].to_sym, { description: e['description'],
100
+ plural: e['nameplural'],
101
+ label: e['name'].to_s.strip,
102
+ sub_class_of: e['subclassof'].nil? || e['subclassof'].empty? ? [] : [e['subclassof']],
103
+ same_as: e['sameas'],
104
+ properties: entity_data })
105
+ end
96
106
 
97
- data = {
98
- entities: entities,
99
- ontologies: {
100
- all: prefixes,
101
- base: {
102
- prefix: graph_prefix,
103
- uri: graph_name
104
- }
105
- },
106
- metadata: ontology_metadata
107
- }
107
+ data = {
108
+ entities: entities,
109
+ ontologies: {
110
+ all: prefixes,
111
+ base: {
112
+ prefix: graph_prefix,
113
+ uri: graph_name
114
+ }
115
+ },
116
+ metadata: ontology_metadata
117
+ }
108
118
 
109
- ::File.open("#{::File.absolute_path(cache_dir)}/#{spreadsheet_id}.json", 'wb') do |f|
110
- f.puts data.to_json
111
- end
119
+ cache_dir = ConfigFile.include?(:cache) ? ConfigFile[:cache] : '/tmp'
120
+ # ::File.open("#{::File.absolute_path(cache_dir)}/#{spreadsheet_id}.json", 'wb') do |f|
121
+ ::File.open("#{::File.absolute_path(cache_dir)}/#{sheet_id}.json", 'wb') do |f|
122
+ f.puts data.to_json
112
123
  end
113
124
 
114
125
  data
@@ -116,11 +127,12 @@ module Solis
116
127
  raise Solis::Error::GeneralError, e.message
117
128
  end
118
129
 
119
- def parse_entity_data(name, graph_prefix, _graph_name, e)
130
+ def parse_entity_data(entity_name, graph_prefix, _graph_name, e, options = {})
120
131
  properties = {}
121
132
  entity_data = e
122
133
  if entity_data && !entity_data.nil? && (entity_data.count > 0)
123
134
  entity_data.each do |p|
135
+ property_name = I18n.transliterate(p['name'].strip)
124
136
  min_max = {}
125
137
 
126
138
  %w[min max].each do |n|
@@ -130,24 +142,54 @@ module Solis
130
142
  ''
131
143
  end
132
144
  end
133
- puts "#{name}.#{p['name']}"
145
+ puts "#{entity_name}.#{property_name}"
134
146
  unless p.key?('name')
147
+ puts "No 'name' property found"
135
148
  pp p
136
149
  end
137
- properties[p['name'].strip] = {
138
- datatype: p['datatype'],
139
- path: "#{graph_prefix}:#{p['name'].to_s.classify}",
140
- cardinality: { min: min_max['min'], max: min_max['max'] },
141
- same_as: p['sameAs'],
142
- description: p['description']
143
- }
150
+
151
+ if properties.key?(property_name)
152
+ puts "Found #{entity_name}.#{property_name}"
153
+ else
154
+ datatype_prefix, datatype_name = p['datatype'].split(':')
155
+ properties[property_name] = {
156
+ datatype: p['datatype'],
157
+ path: "#{graph_prefix}:#{property_name.to_s.classify}",
158
+ cardinality: { min: min_max['min'], max: min_max['max'] },
159
+ same_as: p['sameAs'],
160
+ description: p['description']
161
+ }
162
+
163
+ # unless graph_prefix.eql?(datatype_prefix.to_sym)
164
+ # prefixes = options[:prefixes]
165
+ # if prefixes.key?(datatype_prefix.to_sym) && !prefixes[datatype_prefix.to_sym][:sheet_url].empty?
166
+ # tmp = URI(prefixes[datatype_prefix.to_sym][:sheet_url]).path.split('/')
167
+ # spreadsheet_id = tmp[tmp.index('d') + 1]
168
+ #
169
+ # processed_remote_sheet = {}
170
+ # if prefixes[datatype_prefix.to_sym].key?(:data) && !prefixes[datatype_prefix.to_sym][:data].empty?
171
+ # processed_remote_sheet = prefixes[datatype_prefix.to_sym][:data]
172
+ # else
173
+ # if options[:follow]
174
+ # sleep 30
175
+ # remote_sheet = read_sheets(options[:key], spreadsheet_id, { from_cache: true })
176
+ # processed_remote_sheet = process_sheet(options[:key], remote_sheet, {follow: false})
177
+ # prefixes[datatype_prefix.to_sym][:data] = processed_remote_sheet
178
+ # end
179
+ # end
180
+ #
181
+ # processed_remote_sheet
182
+ # end
183
+ # end
184
+
185
+ end
144
186
  end
145
187
  end
146
188
 
147
189
  properties
148
190
  end
149
191
 
150
- def build_plantuml(data)
192
+ def build_plantuml(datas)
151
193
  out = %(@startuml
152
194
  !pragma layout elk
153
195
  skinparam classFontSize 14
@@ -157,10 +199,11 @@ skinparam componentStyle uml2
157
199
  skinparam wrapMessageWidth 100
158
200
  skinparam ArrowColor #Maroon
159
201
 
160
- title #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}
202
+ title #{datas.first[:metadata][:title]} - #{datas.first[:metadata][:version]} - #{Time.now}
161
203
  )
162
204
 
163
- out += "\npackage #{data[:ontologies][:base][:prefix]} {\n"
205
+ out += "\npackage #{datas.first[:ontologies][:base][:prefix]} {\n"
206
+ datas.each do |data|
164
207
  data[:entities].each do |entity_name, metadata|
165
208
  out += "\nclass #{entity_name}"
166
209
 
@@ -186,7 +229,7 @@ title #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}
186
229
  out += "#{entity_name} --|> #{sub_class.split(':').last}\n" unless sub_class.empty?
187
230
  end
188
231
  end
189
-
232
+ end
190
233
  out += %(
191
234
  hide circle
192
235
  hide methods
@@ -195,6 +238,8 @@ hide empty members
195
238
  )
196
239
 
197
240
  out
241
+ rescue StandardError => e
242
+ puts e.message
198
243
  end
199
244
 
200
245
  def datatype_lookup(datatype, as = :sql)
@@ -210,86 +255,14 @@ hide empty members
210
255
  datatypes[datatype][as]
211
256
  end
212
257
 
213
- def build_plantuml_erd(data)
214
- cardinality_min = { '0' => '|o', '' => '}o', '1' => '||' }
215
- cardinality_max = { '0' => 'o|', '' => 'o{', '1' => '||' }
216
-
217
- out = %(@startuml
218
- skinparam classFontSize 14
219
- !define LIGHTORANGE
220
- skinparam groupInheritance 1
221
- skinparam componentStyle uml2
222
- skinparam wrapMessageWidth 100
223
- skinparam ArrowColor #Maroon
224
- skinparam linetype ortho
225
-
226
- title #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}
227
- )
228
-
229
- out += "\npackage #{data[:ontologies][:base][:prefix]} {\n"
230
- relations = []
231
- data[:entities].each do |_entity_name, metadata|
232
- table_name = metadata[:plural].to_s.underscore
233
- # out += "\nentity \"#{entity_name}\" as #{table_name}"
234
- out += "\nentity \"#{table_name}\" as #{table_name}"
235
-
236
- properties = metadata[:properties]
237
- # relations = []
238
- unless properties.nil? || properties.empty?
239
- out += "{\n"
240
- properties.each do |property, property_metadata|
241
- if property.to_s.eql?('id')
242
- out += "\t *#{property} : #{datatype_lookup(property_metadata[:datatype], :sql)} <<generated>>\n"
243
- out += "--\n"
244
- else
245
- mandatory = property_metadata[:cardinality][:min].to_i > 0
246
- is_fk = property_metadata[:datatype].split(':').first.eql?(data[:ontologies][:base][:prefix].to_s) ? true : false
247
- out += "\t #{mandatory ? '*' : ''}#{property}#{is_fk ? '_id' : ''} : #{datatype_lookup(
248
- property_metadata[:datatype], :sql
249
- )} #{is_fk ? '<<FK>>' : ''} \n"
250
- end
251
-
252
- unless property_metadata[:datatype].split(':').first.eql?(data[:ontologies][:base][:prefix].to_s)
253
- next
254
- end
255
-
256
- cmin = cardinality_min[(property_metadata[:cardinality][:min]).to_s]
257
- cmax = cardinality_max[(property_metadata[:cardinality][:max]).to_s]
258
-
259
- # relations << " #{entity_name.to_s.underscore} #{cmin}--o{ #{property_metadata[:datatype].split(':').last.to_s.underscore} "
260
- # ref_table_name = property_metadata[:datatype].split(':').last.to_s.underscore
261
- ref_table_name = [property_metadata[:datatype].split(':').last.to_sym,
262
- property_metadata[:path].split(':').last.classify.to_sym].map do |m|
263
- data[:entities][m].nil? ? nil : data[:entities][m][:plural].underscore
264
- end.compact.first
265
-
266
- relations << " #{table_name} #{cmin}--#{cmax} #{ref_table_name} "
267
- end
268
- out += "}\n"
269
- end
270
-
271
- out += "\n"
272
- # out += "#{entity_name} }o-- #{metadata[:sub_class_of].split(':').last}\n" unless metadata[:sub_class_of].empty?
273
- end
274
- out += relations.join("\n")
275
-
276
- out += %(
277
- hide circle
278
- hide methods
279
- hide empty members
280
- @enduml
281
- )
282
-
283
- out
284
- end
285
-
286
- def build_shacl(data)
287
- shacl_prefix = data[:ontologies][:all].select { |_, v| v[:uri] =~ /shacl/ }.keys.first
258
+ def build_shacl(datas)
259
+ shacl_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /shacl/ }.keys.first
288
260
  shacl_prefix = 'sh' if shacl_prefix.nil?
289
261
 
290
- out = header(data)
262
+ out = header(datas.first)
291
263
 
292
- data[:entities].each do |entity_name, metadata|
264
+ datas.each do |data|
265
+ data[:entities].each do |entity_name, metadata|
293
266
  graph_prefix = data[:ontologies][:base][:prefix]
294
267
  graph_name = data[:ontologies][:base][:uri]
295
268
 
@@ -301,7 +274,7 @@ hide empty members
301
274
  if node && !node.empty?
302
275
  node = node.first if node.is_a?(Array)
303
276
  node = node.strip
304
- node += 'Shape' if node != /Shape$/ && node =~ /^#{graph_prefix}:/
277
+ node += 'Shape' if node != /Shape$/ # && node =~ /^#{graph_prefix}:/
305
278
  else
306
279
  node = target_class
307
280
  end
@@ -345,22 +318,25 @@ hide empty members
345
318
  end
346
319
  out += ".\n"
347
320
  end
348
-
321
+ end
349
322
  out
323
+ rescue StandardError => e
324
+ puts e.message
350
325
  end
351
326
 
352
- def build_schema(data)
327
+ def build_schema(datas)
353
328
  classes = {}
354
329
  datatype_properties = {}
355
330
  object_properties = {}
356
331
 
357
332
  format = :ttl
358
- graph_prefix = data[:ontologies][:base][:prefix]
359
- graph_name = data[:ontologies][:base][:uri]
333
+ graph_prefix = datas.first[:ontologies][:base][:prefix]
334
+ graph_name = datas.first[:ontologies][:base][:uri]
360
335
 
361
336
  all_prefixes = {}
362
- data[:ontologies][:all].each { |k, v| all_prefixes[k] = v[:uri] }
337
+ datas.first[:ontologies][:all].each { |k, v| all_prefixes[k] = v[:uri] }
363
338
 
339
+ datas.each do |data|
364
340
  data[:entities].each do |entity_name, metadata|
365
341
  classes[entity_name] = {
366
342
  comment: metadata[:description],
@@ -404,15 +380,16 @@ hide empty members
404
380
  data[:entities][entity_name][:sub_class_of] = subclass_data
405
381
  end
406
382
  end
383
+ end
407
384
 
408
385
  lp = RDF::StrictVocabulary(graph_name)
409
386
  o = ::Class.new(lp) do
410
387
  ontology(graph_name.to_sym, {
411
- "dc11:title": data[:metadata][:title].freeze,
412
- "dc11:description": data[:metadata][:description].freeze,
388
+ "dc11:title": datas.first[:metadata][:title].freeze,
389
+ "dc11:description": datas.first[:metadata][:description].freeze,
413
390
  "dc11:date": Time.now.to_s.freeze,
414
- "dc11:creator": data[:metadata][:author].freeze,
415
- "owl:versionInfo": data[:metadata][:version].freeze,
391
+ "dc11:creator": datas.first[:metadata][:author].freeze,
392
+ "owl:versionInfo": datas.first[:metadata][:version].freeze,
416
393
  type: 'owl:Ontology'.freeze
417
394
  })
418
395
 
@@ -433,6 +410,7 @@ hide empty members
433
410
  graph = RDF::Graph.new
434
411
  graph.graph_name = RDF::URI(graph_name)
435
412
 
413
+ datas.each do |data|
436
414
  data[:entities].select { |_k, v| !v[:same_as].empty? }.each do |k, v|
437
415
  prefix, verb = v[:same_as].split(':')
438
416
  rdf_vocabulary = RDF::Vocabulary.from_sym(prefix.upcase)
@@ -442,178 +420,27 @@ hide empty members
442
420
  rescue StandardError => e
443
421
  puts e.message
444
422
  end
423
+ end
445
424
 
446
425
  graph << o.to_enum
447
426
 
448
427
  graph.dump(format, prefixes: all_prefixes)
428
+ rescue StandardError => e
429
+ puts e.message
449
430
  end
450
431
 
451
- def build_inflections(data)
432
+ def build_inflections(datas)
452
433
  inflections = {}
434
+ datas.each do |data|
453
435
  data[:entities].each do |entity, metadata|
454
436
  inflections[entity] = metadata[:plural]
455
437
  inflections[entity.to_s.underscore.to_sym] = metadata[:plural].underscore
456
438
  end
457
-
458
- inflections.to_json
459
- end
460
-
461
- def build_sql(data)
462
- graph_prefix = data[:ontologies][:base][:prefix]
463
- out = "--\n-- #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}\n"
464
- out += "-- description: #{data[:metadata][:description]}\n"
465
- out += "-- author: #{data[:metadata][:author]}\n--\n\n"
466
-
467
- out += %(CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
468
- DROP SCHEMA IF EXISTS #{graph_prefix} CASCADE;
469
- CREATE SCHEMA #{graph_prefix};
470
-
471
- )
472
- data[:entities].each do |entity_name, metadata|
473
- table_name = metadata[:plural].to_s.underscore
474
- out += "CREATE TABLE #{graph_prefix}.#{table_name}(\n"
475
-
476
- properties = metadata[:properties]
477
- properties.each_with_index do |(property, property_metadata), i|
478
- mandatory = property_metadata[:cardinality][:min].to_i > 0
479
- is_fk = property_metadata[:datatype].split(':').first.eql?(data[:ontologies][:base][:prefix].to_s) ? true : false
480
- if data[:entities][property_metadata[:datatype].split(':').last.to_sym].nil? && is_fk
481
- raise Solis::Error::NotFoundError,
482
- "#{entity_name}.#{property} Not found in _ENTITIES tab"
483
- end
484
-
485
- if is_fk
486
- references = data[:entities][property_metadata[:datatype].split(':').last.to_sym][:plural].to_s.underscore
487
- end
488
-
489
- out += ", \n" if i > 0
490
- if property.to_s.eql?('id')
491
- # out += "\t#{property} #{datatype_lookup(property_metadata[:datatype], :sql)}#{mandatory ? ' NOT NULL' : ''} PRIMARY KEY"
492
- out += "\t#{property} SERIAL#{mandatory ? ' NOT NULL' : ''} PRIMARY KEY"
493
- else
494
- out += "\t#{property}#{is_fk ? '_id' : ''} #{datatype_lookup(property_metadata[:datatype],
495
- :sql)}#{mandatory ? ' NOT NULL' : ''}#{is_fk ? " REFERENCES #{graph_prefix}.#{references}(id)" : ''}"
496
- end
497
- end
498
-
499
- out += ");\n\n"
500
- end
501
-
502
- out
503
- end
504
-
505
- def build_erd(data, type = :uml)
506
- out = erd_header(data, type)
507
- all_tables = {}
508
- tables = {}
509
- relations = []
510
- references = {}
511
- every_entity(data).each do |table|
512
- case type
513
- when :uml
514
- d = table[:table].call(type)
515
- out += d[:out]
516
- relations << d[:relations] unless d[:relations].empty?
517
- out += "\n\n"
518
- #references << d[:references]
519
- when :sql
520
- all_tables[table[:name]] = table
521
- d = table[:table].call(type)
522
- tables[table[:name]] = d[:out]
523
-
524
- d[:references].each do |k, v|
525
- references[k] = (references.include?(k) ? references[k] : 0) + v
526
- end
527
- end
528
- end
529
-
530
- references = references.sort_by { |k, v| -v }.to_h #each{|m| r[m[0]] = m[1]}
531
-
532
- r = references.sort_by { |k, v|
533
- k = k[0]
534
- relation = all_tables.key?(k) ? all_tables[k][:properties].map { |s| s[:references] }.compact.first : nil
535
-
536
- a = references.keys.index(k)
537
- b = references.keys.index(relation)
538
- b = 0 if b.nil?
539
- a = 0 if a.nil?
540
-
541
- b = a + b if b < a
542
-
543
- b
544
- }
545
-
546
- references = r.to_h
547
-
548
- if type.eql?(:sql)
549
- all_keys = references.keys
550
- t = tables.sort_by { |k, v| all_keys.include?(k) ? all_keys.index(k) : 0 }
551
- out += t.map { |m| m[1] }.join("\n")
552
439
  end
553
440
 
554
- # ::File.open("#{ConfigFile[:cache]}/test.json", 'wb') {|f| f.puts references.to_json}
555
-
556
- out += relations.sort.uniq.join("\n")
557
- out += erd_footer(data, type)
558
-
559
- out
560
- end
561
-
562
- def erd_header(data, type)
563
- header = ''
564
-
565
- case type
566
- when :uml
567
- header = %(@startuml
568
- skinparam classFontSize 14
569
- !define LIGHTORANGE
570
- skinparam groupInheritance 1
571
- skinparam componentStyle uml2
572
- skinparam wrapMessageWidth 100
573
- skinparam ArrowColor #Maroon
574
- skinparam linetype ortho
575
-
576
- title #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}
577
-
578
- package #{data[:ontologies][:base][:prefix]} {
579
- )
580
- when :sql
581
- graph_prefix = data[:ontologies][:base][:prefix]
582
- header = %(--
583
- -- #{data[:metadata][:title]} - #{data[:metadata][:version]} - #{Time.now}
584
- -- description: #{data[:metadata][:description]}
585
- -- author: #{data[:metadata][:author]}
586
- --
587
-
588
-
589
- CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
590
- DROP SCHEMA IF EXISTS #{graph_prefix} CASCADE;
591
- CREATE SCHEMA #{graph_prefix};
592
-
593
-
594
- )
595
- end
596
-
597
- header
598
- end
599
-
600
- def erd_footer(_data, type = :uml)
601
- footer = ''
602
-
603
- case type
604
- when :uml
605
- footer = %(
606
-
607
- hide circle
608
- hide methods
609
- hide empty members
610
- @enduml
611
- )
612
- when :sql
613
- footer = ''
614
- end
615
-
616
- footer
441
+ inflections.to_json
442
+ rescue StandardError => e
443
+ puts e.message
617
444
  end
618
445
 
619
446
  def every_entity(data)
@@ -756,20 +583,53 @@ hide empty members
756
583
 
757
584
  "#{out}\n"
758
585
  end
586
+
587
+ def spreadsheet_id_from_url(sheet_url)
588
+ tmp = URI(sheet_url).path.split('/')
589
+ spreadsheet_id = tmp[tmp.index('d') + 1]
590
+ end
591
+ end
592
+
593
+ sheet_data = read_sheets(key, spreadsheet_id, options)
594
+
595
+ if sheet_data.is_a?(Hash)
596
+ raise "No _REFERENCES sheet found" unless sheet_data.key?("_REFERENCES")
597
+ #read other ontologies
598
+ Solis::LOGGER.info('Reading referenced ontologies')
599
+ references = sheet_data['_REFERENCES'].map do |reference|
600
+ {sheet_url: reference['sheeturl'], description: reference['description']}
601
+ end
602
+
603
+ cache_dir = ConfigFile.include?(:cache) ? ConfigFile[:cache] : '/tmp'
604
+ ::File.open("#{::File.absolute_path(cache_dir)}/#{spreadsheet_id}.json", 'wb') do |f|
605
+ f.puts references.to_json
606
+ end
607
+ else
608
+ references = sheet_data
609
+ end
610
+
611
+ datas = []
612
+ references.each do |v|
613
+ sheet_id = spreadsheet_id_from_url(v[:sheet_url])
614
+
615
+ sheet_data = read_sheets(key, sheet_id, options)
616
+ if sheet_data.key?("_PREFIXES")
617
+ datas << process_sheet(key, sheet_id, sheet_data)
618
+ sleep 30
619
+ else
620
+ datas << sheet_data
621
+ end
759
622
  end
760
623
 
761
- data = read_sheets(key, spreadsheet_id, options)
762
-
763
- shacl = build_shacl(data)
764
- plantuml = build_plantuml(data)
765
- #plantuml_erd = build_plantuml_erd(data)
766
- plantuml_erd = build_erd(data, :uml)
767
- schema = build_schema(data)
768
- inflections = build_inflections(data)
769
- sql = build_erd(data, :sql)
770
- #erd = build_erd(data, :uml)
771
- { inflections: inflections, shacl: shacl, schema: schema, plantuml: plantuml,
772
- plantuml_erd: plantuml_erd, sql: sql }
624
+ Solis::LOGGER.info('Generating SHACL')
625
+ shacl = build_shacl(datas)
626
+ Solis::LOGGER.info('Generating PLANTUML')
627
+ plantuml = build_plantuml(datas)
628
+ Solis::LOGGER.info('Generating SCHEMA')
629
+ schema = build_schema(datas)
630
+ Solis::LOGGER.info('Generating INFLECTIONS')
631
+ inflections = build_inflections(datas)
632
+ { inflections: inflections, shacl: shacl, schema: schema, plantuml: plantuml}
773
633
  end
774
634
  end
775
635
  end
data/lib/solis/shape.rb CHANGED
@@ -69,6 +69,9 @@ module Solis
69
69
  end
70
70
  elsif datatype.nil? && node.is_a?(RDF::URI)
71
71
  node.value.split('/').last.gsub(/Shape$/, '').to_sym
72
+ # normalize ex."https://data.q.odis.be/person#Name" to Name
73
+ #node.value.split('/').last.gsub(/Shape$/, '').split('#').last.to_sym
74
+ #node.value.split('/').last.gsub(/Shape$/, '').gsub('#','').camelize.to_sym
72
75
  elsif datatype =~ /^http:\/\/www.w3.org\/1999\/02\/22-rdf-syntax-ns/
73
76
  case datatype
74
77
  when /http:\/\/www.w3.org\/1999\/02\/22-rdf-syntax-ns#langString/
data/lib/solis/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Solis
2
- VERSION = "0.69.0"
2
+ VERSION = "0.71.0"
3
3
  end
data/solis.gemspec CHANGED
@@ -27,22 +27,22 @@ Gem::Specification.new do |spec|
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ['lib']
29
29
 
30
- spec.add_runtime_dependency 'activesupport', '~> 6.1'
30
+ spec.add_runtime_dependency 'activesupport', '~> 7.0'
31
31
  spec.add_runtime_dependency 'http', '~> 5.1'
32
32
  spec.add_runtime_dependency 'graphiti', '~> 1.3'
33
- spec.add_runtime_dependency 'moneta', '~> 1.4'
34
- spec.add_runtime_dependency 'linkeddata', '~> 3.2'
33
+ spec.add_runtime_dependency 'moneta', '~> 1.6'
34
+ spec.add_runtime_dependency 'linkeddata', '~> 3.3'
35
35
  spec.add_runtime_dependency 'google_drive', '~> 3.0'
36
- spec.add_runtime_dependency 'json', '~> 2.5'
36
+ spec.add_runtime_dependency 'json', '~> 2.6'
37
37
  spec.add_runtime_dependency 'hashdiff', '~> 1.0'
38
38
  spec.add_runtime_dependency 'iso8601', '~> 0.13.0'
39
- spec.add_runtime_dependency 'connection_pool', '~> 2.2.5'
39
+ spec.add_runtime_dependency 'connection_pool', '~> 2.4'
40
40
  spec.add_runtime_dependency 'uuidtools', '~> 2.2.0'
41
- spec.add_runtime_dependency 'dry-struct', '~> 1.2'
42
- spec.add_runtime_dependency 'psych', '< 4'
41
+ spec.add_runtime_dependency 'dry-struct', '~> 1.6'
42
+ spec.add_runtime_dependency 'psych', '~> 5.1'
43
43
 
44
44
  spec.add_development_dependency 'rake', '~> 13.0'
45
- spec.add_development_dependency 'minitest', '~> 5.15.0'
45
+ spec.add_development_dependency 'minitest', '~> 5.19'
46
46
 
47
47
  # spec.add_development_dependency 'rubocop'
48
48
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: solis
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.69.0
4
+ version: 0.71.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mehmet Celik
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-11-21 00:00:00.000000000 Z
11
+ date: 2023-09-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '6.1'
19
+ version: '7.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '6.1'
26
+ version: '7.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: http
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -58,28 +58,28 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '1.4'
61
+ version: '1.6'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '1.4'
68
+ version: '1.6'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: linkeddata
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '3.2'
75
+ version: '3.3'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '3.2'
82
+ version: '3.3'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: google_drive
85
85
  requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: '2.5'
103
+ version: '2.6'
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: '2.5'
110
+ version: '2.6'
111
111
  - !ruby/object:Gem::Dependency
112
112
  name: hashdiff
113
113
  requirement: !ruby/object:Gem::Requirement
@@ -142,14 +142,14 @@ dependencies:
142
142
  requirements:
143
143
  - - "~>"
144
144
  - !ruby/object:Gem::Version
145
- version: 2.2.5
145
+ version: '2.4'
146
146
  type: :runtime
147
147
  prerelease: false
148
148
  version_requirements: !ruby/object:Gem::Requirement
149
149
  requirements:
150
150
  - - "~>"
151
151
  - !ruby/object:Gem::Version
152
- version: 2.2.5
152
+ version: '2.4'
153
153
  - !ruby/object:Gem::Dependency
154
154
  name: uuidtools
155
155
  requirement: !ruby/object:Gem::Requirement
@@ -170,28 +170,28 @@ dependencies:
170
170
  requirements:
171
171
  - - "~>"
172
172
  - !ruby/object:Gem::Version
173
- version: '1.2'
173
+ version: '1.6'
174
174
  type: :runtime
175
175
  prerelease: false
176
176
  version_requirements: !ruby/object:Gem::Requirement
177
177
  requirements:
178
178
  - - "~>"
179
179
  - !ruby/object:Gem::Version
180
- version: '1.2'
180
+ version: '1.6'
181
181
  - !ruby/object:Gem::Dependency
182
182
  name: psych
183
183
  requirement: !ruby/object:Gem::Requirement
184
184
  requirements:
185
- - - "<"
185
+ - - "~>"
186
186
  - !ruby/object:Gem::Version
187
- version: '4'
187
+ version: '5.1'
188
188
  type: :runtime
189
189
  prerelease: false
190
190
  version_requirements: !ruby/object:Gem::Requirement
191
191
  requirements:
192
- - - "<"
192
+ - - "~>"
193
193
  - !ruby/object:Gem::Version
194
- version: '4'
194
+ version: '5.1'
195
195
  - !ruby/object:Gem::Dependency
196
196
  name: rake
197
197
  requirement: !ruby/object:Gem::Requirement
@@ -212,14 +212,14 @@ dependencies:
212
212
  requirements:
213
213
  - - "~>"
214
214
  - !ruby/object:Gem::Version
215
- version: 5.15.0
215
+ version: '5.19'
216
216
  type: :development
217
217
  prerelease: false
218
218
  version_requirements: !ruby/object:Gem::Requirement
219
219
  requirements:
220
220
  - - "~>"
221
221
  - !ruby/object:Gem::Version
222
- version: 5.15.0
222
+ version: '5.19'
223
223
  description: The SUN in latin or is it SILOS spelled backwards. Turn any SHACL file
224
224
  or Google sheet into an API, ORM, documentation on top of a data store
225
225
  email:
@@ -295,7 +295,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
295
295
  - !ruby/object:Gem::Version
296
296
  version: '0'
297
297
  requirements: []
298
- rubygems_version: 3.1.6
298
+ rubygems_version: 3.4.19
299
299
  signing_key:
300
300
  specification_version: 4
301
301
  summary: Turn any SHACL file into an API, ORM, documentation, ...