solis 0.126.0 → 0.128.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce5b23eb71b6b7189377d4981fa8e6dc076128d60aec207e58310fbd162b8a16
-  data.tar.gz: 57c514df94596d89d76ea012cc946b2b521869f585f1d495609f8e0ec5cbdf71
+  metadata.gz: 8200228367f2710bad52af70fd0430ca70ec589fe284e19e22bdfe4e1a659a98
+  data.tar.gz: 337d425fa58afb14c95cfd3703d848a9093e563a335f1d37b08d420981389761
 SHA512:
-  metadata.gz: 912773a18a5b44da59e1bbb6b7e3472c5d8be9a8044df0b64acbb7da501a3156ead5615363bd23fbd595d42db1b862f3d776235758d92acf8d0942f42f2fde00
-  data.tar.gz: 90c434c26f367159078555cf9bfbc715d4179ad8f3ebffba12ba379000e7696bf25020cd9a033d90b1b1e95375574bd4600eb3492d67e703a07aaff901d6b251
+  metadata.gz: 64cae66429794837abf0e51b72e43847337c195a69b9f6ca5cbffa22bf6888dc472592d7c186d31ca9a7b4eca826d3ecdf1a4c1b0853d047d778cc7931f4ab47
+  data.tar.gz: 449810be62118c3def433b580a8e794f3b19ef84413d8b14ca1fbda72c95986ebe8787ddb5a019cb9d63c2dc73f756983b0fc6249977241585c490217a494a12

data/lib/solis/language_tag.rb

@@ -0,0 +1,22 @@
+module Solis
+  # Normalizes language tags so the same language always ends up under the same key,
+  # no matter if it came from a 'Label_XX' sheet column or from a parsed rdfs:label
+  # (RDF.rb lowercases language tags, ex. 'nl-BE' is read back as :'nl-be')
+  module LanguageTag
+    # 'nl' -> :nl, 'nl_be' -> :'nl-BE', 'zh_hant' -> :'zh-Hant'
+    def self.normalize(tag)
+      return nil if tag.nil? || tag.to_s.strip.empty?
+
+      primary, *subtags = tag.to_s.strip.split(/[-_]/)
+      subtags.map! do |subtag|
+        case subtag.length
+        when 2 then subtag.upcase
+        when 4 then subtag.capitalize
+        else subtag.downcase
+        end
+      end
+
+      ([primary.downcase] + subtags).join('-').to_sym
+    end
+  end
+end

data/lib/solis/model.rb

@@ -583,13 +583,17 @@ values ?s {<#{self.graph_id}>}
         end
 
         attribute_data = { name: attribute,
+                           label: attribute_metadata[:label] || {},
                            data_type: attribute_metadata[:datatype],
                            mandatory: (attribute_metadata[:mincount].to_i > 0),
                            repeatable: (attribute_metadata[:maxcount].to_i > 1 || attribute_metadata[:maxcount].nil?),
                            description: attribute_metadata[:comment]&.value
         }
         attribute_data[:order] = attribute_metadata[:order]&.value.to_i if attribute_metadata.key?(:order) && !attribute_metadata[:order].nil?
-        attribute_data[:group] = attribute_metadata[:group]&.value.gsub(graph_name,'').gsub(/Group$/,'') if attribute_metadata.key?(:group) && !attribute_metadata[:group].nil?
+        if attribute_metadata.key?(:group) && !attribute_metadata[:group].nil?
+          attribute_data[:group] = { name: attribute_metadata[:group][:name],
+                                     label: attribute_metadata[:group][:label] || {} }
+        end
         attribute_data[:attributes] = cm[:attributes] if cm && cm[:attributes]
 
         m[:attributes] << attribute_data

data/lib/solis/shape/reader/sheet.rb

@@ -43,6 +43,12 @@ module Solis
                 raise "_REFERENCES tab must have ['sheeturl', 'description', 'entityrange'] as a header at row 1" unless (%w[sheeturl description entityrange] - references.header).length == 0
               end
 
+              if sheets.key?('_CATEGORIES')
+                categories = sheets['_CATEGORIES']
+                has_name = categories.header.include?('name') || categories.header.any? { |h| language_column('name').match?(h.to_s) }
+                raise "_CATEGORIES tab must have ['name'] or ['name_XX', ...] as a header at row 1" unless has_name
+              end
+
               sheets.each do |sheet_name, sheet|
                 if sheet_name !~ /^_/
                   entities = sheets[sheet_name]
@@ -113,6 +119,7 @@ module Solis
                                                    category: e['categories'],
                                                    plural: e['nameplural'],
                                                    label: e['name'].to_s.strip,
+                                                   labels: parse_labels(e),
                                                    sub_class_of: e['subclassof'].nil? || e['subclassof'].empty? ? [] : [e['subclassof']],
                                                    same_as: e['sameas'],
                                                    properties: entity_data })
@@ -135,6 +142,29 @@ module Solis
               raise Solis::Error::GeneralError, e.message
             end
 
+            # Matches the '<prefix>_XX' columns (header is downcased by ::Sheet#header) where
+            # XX is a language tag, ex. 'Label_NL', 'Label_EN', 'Label_nl-BE', 'Name_nl'
+            def language_column(prefix)
+              /\A#{Regexp.escape(prefix.to_s.downcase)}_(?<language>[a-z]{2,3}([-_][a-z0-9]{2,8})*)\z/
+            end
+
+            # Collects all 'Label_XX' (or '<prefix>_XX') columns of a row into { nl: 'Publiceren', en: 'Publish' }
+            def parse_labels(row, prefix = 'label')
+              labels = {}
+              return labels unless row.respond_to?(:each_pair)
+
+              column_pattern = language_column(prefix)
+              row.each_pair do |column, value|
+                match = column_pattern.match(column.to_s)
+                next if match.nil?
+                next if value.nil? || value.to_s.strip.empty?
+
+                labels[Solis::LanguageTag.normalize(match[:language])] = value.to_s.strip
+              end
+
+              labels
+            end
+
             def parse_entity_data(entity_name, graph_prefix, _graph_name, e, options = {})
               properties = {}
               entity_data = e
@@ -168,31 +198,9 @@ module Solis
                       same_as: p['sameas'],
                       order: p['order'],
                       category: p['categories'],
+                      label: parse_labels(p),
                       description: p['description']
                     }
-
-                    # unless graph_prefix.eql?(datatype_prefix.to_sym)
-                    #   prefixes = options[:prefixes]
-                    #   if prefixes.key?(datatype_prefix.to_sym) && !prefixes[datatype_prefix.to_sym][:sheet_url].empty?
-                    #     tmp = URI(prefixes[datatype_prefix.to_sym][:sheet_url]).path.split('/')
-                    #     spreadsheet_id = tmp[tmp.index('d') + 1]
-                    #
-                    #     processed_remote_sheet = {}
-                    #     if prefixes[datatype_prefix.to_sym].key?(:data) && !prefixes[datatype_prefix.to_sym][:data].empty?
-                    #       processed_remote_sheet = prefixes[datatype_prefix.to_sym][:data]
-                    #     else
-                    #       if options[:follow]
-                    #         sleep 30
-                    #         remote_sheet = read_sheets(options[:key], spreadsheet_id, { from_cache: true })
-                    #         processed_remote_sheet = process_sheet(options[:key], remote_sheet, {follow: false})
-                    #         prefixes[datatype_prefix.to_sym][:data] = processed_remote_sheet
-                    #       end
-                    #     end
-                    #
-                    #     processed_remote_sheet
-                    #   end
-                    # end
-
                   end
                 end
               end
@@ -266,12 +274,32 @@ hide empty members
               datatypes[datatype][as]
             end
 
-            def build_shacl(datas)
+            # Turns { nl: 'Publiceren', en: 'Publish' } into a single rdfs:label statement with
+            # one language tagged literal per language, ex.
+            #   rdfs:label "Publiceren"@nl, "Publish"@en ;
+            def build_labels(labels, rdfs_prefix, indent)
+              return '' if labels.nil? || labels.empty?
+
+              literals = labels.map do |language, value|
+                %("#{value.to_s.gsub('"', "'").gsub(/\n|\r/, '')}"@#{language})
+              end
+
+              "\n#{indent}#{rdfs_prefix}:label #{literals.join(', ')} ;"
+            end
+
+            def build_shacl(datas, categories = [])
               shacl_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /shacl/ }.keys.first
               shacl_prefix = 'sh' if shacl_prefix.nil?
 
+              rdfs_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /rdf-schema/ }.keys.first
+
               out = header(datas.first)
 
+              if rdfs_prefix.nil?
+                rdfs_prefix = 'rdfs'
+                out += "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n"
+              end
+
               datas.each do |data|
                 data[:entities].each do |entity_name, metadata|
                   graph_prefix = data[:ontologies][:base][:prefix]
@@ -300,7 +328,7 @@ hide empty members
                     unless node.nil? || node.empty?
                       "\n    #{shacl_prefix}:node         #{node} ;"
                     end}
-    #{shacl_prefix}:name         "#{label}" ;
+    #{shacl_prefix}:name         "#{label}" ;#{build_labels(metadata[:labels], rdfs_prefix, ' ' * 4)}
 )
                   metadata[:properties].each do |property, property_metadata|
                     attribute = property.to_s.strip
@@ -314,15 +342,16 @@ hide empty members
                     order = property_metadata.key?(:order) && property_metadata[:order] ? property_metadata[:order]&.strip : nil
                     category = property_metadata.key?(:category) && property_metadata[:category] ? property_metadata[:category]&.strip : nil
                     category = category.nil? || category.empty? ? nil : category
+                    labels = build_labels(property_metadata[:label], rdfs_prefix, ' ' * 17)
 
                     unless category.nil?
-                      group = "#{category.classify}Group"
+                      group = "#{category.camelize}Group"
                       groups[group] = category unless groups.key?(group)
                     end
 
                     if datatype =~ /^#{graph_prefix}:/ || datatype =~ /^<#{graph_name}/
                       out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}" ;
+                 #{shacl_prefix}:name "#{attribute}" ;#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:nodeKind #{shacl_prefix}:IRI ;
@@ -332,7 +361,7 @@ hide empty members
                     else
                       if datatype.eql?('rdf:langString') && max_count.eql?('1')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:uniqueLang true ;
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}
@@ -340,7 +369,7 @@ hide empty members
 )
                       elsif datatype.eql?('rdf:langString')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
@@ -348,7 +377,7 @@ hide empty members
 )
                       else
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
     ] ;
@@ -360,10 +389,17 @@ hide empty members
                   out += ".\n"
 
                   groups.each do |group, category|
+                    # use the per language names from _CATEGORIES when the category is referenced by any of them
+                    category_labels = categories.find { |c| c[:name].eql?(category) || c[:labels].value?(category) }&.dig(:labels)
+                    group_label = if category_labels.nil? || category_labels.empty?
+                                    %(#{rdfs_prefix}:label "#{category}")
+                                  else
+                                    build_labels(category_labels, rdfs_prefix, '').strip.chomp(' ;')
+                                  end
                     out += %(
 #{graph_prefix}:#{group}
-	a sh:PropertyGroup ;
-	rdfs:label "#{category}" .
+	a #{shacl_prefix}:PropertyGroup ;
+	#{group_label} .
 
 )
                   end
@@ -489,7 +525,7 @@ hide empty members
               puts e.message
             end
 
-            def build_inflections(datas)
+            def build_inflections(datas, categories = [])
               inflections = {}
               datas.each do |data|
                 data[:entities].each do |entity, metadata|
@@ -498,6 +534,16 @@ hide empty members
                 end
               end
 
+              # categories are uninflected, singular and plural are the same word.
+              # an entity by the same name keeps its own plural.
+              # categories.each do |category|
+              #   name = category[:name].to_s.strip
+              #   next if name.empty?
+              #
+              #   inflections[name.to_sym] = name unless inflections.key?(name.to_sym) || inclections.value?(name.to_s)
+              #   inflections[name.underscore.to_sym] = name.underscore unless inflections.key?(name.underscore.to_sym) || inclections.value?(name.to_s)
+              # end
+
               inflections.to_json
             rescue StandardError => e
               puts e.message
@@ -743,6 +789,23 @@ hide empty members
 
           options[:prefixes] = prefixes
           options[:metadata] = metadata
+
+          # like _REFERENCES, _CATEGORIES only lives in the main sheet
+          # a category can have a name per language ('Name_nl', 'Name_en', ...) next to, or instead of, 'Name'.
+          # entity sheets refer to a category by 'Name' or, when missing, by the first 'Name_XX' column.
+          categories = []
+          if sheet_data.is_a?(Hash) && sheet_data.key?('_CATEGORIES')
+            categories = sheet_data['_CATEGORIES'].map do |c|
+              labels = parse_labels(c, 'name')
+              name = c['name'].to_s.strip
+              name = labels.values.first.to_s if name.empty?
+              next if name.empty?
+
+              { name: name, labels: labels }
+            end.compact
+          end
+          Solis::LOGGER.info("Found #{categories.length} categories") unless categories.empty?
+
           #TODO: cleanup
           if sheet_data.is_a?(Hash)
             raise "No _REFERENCES sheet found" unless sheet_data.key?("_REFERENCES")
@@ -780,13 +843,13 @@ hide empty members
           end
 
           Solis::LOGGER.info('Generating SHACL')
-          shacl = build_shacl(datas)
+          shacl = build_shacl(datas, categories)
           Solis::LOGGER.info('Generating PLANTUML')
           plantuml = build_plantuml(datas)
           Solis::LOGGER.info('Generating SCHEMA')
           schema = build_schema(datas)
           Solis::LOGGER.info('Generating INFLECTIONS')
-          inflections = build_inflections(datas)
+          inflections = build_inflections(datas, categories)
           Solis::LOGGER.info('Generating JSON SCHEMA')
           json_schema = build_json_schema(shacl)
 

data/lib/solis/shape.rb

@@ -14,6 +14,8 @@ module Solis
             parse_solution(shapes, solution)
           end
 
+          add_labels(shapes, graph)
+
           # shapes = add_missing_attributes(shapes)
           shapes
         rescue Solis::Error::GeneralError => e
@@ -137,6 +139,60 @@ module Solis
           shapes[shape_name] = shape
         end
 
+        # Reads the rdfs:label statements of every node shape and property shape into
+        # { nl: 'Publiceren', en: 'Publish' }. This is a second pass on purpose: adding
+        # rdfs:label to the main query would multiply every solution by the number of
+        # languages.
+        def add_labels(shapes, graph)
+          graph.query([nil, RDF.type, RDF::Vocab::SHACL.NodeShape]) do |node_shape|
+            shape_name = name_of(graph, node_shape.subject)
+            shape = shapes[shape_name]
+            next if shape.nil?
+
+            shape[:label] = labels_of(graph, node_shape.subject)
+
+            graph.query([node_shape.subject, RDF::Vocab::SHACL.property, nil]) do |property|
+              attribute = shape[:attributes][name_of(graph, property.object)]
+              next if attribute.nil?
+
+              attribute[:label] = labels_of(graph, property.object)
+              attribute[:group] = group_of(graph, attribute[:group]) unless attribute[:group].nil?
+            end
+          end
+
+          shapes
+        end
+
+        # Turns a sh:PropertyGroup URI into
+        #   { uri: RDF::URI, name: 'OpleidingEnLoopbaan', label: { nl: 'opleiding_en_loopbaan', en: 'education_and_career' } }
+        def group_of(graph, group)
+          return group if group.is_a?(Hash)
+
+          {
+            uri: group,
+            name: group.to_s.split(%r{[/#]}).last.to_s.gsub(/Group$/, ''),
+            label: labels_of(graph, group)
+          }
+        end
+
+        def name_of(graph, subject)
+          graph.query([subject, RDF::Vocab::SHACL.name, nil]).first&.object&.value
+        end
+
+        def labels_of(graph, subject)
+          labels = {}
+
+          graph.query([subject, RDF::RDFS.label, nil]).each do |statement|
+            literal = statement.object
+            language = Solis::LanguageTag.normalize(literal.language) if literal.is_a?(RDF::Literal)
+            next if language.nil?
+
+            labels[language] = literal.value
+          end
+
+          labels
+        end
+
         def query
           SPARQL.parse %(
 PREFIX sh: <http://www.w3.org/ns/shacl#>

data/lib/solis/version.rb

@@ -1,3 +1,3 @@
 module Solis
-  VERSION = "0.126.0"
+  VERSION = "0.128.0"
 end

data/lib/solis.rb

@@ -7,6 +7,7 @@ require "solis/version"
 require 'solis/config_file'
 require "solis/error"
 require 'solis/rdf_edtf_literal'
+require 'solis/language_tag'
 require 'solis/graph'
 require 'solis/shape'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solis
 version: !ruby/object:Gem::Version
-  version: 0.126.0
+  version: 0.128.0
 platform: ruby
 authors:
 - Mehmet Celik
@@ -307,6 +307,7 @@ files:
 - lib/solis/error/not_found_error.rb
 - lib/solis/error/query_error.rb
 - lib/solis/graph.rb
+- lib/solis/language_tag.rb
 - lib/solis/model.rb
 - lib/solis/model.rb.ok
 - lib/solis/options.rb