solis 0.126.0 → 0.127.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce5b23eb71b6b7189377d4981fa8e6dc076128d60aec207e58310fbd162b8a16
-  data.tar.gz: 57c514df94596d89d76ea012cc946b2b521869f585f1d495609f8e0ec5cbdf71
+  metadata.gz: d7cf89eceec897d7b5fe15ef1669b031480a7aab2850bd5638cfac379e4798ae
+  data.tar.gz: 56a9ce9652925e1be2904dc150bba6d31f3f2ca13870641c883fb0dde65b341c
 SHA512:
-  metadata.gz: 912773a18a5b44da59e1bbb6b7e3472c5d8be9a8044df0b64acbb7da501a3156ead5615363bd23fbd595d42db1b862f3d776235758d92acf8d0942f42f2fde00
-  data.tar.gz: 90c434c26f367159078555cf9bfbc715d4179ad8f3ebffba12ba379000e7696bf25020cd9a033d90b1b1e95375574bd4600eb3492d67e703a07aaff901d6b251
+  metadata.gz: 499ff31724baaa76c0ae960916d5b01c0f25f535e07e89fd0cba72cbc042fe271898b5c422842201463daef4903d1207bf4fdcbd248e3bbff346153f4dba1765
+  data.tar.gz: 264f64d77cf8b15d708da87cbf0602a6f8be5282c04a365f52966e865895032e9c08b4952e99abbcb6e5f4aa80fcbd20887e3e4d9c961398e350a4cbaff17b0b

data/lib/solis/language_tag.rb

@@ -0,0 +1,22 @@
+module Solis
+  # Normalizes language tags so the same language always ends up under the same key,
+  # no matter if it came from a 'Label_XX' sheet column or from a parsed rdfs:label
+  # (RDF.rb lowercases language tags, ex. 'nl-BE' is read back as :'nl-be')
+  module LanguageTag
+    # 'nl' -> :nl, 'nl_be' -> :'nl-BE', 'zh_hant' -> :'zh-Hant'
+    def self.normalize(tag)
+      return nil if tag.nil? || tag.to_s.strip.empty?
+
+      primary, *subtags = tag.to_s.strip.split(/[-_]/)
+      subtags.map! do |subtag|
+        case subtag.length
+        when 2 then subtag.upcase
+        when 4 then subtag.capitalize
+        else subtag.downcase
+        end
+      end
+
+      ([primary.downcase] + subtags).join('-').to_sym
+    end
+  end
+end

data/lib/solis/model.rb

@@ -583,6 +583,7 @@ values ?s {<#{self.graph_id}>}
         end
 
         attribute_data = { name: attribute,
+                           label: attribute_metadata[:label] || {},
                            data_type: attribute_metadata[:datatype],
                            mandatory: (attribute_metadata[:mincount].to_i > 0),
                            repeatable: (attribute_metadata[:maxcount].to_i > 1 || attribute_metadata[:maxcount].nil?),

data/lib/solis/shape/reader/sheet.rb

@@ -43,6 +43,11 @@ module Solis
                 raise "_REFERENCES tab must have ['sheeturl', 'description', 'entityrange'] as a header at row 1" unless (%w[sheeturl description entityrange] - references.header).length == 0
               end
 
+              if sheets.key?('_CATEGORIES')
+                categories = sheets['_CATEGORIES']
+                raise "_CATEGORIES tab must have ['name'] as a header at row 1" unless (%w[name] - categories.header).length == 0
+              end
+
               sheets.each do |sheet_name, sheet|
                 if sheet_name !~ /^_/
                   entities = sheets[sheet_name]
@@ -113,6 +118,7 @@ module Solis
                                                    category: e['categories'],
                                                    plural: e['nameplural'],
                                                    label: e['name'].to_s.strip,
+                                                   labels: parse_labels(e),
                                                    sub_class_of: e['subclassof'].nil? || e['subclassof'].empty? ? [] : [e['subclassof']],
                                                    same_as: e['sameas'],
                                                    properties: entity_data })
@@ -135,6 +141,26 @@ module Solis
               raise Solis::Error::GeneralError, e.message
             end
 
+            # Matches the 'Label_XX' columns (header is downcased by ::Sheet#header) where
+            # XX is a language tag, ex. 'Label_NL', 'Label_EN', 'Label_nl-BE'
+            LANGUAGE_LABEL_COLUMN = /\Alabel_(?<language>[a-z]{2,3}([-_][a-z0-9]{2,8})*)\z/.freeze
+
+            # Collects all 'Label_XX' columns of a row into { nl: 'Publiceren', en: 'Publish' }
+            def parse_labels(row)
+              labels = {}
+              return labels unless row.respond_to?(:each_pair)
+
+              row.each_pair do |column, value|
+                match = LANGUAGE_LABEL_COLUMN.match(column.to_s)
+                next if match.nil?
+                next if value.nil? || value.to_s.strip.empty?
+
+                labels[Solis::LanguageTag.normalize(match[:language])] = value.to_s.strip
+              end
+
+              labels
+            end
+
             def parse_entity_data(entity_name, graph_prefix, _graph_name, e, options = {})
               properties = {}
               entity_data = e
@@ -168,31 +194,9 @@ module Solis
                       same_as: p['sameas'],
                       order: p['order'],
                       category: p['categories'],
+                      label: parse_labels(p),
                       description: p['description']
                     }
-
-                    # unless graph_prefix.eql?(datatype_prefix.to_sym)
-                    #   prefixes = options[:prefixes]
-                    #   if prefixes.key?(datatype_prefix.to_sym) && !prefixes[datatype_prefix.to_sym][:sheet_url].empty?
-                    #     tmp = URI(prefixes[datatype_prefix.to_sym][:sheet_url]).path.split('/')
-                    #     spreadsheet_id = tmp[tmp.index('d') + 1]
-                    #
-                    #     processed_remote_sheet = {}
-                    #     if prefixes[datatype_prefix.to_sym].key?(:data) && !prefixes[datatype_prefix.to_sym][:data].empty?
-                    #       processed_remote_sheet = prefixes[datatype_prefix.to_sym][:data]
-                    #     else
-                    #       if options[:follow]
-                    #         sleep 30
-                    #         remote_sheet = read_sheets(options[:key], spreadsheet_id, { from_cache: true })
-                    #         processed_remote_sheet = process_sheet(options[:key], remote_sheet, {follow: false})
-                    #         prefixes[datatype_prefix.to_sym][:data] = processed_remote_sheet
-                    #       end
-                    #     end
-                    #
-                    #     processed_remote_sheet
-                    #   end
-                    # end
-
                   end
                 end
               end
@@ -266,12 +270,32 @@ hide empty members
               datatypes[datatype][as]
             end
 
+            # Turns { nl: 'Publiceren', en: 'Publish' } into a single rdfs:label statement with
+            # one language tagged literal per language, ex.
+            #   rdfs:label "Publiceren"@nl, "Publish"@en ;
+            def build_labels(labels, rdfs_prefix, indent)
+              return '' if labels.nil? || labels.empty?
+
+              literals = labels.map do |language, value|
+                %("#{value.to_s.gsub('"', "'").gsub(/\n|\r/, '')}"@#{language})
+              end
+
+              "\n#{indent}#{rdfs_prefix}:label #{literals.join(', ')} ;"
+            end
+
             def build_shacl(datas)
               shacl_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /shacl/ }.keys.first
               shacl_prefix = 'sh' if shacl_prefix.nil?
 
+              rdfs_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /rdf-schema/ }.keys.first
+
               out = header(datas.first)
 
+              if rdfs_prefix.nil?
+                rdfs_prefix = 'rdfs'
+                out += "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n"
+              end
+
               datas.each do |data|
                 data[:entities].each do |entity_name, metadata|
                   graph_prefix = data[:ontologies][:base][:prefix]
@@ -300,7 +324,7 @@ hide empty members
                     unless node.nil? || node.empty?
                       "\n    #{shacl_prefix}:node         #{node} ;"
                     end}
-    #{shacl_prefix}:name         "#{label}" ;
+    #{shacl_prefix}:name         "#{label}" ;#{build_labels(metadata[:labels], rdfs_prefix, ' ' * 4)}
 )
                   metadata[:properties].each do |property, property_metadata|
                     attribute = property.to_s.strip
@@ -314,15 +338,16 @@ hide empty members
                     order = property_metadata.key?(:order) && property_metadata[:order] ? property_metadata[:order]&.strip : nil
                     category = property_metadata.key?(:category) && property_metadata[:category] ? property_metadata[:category]&.strip : nil
                     category = category.nil? || category.empty? ? nil : category
+                    labels = build_labels(property_metadata[:label], rdfs_prefix, ' ' * 17)
 
                     unless category.nil?
-                      group = "#{category.classify}Group"
+                      group = "#{category.camelize}Group"
                       groups[group] = category unless groups.key?(group)
                     end
 
                     if datatype =~ /^#{graph_prefix}:/ || datatype =~ /^<#{graph_name}/
                       out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}" ;
+                 #{shacl_prefix}:name "#{attribute}" ;#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:nodeKind #{shacl_prefix}:IRI ;
@@ -332,7 +357,7 @@ hide empty members
                     else
                       if datatype.eql?('rdf:langString') && max_count.eql?('1')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:uniqueLang true ;
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}
@@ -340,7 +365,7 @@ hide empty members
 )
                       elsif datatype.eql?('rdf:langString')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
@@ -348,7 +373,7 @@ hide empty members
 )
                       else
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
     ] ;
@@ -362,8 +387,8 @@ hide empty members
                   groups.each do |group, category|
                     out += %(
 #{graph_prefix}:#{group}
-	a sh:PropertyGroup ;
-	rdfs:label "#{category}" .
+	a #{shacl_prefix}:PropertyGroup ;
+	#{rdfs_prefix}:label "#{category}" .
 
 )
                   end
@@ -489,7 +514,7 @@ hide empty members
               puts e.message
             end
 
-            def build_inflections(datas)
+            def build_inflections(datas, categories = [])
               inflections = {}
               datas.each do |data|
                 data[:entities].each do |entity, metadata|
@@ -498,6 +523,16 @@ hide empty members
                 end
               end
 
+              # categories are uninflected, singular and plural are the same word.
+              # an entity by the same name keeps its own plural.
+              # categories.each do |category|
+              #   name = category.to_s.strip
+              #   next if name.empty?
+              #
+              #   inflections[name.to_sym] = name unless inflections.key?(name.to_sym) || inclections.value?(name.to_s)
+              #   inflections[name.underscore.to_sym] = name.underscore unless inflections.key?(name.underscore.to_sym) || inclections.value?(name.to_s)
+              # end
+
               inflections.to_json
             rescue StandardError => e
               puts e.message
@@ -743,6 +778,11 @@ hide empty members
 
           options[:prefixes] = prefixes
           options[:metadata] = metadata
+
+          # like _REFERENCES, _CATEGORIES only lives in the main sheet
+          categories = sheet_data.is_a?(Hash) && sheet_data.key?('_CATEGORIES') ? sheet_data['_CATEGORIES'].map { |c| c['name'] } : []
+          Solis::LOGGER.info("Found #{categories.length} categories") unless categories.empty?
+
           #TODO: cleanup
           if sheet_data.is_a?(Hash)
             raise "No _REFERENCES sheet found" unless sheet_data.key?("_REFERENCES")
@@ -786,7 +826,7 @@ hide empty members
           Solis::LOGGER.info('Generating SCHEMA')
           schema = build_schema(datas)
           Solis::LOGGER.info('Generating INFLECTIONS')
-          inflections = build_inflections(datas)
+          inflections = build_inflections(datas, categories)
           Solis::LOGGER.info('Generating JSON SCHEMA')
           json_schema = build_json_schema(shacl)
 

data/lib/solis/shape.rb

@@ -14,6 +14,8 @@ module Solis
             parse_solution(shapes, solution)
           end
 
+          add_labels(shapes, graph)
+
           # shapes = add_missing_attributes(shapes)
           shapes
         rescue Solis::Error::GeneralError => e
@@ -137,6 +139,47 @@ module Solis
           shapes[shape_name] = shape
         end
 
+        # Reads the rdfs:label statements of every node shape and property shape into
+        # { nl: 'Publiceren', en: 'Publish' }. This is a second pass on purpose: adding
+        # rdfs:label to the main query would multiply every solution by the number of
+        # languages.
+        def add_labels(shapes, graph)
+          graph.query([nil, RDF.type, RDF::Vocab::SHACL.NodeShape]) do |node_shape|
+            shape_name = name_of(graph, node_shape.subject)
+            shape = shapes[shape_name]
+            next if shape.nil?
+
+            shape[:label] = labels_of(graph, node_shape.subject)
+
+            graph.query([node_shape.subject, RDF::Vocab::SHACL.property, nil]) do |property|
+              attribute = shape[:attributes][name_of(graph, property.object)]
+              next if attribute.nil?
+
+              attribute[:label] = labels_of(graph, property.object)
+            end
+          end
+
+          shapes
+        end
+
+        def name_of(graph, subject)
+          graph.query([subject, RDF::Vocab::SHACL.name, nil]).first&.object&.value
+        end
+
+        def labels_of(graph, subject)
+          labels = {}
+
+          graph.query([subject, RDF::RDFS.label, nil]).each do |statement|
+            literal = statement.object
+            language = Solis::LanguageTag.normalize(literal.language) if literal.is_a?(RDF::Literal)
+            next if language.nil?
+
+            labels[language] = literal.value
+          end
+
+          labels
+        end
+
         def query
           SPARQL.parse %(
 PREFIX sh: <http://www.w3.org/ns/shacl#>

data/lib/solis/version.rb

@@ -1,3 +1,3 @@
 module Solis
-  VERSION = "0.126.0"
+  VERSION = "0.127.0"
 end

data/lib/solis.rb

@@ -7,6 +7,7 @@ require "solis/version"
 require 'solis/config_file'
 require "solis/error"
 require 'solis/rdf_edtf_literal'
+require 'solis/language_tag'
 require 'solis/graph'
 require 'solis/shape'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solis
 version: !ruby/object:Gem::Version
-  version: 0.126.0
+  version: 0.127.0
 platform: ruby
 authors:
 - Mehmet Celik
@@ -307,6 +307,7 @@ files:
 - lib/solis/error/not_found_error.rb
 - lib/solis/error/query_error.rb
 - lib/solis/graph.rb
+- lib/solis/language_tag.rb
 - lib/solis/model.rb
 - lib/solis/model.rb.ok
 - lib/solis/options.rb