solis 0.125.0 → 0.127.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c29d1cb5fcfe9ad5579f22c3971aa794edc602b6afa3679a2113e0caecd01aa4
-  data.tar.gz: e067c8d0ad39a30dd49c921f5beded800d0f93de83817911c6fafd32486b89dd
+  metadata.gz: d7cf89eceec897d7b5fe15ef1669b031480a7aab2850bd5638cfac379e4798ae
+  data.tar.gz: 56a9ce9652925e1be2904dc150bba6d31f3f2ca13870641c883fb0dde65b341c
 SHA512:
-  metadata.gz: 939e20fe0909c981ee02d29766b5ac43446db72d94df6a49ca68208a4a7c8d969a543c859b9f8c9e18bb843f5dd34e186356cdd245aa0abb8120b3bee0a7e4ad
-  data.tar.gz: 5fb1ae2ea0fd011eb0aab4d306252a27f9f238a0b908fe458895bd1d238974da59ee57d4520dc14a34e49e2fe2fa60bb6b4d8366f3cd4b9fed4c63ce15594527
+  metadata.gz: 499ff31724baaa76c0ae960916d5b01c0f25f535e07e89fd0cba72cbc042fe271898b5c422842201463daef4903d1207bf4fdcbd248e3bbff346153f4dba1765
+  data.tar.gz: 264f64d77cf8b15d708da87cbf0602a6f8be5282c04a365f52966e865895032e9c08b4952e99abbcb6e5f4aa80fcbd20887e3e4d9c961398e350a4cbaff17b0b

data/lib/solis/language_tag.rb

@@ -0,0 +1,22 @@
+module Solis
+  # Normalizes language tags so the same language always ends up under the same key,
+  # no matter if it came from a 'Label_XX' sheet column or from a parsed rdfs:label
+  # (RDF.rb lowercases language tags, ex. 'nl-BE' is read back as :'nl-be')
+  module LanguageTag
+    # 'nl' -> :nl, 'nl_be' -> :'nl-BE', 'zh_hant' -> :'zh-Hant'
+    def self.normalize(tag)
+      return nil if tag.nil? || tag.to_s.strip.empty?
+
+      primary, *subtags = tag.to_s.strip.split(/[-_]/)
+      subtags.map! do |subtag|
+        case subtag.length
+        when 2 then subtag.upcase
+        when 4 then subtag.capitalize
+        else subtag.downcase
+        end
+      end
+
+      ([primary.downcase] + subtags).join('-').to_sym
+    end
+  end
+end

data/lib/solis/model.rb

@@ -583,6 +583,7 @@ values ?s {<#{self.graph_id}>}
         end
 
         attribute_data = { name: attribute,
+                           label: attribute_metadata[:label] || {},
                            data_type: attribute_metadata[:datatype],
                            mandatory: (attribute_metadata[:mincount].to_i > 0),
                            repeatable: (attribute_metadata[:maxcount].to_i > 1 || attribute_metadata[:maxcount].nil?),

data/lib/solis/query.rb

@@ -11,6 +11,15 @@ module Solis
     include Enumerable
     include Solis::QueryFilter
 
+    # XSD numeric datatypes must be ordered by value; wrapping them in STR() would sort
+    # them lexically ("100" < "9"). Every other datatype is STR()-wrapped in the outer
+    # ORDER BY (see #sort_key_expression) to dodge a Virtuoso collation bug.
+    XSD_NUMERIC_DATATYPES = %w[
+      integer decimal float double long int short byte
+      nonNegativeInteger nonPositiveInteger negativeInteger positiveInteger
+      unsignedLong unsignedInt unsignedShort unsignedByte
+    ].map { |t| "http://www.w3.org/2001/XMLSchema##{t}" }.freeze
+
     def self.run(entity, query, options = {})
       Solis::Query::Runner.run(entity, query, options)
     end
@@ -96,6 +105,13 @@ module Solis
       @filter = {values: ["VALUES ?type {#{target_class}}"], concepts: ['?concept a ?type .'] }
       @sort = 'ORDER BY ?concept'
       @sort_select = ''
+      # @sort_project carries the sort key(s) out of the inner (paginated) subquery so the
+      # outer query can re-sort on them; @sort_outer is that outer ORDER BY. A subquery's
+      # ORDER BY only decides which rows survive LIMIT/OFFSET — it does NOT propagate
+      # through the enclosing join — so the outer query must sort too. See #sort for the
+      # datatype-aware STR() handling that the outer ORDER BY needs.
+      @sort_project = ''
+      @sort_outer = ''
       @language = Graphiti.context[:object]&.language || Solis::Options.instance.get[:language] || 'en'
       @query_cache = self.class.shared_query_cache
     end
@@ -113,19 +129,31 @@ module Solis
     def sort(params)
       @sort = ''
       @sort_select = ''
+      @sort_project = ''
+      @sort_outer = ''
       if params.key?(:sort)
         i = 0
+        outer = ''
         params[:sort].each do |attribute, direction|
-          path = @model.class.metadata[:attributes][attribute.to_s][:path]
-          @sort_select += "optional {\n" if @model.class.metadata[:attributes][attribute.to_s][:mincount] == 0
+          meta = @model.class.metadata[:attributes][attribute.to_s]
+          path = meta[:path]
+          @sort_select += "optional {\n" if meta[:mincount] == 0
           @sort_select += "?concept <#{path}> ?__#{attribute} . "
-          @sort_select += "}\n" if @model.class.metadata[:attributes][attribute.to_s][:mincount] == 0
+          @sort_select += "}\n" if meta[:mincount] == 0
           @sort += ',' if i.positive?
           @sort += "#{direction.to_s.upcase}(?__#{attribute})"
+          # Carry the sort key out of the subquery so the outer query can re-sort on it.
+          @sort_project += " ?__#{attribute}"
+          outer += ',' if i.positive?
+          outer += "#{direction.to_s.upcase}(#{sort_key_expression(attribute, meta)})"
           i += 1
         end
 
-        @sort = "ORDER BY #{@sort}" if i.positive?
+        if i.positive?
+          @sort = "ORDER BY #{@sort}"
+          # ?s tiebreaker keeps each subject's triples contiguous and ties deterministic.
+          @sort_outer = "ORDER BY #{outer} ?s"
+        end
       end
 
       self
@@ -219,6 +247,11 @@ module Solis
       # object when the open `?s ?p ?o` pattern is combined with the disjunctive
       # language_filter, returning the rdf:type object for every row. Aliasing through
       # BIND blocks that optimizer rewrite and is a no-op on engines without the bug.
+      # Outer ordering: the inner subquery's ORDER BY only selects which rows survive
+      # LIMIT/OFFSET; it does NOT propagate through the enclosing join. The outer query
+      # therefore re-sorts on the same key(s) (@sort_outer, datatype-aware STR()). When
+      # no sort is requested, `order by ?s` gives a cheap deterministic default order.
+      outer_order = @sort_outer.empty? ? 'order by ?s' : @sort_outer
       query = %(
       #{prefixes}
 SELECT ?s ?p ?o WHERE {
@@ -229,7 +262,7 @@ SELECT ?s ?p ?o WHERE {
 }
 #{language_filter}
 }
-order by ?s
+#{outer_order}
 )
 
       Solis::LOGGER.info(query) if ConfigFile[:debug]
@@ -274,8 +307,12 @@ order by ?s
     end
 
     def core_query(relationship)
+      # @sort_project re-exposes the sort key(s) bound in @sort_select so the OUTER query
+      # can ORDER BY them. NB: for a multi-valued sort attribute this turns DISTINCT into
+      # one row per (concept, value) pair, which would skew LIMIT/OFFSET — sort attributes
+      # are expected to be single-valued.
       core_query = %(
-  SELECT distinct (?concept AS ?s) WHERE {
+  SELECT distinct (?concept AS ?s)#{@sort_project} WHERE {
     #{@filter[:values].join("\n")}
     #{relationship}
     #{@filter[:concepts].join("\n")}
@@ -286,6 +323,21 @@ order by ?s
 )
     end
 
+    # SPARQL expression to sort an attribute by in the OUTER query.
+    #
+    # The outer (post-join) ORDER BY on this Virtuoso build (08.03.3335) mis-collates
+    # language-tagged literals, so string-ish keys are wrapped in STR() — which fixes the
+    # ordering and is order-preserving for xsd:string, dates, gYear, booleans and EDTF.
+    # Numeric keys must NOT be wrapped: STR() would order them lexically ("100" < "9"),
+    # and numbers carry no language tag so they are unaffected by the collation bug.
+    def sort_key_expression(attribute, meta = @model.class.metadata[:attributes][attribute.to_s])
+      if XSD_NUMERIC_DATATYPES.include?(meta[:datatype_rdf].to_s)
+        "?__#{attribute}"
+      else
+        "STR(?__#{attribute})"
+      end
+    end
+
     def prefixes
       "
 PREFIX sh: <http://www.w3.org/ns/shacl#>

data/lib/solis/shape/reader/sheet.rb

@@ -43,6 +43,11 @@ module Solis
                 raise "_REFERENCES tab must have ['sheeturl', 'description', 'entityrange'] as a header at row 1" unless (%w[sheeturl description entityrange] - references.header).length == 0
               end
 
+              if sheets.key?('_CATEGORIES')
+                categories = sheets['_CATEGORIES']
+                raise "_CATEGORIES tab must have ['name'] as a header at row 1" unless (%w[name] - categories.header).length == 0
+              end
+
               sheets.each do |sheet_name, sheet|
                 if sheet_name !~ /^_/
                   entities = sheets[sheet_name]
@@ -113,6 +118,7 @@ module Solis
                                                    category: e['categories'],
                                                    plural: e['nameplural'],
                                                    label: e['name'].to_s.strip,
+                                                   labels: parse_labels(e),
                                                    sub_class_of: e['subclassof'].nil? || e['subclassof'].empty? ? [] : [e['subclassof']],
                                                    same_as: e['sameas'],
                                                    properties: entity_data })
@@ -135,6 +141,26 @@ module Solis
               raise Solis::Error::GeneralError, e.message
             end
 
+            # Matches the 'Label_XX' columns (header is downcased by ::Sheet#header) where
+            # XX is a language tag, ex. 'Label_NL', 'Label_EN', 'Label_nl-BE'
+            LANGUAGE_LABEL_COLUMN = /\Alabel_(?<language>[a-z]{2,3}([-_][a-z0-9]{2,8})*)\z/.freeze
+
+            # Collects all 'Label_XX' columns of a row into { nl: 'Publiceren', en: 'Publish' }
+            def parse_labels(row)
+              labels = {}
+              return labels unless row.respond_to?(:each_pair)
+
+              row.each_pair do |column, value|
+                match = LANGUAGE_LABEL_COLUMN.match(column.to_s)
+                next if match.nil?
+                next if value.nil? || value.to_s.strip.empty?
+
+                labels[Solis::LanguageTag.normalize(match[:language])] = value.to_s.strip
+              end
+
+              labels
+            end
+
             def parse_entity_data(entity_name, graph_prefix, _graph_name, e, options = {})
               properties = {}
               entity_data = e
@@ -168,31 +194,9 @@ module Solis
                       same_as: p['sameas'],
                       order: p['order'],
                       category: p['categories'],
+                      label: parse_labels(p),
                       description: p['description']
                     }
-
-                    # unless graph_prefix.eql?(datatype_prefix.to_sym)
-                    #   prefixes = options[:prefixes]
-                    #   if prefixes.key?(datatype_prefix.to_sym) && !prefixes[datatype_prefix.to_sym][:sheet_url].empty?
-                    #     tmp = URI(prefixes[datatype_prefix.to_sym][:sheet_url]).path.split('/')
-                    #     spreadsheet_id = tmp[tmp.index('d') + 1]
-                    #
-                    #     processed_remote_sheet = {}
-                    #     if prefixes[datatype_prefix.to_sym].key?(:data) && !prefixes[datatype_prefix.to_sym][:data].empty?
-                    #       processed_remote_sheet = prefixes[datatype_prefix.to_sym][:data]
-                    #     else
-                    #       if options[:follow]
-                    #         sleep 30
-                    #         remote_sheet = read_sheets(options[:key], spreadsheet_id, { from_cache: true })
-                    #         processed_remote_sheet = process_sheet(options[:key], remote_sheet, {follow: false})
-                    #         prefixes[datatype_prefix.to_sym][:data] = processed_remote_sheet
-                    #       end
-                    #     end
-                    #
-                    #     processed_remote_sheet
-                    #   end
-                    # end
-
                   end
                 end
               end
@@ -266,12 +270,32 @@ hide empty members
               datatypes[datatype][as]
             end
 
+            # Turns { nl: 'Publiceren', en: 'Publish' } into a single rdfs:label statement with
+            # one language tagged literal per language, ex.
+            #   rdfs:label "Publiceren"@nl, "Publish"@en ;
+            def build_labels(labels, rdfs_prefix, indent)
+              return '' if labels.nil? || labels.empty?
+
+              literals = labels.map do |language, value|
+                %("#{value.to_s.gsub('"', "'").gsub(/\n|\r/, '')}"@#{language})
+              end
+
+              "\n#{indent}#{rdfs_prefix}:label #{literals.join(', ')} ;"
+            end
+
             def build_shacl(datas)
               shacl_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /shacl/ }.keys.first
               shacl_prefix = 'sh' if shacl_prefix.nil?
 
+              rdfs_prefix = datas.first[:ontologies][:all].select { |_, v| v[:uri] =~ /rdf-schema/ }.keys.first
+
               out = header(datas.first)
 
+              if rdfs_prefix.nil?
+                rdfs_prefix = 'rdfs'
+                out += "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n"
+              end
+
               datas.each do |data|
                 data[:entities].each do |entity_name, metadata|
                   graph_prefix = data[:ontologies][:base][:prefix]
@@ -300,7 +324,7 @@ hide empty members
                     unless node.nil? || node.empty?
                       "\n    #{shacl_prefix}:node         #{node} ;"
                     end}
-    #{shacl_prefix}:name         "#{label}" ;
+    #{shacl_prefix}:name         "#{label}" ;#{build_labels(metadata[:labels], rdfs_prefix, ' ' * 4)}
 )
                   metadata[:properties].each do |property, property_metadata|
                     attribute = property.to_s.strip
@@ -314,15 +338,16 @@ hide empty members
                     order = property_metadata.key?(:order) && property_metadata[:order] ? property_metadata[:order]&.strip : nil
                     category = property_metadata.key?(:category) && property_metadata[:category] ? property_metadata[:category]&.strip : nil
                     category = category.nil? || category.empty? ? nil : category
+                    labels = build_labels(property_metadata[:label], rdfs_prefix, ' ' * 17)
 
                     unless category.nil?
-                      group = "#{category.classify}Group"
+                      group = "#{category.camelize}Group"
                       groups[group] = category unless groups.key?(group)
                     end
 
                     if datatype =~ /^#{graph_prefix}:/ || datatype =~ /^<#{graph_name}/
                       out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}" ;
+                 #{shacl_prefix}:name "#{attribute}" ;#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:nodeKind #{shacl_prefix}:IRI ;
@@ -332,7 +357,7 @@ hide empty members
                     else
                       if datatype.eql?('rdf:langString') && max_count.eql?('1')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:uniqueLang true ;
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}
@@ -340,7 +365,7 @@ hide empty members
 )
                       elsif datatype.eql?('rdf:langString')
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{category.nil? ? '' : "\n                 #{shacl_prefix}:group #{graph_prefix}:#{group} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
@@ -348,7 +373,7 @@ hide empty members
 )
                       else
                         out += %(    #{shacl_prefix}:property [#{shacl_prefix}:path #{path} ;
-                 #{shacl_prefix}:name "#{attribute}";
+                 #{shacl_prefix}:name "#{attribute}";#{labels}
                  #{shacl_prefix}:description "#{description}" ;#{order.nil? ? '' : "\n                 #{shacl_prefix}:order #{order} ;"}
                  #{shacl_prefix}:datatype #{datatype} ;#{min_count =~ /\d+/ ? "\n                 #{shacl_prefix}:minCount #{min_count} ;" : ''}#{max_count =~ /\d+/ ? "\n                 #{shacl_prefix}:maxCount #{max_count} ;" : ''}
     ] ;
@@ -362,8 +387,8 @@ hide empty members
                   groups.each do |group, category|
                     out += %(
 #{graph_prefix}:#{group}
-	a sh:PropertyGroup ;
-	rdfs:label "#{category}" .
+	a #{shacl_prefix}:PropertyGroup ;
+	#{rdfs_prefix}:label "#{category}" .
 
 )
                   end
@@ -489,7 +514,7 @@ hide empty members
               puts e.message
             end
 
-            def build_inflections(datas)
+            def build_inflections(datas, categories = [])
               inflections = {}
               datas.each do |data|
                 data[:entities].each do |entity, metadata|
@@ -498,6 +523,16 @@ hide empty members
                 end
               end
 
+              # categories are uninflected, singular and plural are the same word.
+              # an entity by the same name keeps its own plural.
+              # categories.each do |category|
+              #   name = category.to_s.strip
+              #   next if name.empty?
+              #
+              #   inflections[name.to_sym] = name unless inflections.key?(name.to_sym) || inclections.value?(name.to_s)
+              #   inflections[name.underscore.to_sym] = name.underscore unless inflections.key?(name.underscore.to_sym) || inclections.value?(name.to_s)
+              # end
+
               inflections.to_json
             rescue StandardError => e
               puts e.message
@@ -743,6 +778,11 @@ hide empty members
 
           options[:prefixes] = prefixes
           options[:metadata] = metadata
+
+          # like _REFERENCES, _CATEGORIES only lives in the main sheet
+          categories = sheet_data.is_a?(Hash) && sheet_data.key?('_CATEGORIES') ? sheet_data['_CATEGORIES'].map { |c| c['name'] } : []
+          Solis::LOGGER.info("Found #{categories.length} categories") unless categories.empty?
+
           #TODO: cleanup
           if sheet_data.is_a?(Hash)
             raise "No _REFERENCES sheet found" unless sheet_data.key?("_REFERENCES")
@@ -786,7 +826,7 @@ hide empty members
           Solis::LOGGER.info('Generating SCHEMA')
           schema = build_schema(datas)
           Solis::LOGGER.info('Generating INFLECTIONS')
-          inflections = build_inflections(datas)
+          inflections = build_inflections(datas, categories)
           Solis::LOGGER.info('Generating JSON SCHEMA')
           json_schema = build_json_schema(shacl)
 

data/lib/solis/shape.rb

@@ -14,6 +14,8 @@ module Solis
             parse_solution(shapes, solution)
           end
 
+          add_labels(shapes, graph)
+
           # shapes = add_missing_attributes(shapes)
           shapes
         rescue Solis::Error::GeneralError => e
@@ -137,6 +139,47 @@ module Solis
           shapes[shape_name] = shape
         end
 
+        # Reads the rdfs:label statements of every node shape and property shape into
+        # { nl: 'Publiceren', en: 'Publish' }. This is a second pass on purpose: adding
+        # rdfs:label to the main query would multiply every solution by the number of
+        # languages.
+        def add_labels(shapes, graph)
+          graph.query([nil, RDF.type, RDF::Vocab::SHACL.NodeShape]) do |node_shape|
+            shape_name = name_of(graph, node_shape.subject)
+            shape = shapes[shape_name]
+            next if shape.nil?
+
+            shape[:label] = labels_of(graph, node_shape.subject)
+
+            graph.query([node_shape.subject, RDF::Vocab::SHACL.property, nil]) do |property|
+              attribute = shape[:attributes][name_of(graph, property.object)]
+              next if attribute.nil?
+
+              attribute[:label] = labels_of(graph, property.object)
+            end
+          end
+
+          shapes
+        end
+
+        def name_of(graph, subject)
+          graph.query([subject, RDF::Vocab::SHACL.name, nil]).first&.object&.value
+        end
+
+        def labels_of(graph, subject)
+          labels = {}
+
+          graph.query([subject, RDF::RDFS.label, nil]).each do |statement|
+            literal = statement.object
+            language = Solis::LanguageTag.normalize(literal.language) if literal.is_a?(RDF::Literal)
+            next if language.nil?
+
+            labels[language] = literal.value
+          end
+
+          labels
+        end
+
         def query
           SPARQL.parse %(
 PREFIX sh: <http://www.w3.org/ns/shacl#>

data/lib/solis/version.rb

@@ -1,3 +1,3 @@
 module Solis
-  VERSION = "0.125.0"
+  VERSION = "0.127.0"
 end

data/lib/solis.rb

@@ -7,6 +7,7 @@ require "solis/version"
 require 'solis/config_file'
 require "solis/error"
 require 'solis/rdf_edtf_literal'
+require 'solis/language_tag'
 require 'solis/graph'
 require 'solis/shape'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solis
 version: !ruby/object:Gem::Version
-  version: 0.125.0
+  version: 0.127.0
 platform: ruby
 authors:
 - Mehmet Celik
@@ -307,6 +307,7 @@ files:
 - lib/solis/error/not_found_error.rb
 - lib/solis/error/query_error.rb
 - lib/solis/graph.rb
+- lib/solis/language_tag.rb
 - lib/solis/model.rb
 - lib/solis/model.rb.ok
 - lib/solis/options.rb