solis 0.64.0

data/lib/solis/query/filter.rb

@@ -0,0 +1,133 @@
+module Solis
+    module QueryFilter
+      def filter(params)
+        if params.key?(:language)
+          @language = params[:language].nil? || params[:language].blank? ? nil : params[:language]
+        end
+
+        parsed_filters = {values: ["VALUES ?type {#{target_class}}"], concepts: ['?concept a ?type .'] }
+        if params.key?(:filters)
+          filters = params[:filters]
+
+          if filters.is_a?(String)
+            parsed_filters[:concepts] << concept_by_string(filters)
+          else
+            i=0
+            filters.each do |key, value|
+              if is_key_a_shacl_entity?(key) #nodeKind sh:URI
+                parsed_filters[:values] << values_for(key, value)
+                parsed_filters[:concepts] << concepts_for(key)
+              else
+                parsed_filters[:concepts] << other_stuff(key, value, i)
+              end
+              i+=1
+            end
+          end
+        end
+
+        @filter = parsed_filters
+        self
+      rescue StandardError => e
+        LOGGER.error(e.message)
+        LOGGER.error(e.backtrace.join("\n"))
+        raise Error::GeneralError, e.message
+      end
+
+      private
+
+      def is_key_a_shacl_entity?(key)
+        @metadata[:attributes].key?(key.to_s) && @metadata[:attributes][key.to_s][:node_kind] && @metadata[:attributes][key.to_s][:node_kind]&.vocab == RDF::Vocab::SH
+      end
+
+      def values_for(key, value, i=0)
+        values_model = @model.class.graph.shape_as_model(@metadata[:attributes][key.to_s][:datatype].to_s)&.new
+        if value.is_a?(Hash)
+          other_stuff(key, value, i)
+        else
+          "VALUES ?filter_by_#{key}_id{#{value.split(',').map {|v| target_class_by_model(values_model, v)}.join(' ')}}" if values_model
+        end
+      end
+
+      def concepts_for(key)
+        filter_predicate = URI.parse(@metadata[:attributes][key.to_s][:path])
+        filter_predicate.path = "/#{key.to_s.downcase}"
+
+        "?concept <#{filter_predicate.to_s}> ?filter_by_#{key}_id ."
+      end
+
+      def concept_by_string(filters)
+        contains = filters.split(',').map { |m| "CONTAINS(LCASE(str(?__search)), LCASE(\"#{m}\"))" }.join(' || ')
+        "?concept (#{@metadata[:attributes].map { |_, m| "(<#{m[:path]}>)" }.join('|')}) ?__search FILTER CONTAINS(LCASE(str(?__search)), LCASE(\"#{contains}\")) ."
+      end
+
+      def other_stuff(key, value, i)
+        filter = ''
+        unless value.is_a?(Hash) && value.key?(:value)
+          #TODO: only handles 'eq' for now
+          value = { value: value.is_a?(Array) ? value.first : value, operator: '=', is_not: false }
+        end
+
+        if value[:value].is_a?(String)
+          contains = value[:value].split(',').map { |m| "CONTAINS(LCASE(str(?__search#{i})), LCASE(\"#{m}\"))" }.join(' || ')
+        else
+          value[:value] = [value[:value]] unless value[:value].is_a?(Array)
+          value[:value].flatten!
+          contains = value[:value].map { |m| m.is_a?(String) ? "CONTAINS(LCASE(str(?__search#{i})), LCASE(\"#{m}\"))" : next }.join(' || ')
+        end
+
+        metadata = @metadata[:attributes][key.to_s]
+        if metadata
+          if metadata[:path] =~ %r{/id$}
+            if value[:value].is_a?(String)
+              contains = value[:value].split(',').map { |m| "\"#{m}\"" }.join(',')
+            else
+              value[:value].flatten!
+              contains = value[:value].map { |m| "\"#{m}\"" }.join(',')
+            end
+            if value[:is_not]
+              value[:value].each do |v|
+                v=normalize_string(v)
+                filter = "filter( !exists {?concept <#{@model.class.graph_name}id> \"#{v}\"})"
+              end
+            else
+              filter = "?concept <#{@model.class.graph_name}id> ?__search FILTER (?__search IN(#{contains})) .\n"
+            end
+          else
+            datatype = ''
+            datatype = "^^<http://www.w3.org/2001/XMLSchema#boolean>" if metadata[:datatype].eql?(:boolean)
+
+            if ["=", "<", ">"].include?(value[:operator])
+              not_operator = value[:is_not] ? '!' : ''
+              value[:value].each do |v|
+                if metadata[:datatype_rdf].eql?('http://www.w3.org/1999/02/22-rdf-syntax-ns#langString')
+                    filter  = "?concept <#{metadata[:path]}> ?__search#{i} "
+                    filter += "FILTER(langMatches( lang(?__search#{i}), \"#{v[:"@language"]}\" )). "
+                    search_for = v[:"@value"].is_a?(Array) ? v[:"@value"].first : v[:"@value"]
+                    search_for = normalize_string(search_for)
+                    filter += "FILTER(str(?__search#{i}) #{not_operator}#{value[:operator]} \"#{search_for}\"#{datatype}) .\n"
+                else
+                  v=normalize_string(v)
+                  filter = "?concept <#{metadata[:path]}> ?__search#{i} FILTER(?__search#{i} #{not_operator}#{value[:operator]} \"#{v}\"#{datatype}) .\n"
+                end
+
+
+              end
+            else
+              filter = "?concept <#{metadata[:path]}> ?__search#{i} FILTER(#{contains.empty? ? '""' : contains}) .\n"
+            end
+          end
+        end
+
+        filter
+      end
+
+      def normalize_string(string)
+        if string.is_a?(String)
+          string.gsub(/\t/, '\t').gsub(/\n/,'\n').gsub(/\r/,'\r').gsub(/\f/,'\f').gsub(/"/,'\"').gsub(/'/,'\'').gsub(/\\/,'\\')
+        else
+          string
+        end
+      end
+
+    end
+end

data/lib/solis/query/run.rb

@@ -0,0 +1,97 @@
+require 'solis/store/sparql/client'
+require 'solis/config_file'
+
+class Solis::Query::Runner
+    def self.run(entity, query)
+      result = {}
+      context = JSON.parse %(
+{
+    "@context": {
+        "@vocab": "#{Solis::Options.instance.get[:graph_name]}",
+        "id": "@id"
+    },
+    "@type": "#{entity}",
+    "@embed": "@always"
+}
+   )
+
+      c = Solis::Store::Sparql::Client.new(Solis::Options.instance.get[:sparql_endpoint], Solis::Options.instance.get[:graph_name])
+      r = c.query(query)
+      if r.is_a?(SPARQL::Client)
+        g = RDF::Graph.new
+        t = r.query('select * where{?s ?p ?o}')
+        t.each do |s|
+          g << [s.s, s.p, s.o]
+        end
+
+        framed = nil
+        JSON::LD::API.fromRDF(g) do |e|
+          framed = JSON::LD::API.frame(e, context)
+        end
+        result = sanitize_result(framed)
+      else
+        t = []
+        r.each do |s|
+          t << s.to_h
+        end
+        result = sanitize_result({'@graph' => t})
+      end
+      result
+    rescue StandardError => e
+      puts e.message
+      raise e
+    end
+
+    private
+
+    def self.sanitize_result(framed)
+      data = framed&.key?('@graph') ? framed['@graph'] : [framed]
+
+      data.map do |d|
+        d.delete_if { |e| e =~ /^@/ }
+        if d.is_a?(Hash)
+          new_d = {}
+          d.each do |k,v|
+            if v.is_a?(Hash)
+              if v.key?('@type')
+                type = v['@type']
+                if v.key?('@value')
+                  value = v['@value']
+                  case type
+                  when "http://www.w3.org/2001/XMLSchema#dateTime"
+                    value = DateTime.parse(value)
+                  when "http://www.w3.org/2001/XMLSchema#date"
+                    value = Date.parse(value)
+                  when "http://www.w3.org/2006/time#DateTimeInterval"
+                    value = ISO8601::TimeInterval.parse(value)
+                  end
+                  v = value
+                end
+                v = sanitize_result(v) if v.is_a?(Hash)
+              end
+              new_d[k] = v.class.method_defined?(:value) ? v.value : v
+            elsif v.is_a?(Array) #todo: make recursive
+              new_d[k] = []
+              v.each do |vt|
+                if vt.is_a?(Hash)
+                  if vt.key?('@value')
+                    new_d[k] << vt['@value']
+                  else
+                    new_d[k] << (vt.is_a?(String) ? vt : sanitize_result(vt))
+                  end
+                else
+                  new_d[k] << (vt.is_a?(String) ? vt : sanitize_result(vt))
+                end
+              end
+              new_d[k].flatten!
+            else
+              new_d[k] = v.class.method_defined?(:value) ? v.value : v
+            end
+          end
+          d = new_d
+        end
+
+        d
+      end
+    end
+end

data/lib/solis/query.rb

@@ -0,0 +1,347 @@
+# frozen_string_literal: true
+
+require 'moneta'
+require 'solis/query/filter'
+require 'solis/query/construct'
+require 'solis/query/run'
+require 'uuidtools'
+
+module Solis
+  class Query
+    include Enumerable
+    include Solis::QueryFilter
+
+    def self.run(entity, query)
+      Solis::Query::Runner.run(entity, query)
+    end
+
+    def self.run_construct_with_file(filename, id_name, entity, ids, from_cache = '1')
+      f = File.read(filename)
+      run_construct(f, id_name, entity, ids, from_cache)
+    end
+
+    def self.uuid(key)
+      UUIDTools::UUID.sha1_create(UUIDTools::UUID_URL_NAMESPACE, key).to_s
+    end
+
+    def self.run_construct(query, id_name, entity, ids, from_cache = '1')
+      raise 'Please supply one or more uuid\'s' if ids.nil? || ids.empty?
+
+      result = {}
+
+      key = uuid("#{entity}-#{ids}")
+
+      if result.nil? || result.empty? || (from_cache.eql?('0'))
+        ids = ids.split(',') if ids.is_a?(String)
+        ids = [ids] unless ids.is_a?(Array)
+        ids = ids.map do |m|
+          if URI(m).class.eql?(URI::Generic)
+            "<#{Solis::Options.instance.get[:graph_name]}#{entity.tableize}/#{m}>"
+          else
+            "<#{m}>"
+          end
+        end
+        ids = ids.join(" ")
+
+
+        q = query.gsub(/{ ?{ ?VALUES ?} ?}/, "VALUES ?#{id_name} { #{ids} }")
+
+        result = Solis::Query.run(entity, q)
+      end
+      result
+    rescue StandardError => e
+      puts e.message
+      raise e
+    end
+
+    def initialize(model)
+      @construct_cache = File.absolute_path(Solis::Options.instance.get[:cache])
+      @model = model
+      @shapes = @model.class.shapes
+      @metadata = @model.class.metadata
+      @sparql_endpoint = @model.class.sparql_endpoint
+      @sparql_client = SPARQL::Client.new(@sparql_endpoint, graph: @model.class.graph_name)
+      @filter = {values: ["VALUES ?type {#{target_class}}"], concepts: ['?concept a ?type .'] }
+      @sort = 'ORDER BY ?s'
+      @sort_select = ''
+      @language = Graphiti.context[:object]&.language || Solis::Options.instance.get[:language] || 'en'
+      @query_cache = Moneta.new(:HashFile, dir: @construct_cache)
+    end
+
+    def each(&block)
+      data = query
+      return unless data.methods.include?(:each)
+      data.each(&block)
+    rescue StandardError => e
+      message = "Unable to get next record: #{e.message}"
+      LOGGER.error(message)
+      raise Error::CursorError, message
+    end
+
+    def sort(params)
+      @sort = ''
+      @sort_select = ''
+      if params.key?(:sort)
+        i = 0
+        params[:sort].each do |attribute, direction|
+          path = @model.class.metadata[:attributes][attribute.to_s][:path]
+          @sort_select += "optional {\n" if @model.class.metadata[:attributes][attribute.to_s][:mincount] == 0
+          @sort_select += "?concept <#{path}> ?__#{attribute} . "
+          @sort_select += "}\n" if @model.class.metadata[:attributes][attribute.to_s][:mincount] == 0
+          @sort += ',' if i.positive?
+          @sort += "#{direction.to_s.upcase}(?__#{attribute})"
+          i += 1
+        end
+
+        @sort = "ORDER BY #{@sort}" if i.positive?
+      end
+
+      self
+    end
+
+    def paging(params = {})
+      current_page = params[:current_page] || 1
+      per_page = params[:per_page] || 10
+
+      @offset = 0
+      @offset = current_page * per_page if current_page > 1
+      @limit = per_page
+      self
+    end
+
+    def count
+      sparql_client = @sparql_client
+      if model_construct?
+        sparql_client = Solis::Query::Construct.new(@model).run
+      end
+
+      relationship = ''
+      core_query = core_query(relationship)
+      count_query = core_query.gsub(/SELECT .* WHERE/, 'SELECT (COUNT(distinct ?concept) as ?count) WHERE')
+
+      result = sparql_client.query(count_query)
+      solution = result.first
+      solution.nil? ? 0 : solution[:count].object || 0
+    end
+
+    private
+
+    def model_construct?
+      File.exist?("#{ConfigFile.path}/constructs/#{@model.name.tableize.singularize}.sparql")
+    end
+
+    def target_class
+      #      descendants = ObjectSpace.each_object(Class).select { |klass| klass < @model.class }.map { |m| m.class.name.eql?('Class') ? m.superclass : m }.map { |m| m.metadata[:target_class].value }
+      descendants = ObjectSpace.each_object(Class).select { |klass| klass < @model.class }.reject { |m| m.metadata.nil? }.map { |m| m.metadata[:target_class].value }
+      descendants << @model.class.metadata[:target_class].value
+      descendants.map { |m| "<#{m}>" }.join(' ')
+    end
+
+    def target_class_by_model(model, id=nil)
+      descendants = ObjectSpace.each_object(Class).select { |klass| klass < model.class }.reject { |m| m.metadata.nil? }.map { |m| m.metadata[:target_class].value.tableize }
+      descendants << model.class.metadata[:target_class].value.tableize
+      if id.nil?
+        descendants.map { |m| "<#{m}>" }.join(' ')
+      else
+        descendants.map { |m| "<#{m}/#{id}>" }.join(' ')
+      end
+    end
+
+
+    def query(options = {})
+      limit = @limit || 10
+      offset = @offset || 0
+
+      sparql_client = model_construct? ? Solis::Query::Construct.new(@model).run : @sparql_client
+      #sparql_client = model_construct? ? SPARQL::Client.new(run_construct) : @sparql_client
+
+      relationship = ''
+      if options.key?(:relationship)
+        link = "#{@model.class.graph_name}#{ActiveSupport::Inflector.pluralize(@klass.name).downcase}/#{id}"
+        path = @model.class.metadata[:attributes][options[:relationship]][:path]
+        relationship = "<#{link}> <#{path}> ?o ."
+      end
+
+      core_query = core_query(relationship)
+      if core_query =~ /IN\((.*?)\)/
+        #limit = $1.gsub('"','').split(',').length
+      else
+        core_query += " LIMIT #{limit} OFFSET #{offset}"
+      end
+
+      query = %(
+      #{prefixes}
+SELECT ?s ?p ?o WHERE {
+ ?s ?p ?o
+{
+  #{core_query}
+}
+#{language_filter}
+}
+order by ?s
+)
+
+      Solis::LOGGER.info(query) if ConfigFile[:debug]
+
+      query_key = "#{@model.name}-#{Digest::MD5.hexdigest(query)}"
+
+      result = nil
+
+      from_cache = Graphiti.context[:object].from_cache || '1'
+      if @query_cache.key?(query_key) && from_cache.eql?('1')
+        result = @query_cache[query_key]
+        Solis::LOGGER.info("CACHE: from #{query_key}")# if ConfigFile[:debug]
+      else
+        result = graph_to_object(sparql_client.query(query))
+        @query_cache[query_key] = result unless result.nil? || result.empty?
+        Solis::LOGGER.info("CACHE: to #{query_key}")# if ConfigFile[:debug]
+      end
+
+      result
+    rescue StandardError => e
+      Solis::LOGGER.error(e.message)
+      Solis::LOGGER.error(e.backtrace.join("\n"))
+    end
+
+    def language_filter
+      if @language.nil?
+        ''
+      else
+        %(
+        filter (
+            !isLiteral(?o) ||
+            langmatches(lang(?o), "#{@language}")
+            || (langmatches(lang(?o), "") && not exists {
+                    ?s ?p ?other.
+                    filter(isLiteral(?other) && langmatches(lang(?other), "#{@language}"))
+                }))
+        )
+      end
+    end
+
+    def core_query(relationship)
+      core_query = %(
+  SELECT distinct (?concept AS ?s) WHERE {
+    #{@filter[:values].join("\n")}
+    ?concept ?role ?objects.
+    #{relationship}
+    #{@filter[:concepts].join("\n")}
+
+    #{@sort_select}
+  }
+#{@sort}
+)
+    end
+
+    def prefixes
+      "
+PREFIX sh: <http://www.w3.org/ns/shacl#>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX schema: <http://schema.org/>
+PREFIX rdfv: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+PREFIX #{@model.class.graph_prefix}: <#{@model.class.graph_name}>"
+    end
+
+    def graph_to_object(solutions)
+      return [] if solutions.empty?
+      target_class = @model.class.metadata[:target_class].value.split('/').last
+      result = []
+      record_uri = ''
+
+      begin
+        # solutions.sort! { |x, y| x.s.value <=> y.s.value }.map { |m| m.s.value }
+        solution_types = solutions.dup.filter!(p: RDF::RDFV.type)
+        solution_types.each do |type|
+          solution_model = @model.class.graph.shape_as_model(type.o.value.split('/').last)
+          data = {}
+          statements = solutions.dup.filter!(s: type.s)
+          statements.each do |statement|
+            next if statement.p.eql?(RDF::RDFV.type)
+
+            begin
+              record_uri = statement.s.value
+              attribute = statement.p.value.split('/').last.underscore
+
+              if statement.o.valid?
+                if statement.o.is_a?(RDF::URI)
+                  object = statement.o.canonicalize.value
+                else
+                  object = statement.o.canonicalize.object
+                end
+              else
+                object = Integer(statement.o.value) if Integer(statement.o.value) rescue nil
+                object = Float(statement.o.value) if object.nil? && Float(statement.o.value) rescue nil
+                object = statement.o.value if object.nil?
+              end
+
+              begin
+                datatype = RDF::Vocabulary.find_term(@model.class.metadata[:attributes][attribute][:datatype_rdf])
+                if RDF::Literal(statement.o).datatype.value != datatype
+                  if statement.o.is_a?(RDF::URI)
+                    object = RDF::Literal.new(statement.o.canonicalize.value, datatype: RDF::URI).object
+                  else
+                    object = RDF::Literal.new(statement.o.canonicalize.object, datatype: datatype).object
+                  end
+                end
+              rescue StandardError => e
+                if object.is_a?(Hash)
+                  object = if object.key?(:fragment) && !object[:fragment].nil?
+                             "#{object[:path]}##{object[:fragment]}"
+                           else
+                             object[:path]
+                           end
+                end
+              end
+
+              # fix non matching attributes by data type
+              if solution_model.metadata[:attributes][attribute].nil?
+                candidates = solution_model.metadata[:attributes].select { |_k, s| s[:class] == statement.p }.keys - data.keys
+                attribute = candidates.first unless candidates.empty?
+              end
+
+              begin
+                unless solution_model.metadata[:attributes][attribute][:node_kind].nil?
+                  node_class = solution_model.metadata[:attributes][attribute][:class].value.split('/').last
+                  object = solution_model.graph.shape_as_model(node_class).new({ id: object.split('/').last })
+                end
+              rescue StandardError => e
+                puts e.message
+              end
+
+              if data.key?(attribute) # attribute exists
+                raise "Cardinality error, max = #{solution_model.metadata[:attributes][attribute][:maxcount]}" if solution_model.metadata[:attributes][attribute][:maxcount] == 0
+                if solution_model.metadata[:attributes][attribute][:maxcount] == 1 && data.key?(attribute)
+                  raise "Cardinality error, max = #{solution_model.metadata[:attributes][attribute][:maxcount]}"
+                elsif solution_model.metadata[:attributes][attribute][:maxcount] == 1
+                  data[attribute] = object
+                else
+                  data[attribute] = [data[attribute]] unless data[attribute].is_a?(Array)
+                  data[attribute] << object
+                end
+              else
+                if solution_model.metadata[:attributes][attribute][:maxcount].nil? || solution_model.metadata[:attributes][attribute][:maxcount] > 1
+                  data[attribute] = [object]
+                else
+                  data[attribute] = object
+                end
+              end
+            rescue StandardError => e
+              puts e.backtrace.first
+              Solis::LOGGER.error("#{record_uri} - graph_to_object - #{attribute} - #{e.message}")
+              g = RDF::Graph.new
+              g << [statement.s, statement.p, statement.o]
+              Solis::LOGGER.error(g.dump(:ttl).to_s)
+            end
+          end
+          result << solution_model.new(data) unless data.empty?
+        end
+      rescue StandardError => e
+        Solis::LOGGER.error("#{record_uri} - graph_to_object - #{e.message}")
+      end
+
+      # result << solution_model.new(data) unless data.empty?
+      result
+    end
+  end
+end

data/lib/solis/resource.rb

@@ -0,0 +1,37 @@
+require_relative 'sparql_adaptor'
+require_relative 'config_file'
+require_relative 'options'
+
+module Solis
+  class Resource < ::Graphiti::Resource
+
+    self.abstract_class = true
+    self.adapter = Solis::SparqlAdaptor
+    self.endpoint_namespace = Solis::Options.instance.get[:base_path] rescue ''
+    self.validate_endpoints = true
+
+    def self.sparql_endpoint
+      @sparql_endpoint
+    end
+
+    def self.sparql_endpoint=(sparql_endpoint)
+      @sparql_endpoint = sparql_endpoint
+    end
+  end
+
+  class NoopEndpoint
+    def initialize(path, action)
+      @path = path
+      @action = action
+    end
+
+    def sideload_allowlist
+      @allow_list
+    end
+
+    def sideload_allowlist=(val)
+      @allow_list = JSONAPI::IncludeDirective.new(val).to_hash
+      super(@allow_list)
+    end
+  end
+end