chronicle-core 0.2.1 → 0.3.0

data/lib/chronicle/schema/rdf_parsing/graph_transformer.rb

@@ -0,0 +1,122 @@
+module Chronicle
+  module Schema
+    module RDFParsing
+      # A class that inteprets a DSL to transform a base schema graph into
+      # a new one by walking through the types and properties of the base
+      # and selecting which ones to include in the new graph.
+      class GraphTransformer
+        attr_reader :graph, :base_graph, :new_graph
+
+        def initialize
+          @base_graph = nil
+          @new_graph = Chronicle::Schema::SchemaGraph.new(default_namespace: 'https://schema.chronicle.app/')
+          @current_parent = nil
+        end
+
+        def self.transform(&)
+          transformer = new
+          transformer.start_evaluating(&)
+
+          # TODO: figure out if we need to this still
+          transformer.new_graph.properties.each do |property|
+            property.domain = property.domain.select do |type_id|
+              transformer.new_graph.find_type_by_id(type_id)
+            end
+          end
+
+          transformer.new_graph.build_references!
+          transformer.new_graph
+        end
+
+        def self.transform_from_file(definition_file_path)
+          dsl_commands = File.read(definition_file_path)
+          transform_from_string(dsl_commands)
+        end
+
+        def self.transform_from_string(dsl_definition)
+          transform do
+            instance_eval(dsl_definition)
+          end
+        end
+
+        def start_evaluating(&)
+          instance_eval(&) if block_given?
+        end
+
+        private
+
+        def version(version)
+          @new_graph.version = version
+        end
+
+        def set_base_graph(name, version)
+          case name
+          when 'schema.org'
+            @base_graph = Chronicle::Schema::RDFParsing::Schemaorg.graph_for_version(version)
+          else
+            raise ArgumentError, "Unknown base graph: #{name}"
+          end
+        end
+
+        def pick_type(subtype_identifier, &)
+          id = @base_graph.identifier_to_uri(subtype_identifier)
+          type = @base_graph.find_type_by_id(id)
+          raise ArgumentError, "Subtype not found: #{subtype_identifier}" unless type
+
+          new_subtype = @new_graph.add_type(subtype_identifier)
+          new_subtype.comment = type.comment
+          new_subtype.see_also = type.id
+
+          @current_parent&.add_subtype_id(new_subtype.id)
+
+          previous_parent = @current_parent
+          @current_parent = new_subtype
+
+          instance_eval(&) if block_given?
+
+          @current_parent = previous_parent
+        end
+
+        def pick_all_subtypes(&)
+          @base_graph.find_type(@current_parent.short_id.to_sym).subtype_ids.each do |subtype_id|
+            identifier = @base_graph.id_to_identifier(subtype_id)
+            pick_type(identifier, &)
+          end
+        end
+
+        def apply_property(property_identifier, many: false, required: false)
+          property = @base_graph.find_property(property_identifier)
+          raise ArgumentError, "Property not found: #{property_identifier}" unless property
+
+          new_property = @new_graph.add_property(property_identifier)
+          new_property.range = property.range.map do |p|
+            base_range_type = @base_graph.find_type_by_id(p)
+            @new_graph.identifier_to_uri(base_range_type.identifier)
+          end
+          new_property.comment = property.comment
+          new_property.many = many
+          new_property.required = required
+          new_property.domain += [@current_parent.id]
+          new_property.see_also = property.id
+        end
+
+        def add_property(property_identifier, many: false, required: false, comment: nil)
+          new_property = @new_graph.add_property(property_identifier)
+          new_property.domain += [@current_parent.id]
+          # TODO: expand this to handle multiple ranges
+          new_property.range = [@new_graph.identifier_to_uri(:Text)]
+          new_property.comment = comment
+          new_property.many = many
+          new_property.required = required
+        end
+
+        def pick_all_properties
+          @base_graph.find_type_by_id(@current_parent.id).properties.each do |property|
+            identifier = @base_graph.id_to_identifier(property.id)
+            apply_property(identifier)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/schema/rdf_parsing/rdf_serializer.rb

@@ -0,0 +1,138 @@
+require 'rdf/turtle'
+
+module Chronicle
+  module Schema
+    module RDFParsing
+      PREFIXES = {
+        schemaorg: 'https://schema.org/',
+        owl: 'http://www.w3.org/2002/07/owl#',
+        dc: 'http://purl.org/dc/terms/',
+        rdf: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
+        rdfs: 'http://www.w3.org/2000/01/rdf-schema#',
+        xml: 'http://www.w3.org/XML/1998/namespace',
+        xsd: 'http://www.w3.org/2001/XMLSchema#'
+      }.freeze
+
+      # Take a graph and serialize it as a ttl string
+      class RDFSerializer
+        attr_reader :graph
+
+        def initialize(graph)
+          raise ArgumentError, 'graph must be a SchemaGraph' unless graph.is_a?(Chronicle::Schema::SchemaGraph)
+
+          @graph = graph
+        end
+
+        def self.serialize(graph, include_generator_comment: true)
+          new(graph).serialize(include_generator_comment:)
+        end
+
+        def serialize(include_generator_comment: true)
+          schema_graph = RDF::Graph.new
+
+          schema_graph << ontology_triple
+          schema_graph << version_triple
+
+          graph.types.each do |klass|
+            serialize_class(klass).each do |triple|
+              # binding.pry
+              schema_graph << triple
+            end
+          end
+
+          graph.properties.each do |property|
+            serialize_property(property).each do |triple|
+              schema_graph << triple
+            end
+          end
+
+          prefixes = {
+            '': default_namespace
+          }.merge(PREFIXES)
+
+          output_str = ''
+          output_str += generation_header if include_generator_comment
+          output_str + schema_graph.dump(:ttl, prefixes:)
+        end
+
+        private
+
+        def default_namespace
+          @graph.default_namespace
+        end
+
+        def generation_header
+          <<~TTL
+            # This file was generated from schema/chronicle_schema_v#{graph.version}.rb.
+            #
+            # Do not edit this file directly, as it will be overwritten.
+            #
+            # To generate a new version, run `rake generate`
+
+          TTL
+        end
+
+        def ontology_triple
+          RDF::Statement(RDF::URI.new('https://schema.chronicle.app'), RDF.type, RDF::OWL.Ontology)
+        end
+
+        def version_triple
+          RDF::Statement(RDF::URI.new('https://schema.chronicle.app'), RDF::OWL.versionInfo, graph.version.to_s)
+        end
+
+        def serialize_class(klass)
+          statements = []
+          statements << RDF::Statement(RDF::URI.new(klass.id), RDF.type, RDF::RDFS.Class)
+          statements << RDF::Statement(RDF::URI.new(klass.id), RDF::RDFS.comment, klass.comment) if klass.comment
+
+          klass.subtype_ids.each do |subtype_id|
+            statements << RDF::Statement(RDF::URI.new(subtype_id), RDF::RDFS.subClassOf, RDF::URI.new(klass.id))
+          end
+
+          if klass.see_also
+            statements << RDF::Statement(RDF::URI.new(klass.id), RDF::RDFS.seeAlso, RDF::URI.new(klass.see_also))
+          end
+
+          statements
+        end
+
+        def serialize_property(property)
+          statements = []
+
+          statements << RDF::Statement(RDF::URI.new(property.id), RDF.type, RDF::RDFV.Property)
+
+          property.range.each do |range|
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::URI.new("#{@graph.default_namespace}rangeIncludes"),
+              RDF::URI.new(range))
+          end
+
+          property.domain.each do |domain|
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::URI.new("#{@graph.default_namespace}domainIncludes"),
+              RDF::URI.new(domain))
+          end
+
+          if property.comment
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::RDFS.comment,
+              property.comment)
+          end
+
+          if property.see_also
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::RDFS.seeAlso, RDF::URI.new(property.see_also))
+          end
+
+          if property.required?
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::OWL.minCardinality,
+              RDF::Literal.new(1, datatype: RDF::XSD.integer))
+          end
+
+          unless property.many
+            statements << RDF::Statement(RDF::URI.new(property.id), RDF::OWL.maxCardinality,
+              RDF::Literal.new(1, datatype: RDF::XSD.integer))
+          end
+
+          statements
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/schema/rdf_parsing/schemaorg.rb

@@ -0,0 +1,50 @@
+require 'open-uri'
+require 'net/http'
+
+module Chronicle
+  module Schema
+    module RDFParsing
+      module Schemaorg
+        @memoized_graphs = {}
+
+        DEFAULT_NAMESPACE = 'https://schema.org/'.freeze
+
+        def self.graph_for_version(version)
+          @memoized_graphs[version] ||= build_graph(version)
+        end
+
+        def self.build_graph(version)
+          ttl = ttl_for_version(version)
+          Chronicle::Schema::RDFParsing::TTLGraphBuilder.build_from_ttl(ttl, default_namespace: DEFAULT_NAMESPACE)
+        end
+
+        def self.ttl_for_version(version)
+          url = url_for_version(version)
+          ttl_via_download(url)
+        end
+
+        def self.ttl_via_download(url)
+          uri = URI(url)
+          response = Net::HTTP.get_response(uri)
+          raise "Error: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
+
+          response.body
+        end
+
+        def self.seed_graph_from_file(version, file_path)
+          ttl = File.read(file_path)
+          graph = Chronicle::Schema::RDFParsing::TTLGraphBuilder.build_from_ttl(ttl,
+            default_namespace: DEFAULT_NAMESPACE)
+          @memoized_graphs[version] = graph
+        end
+
+        def self.url_for_version(version)
+          # Previously just used this one but it's missing ontologies
+          # 'https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/schema.ttl'
+
+          "https://raw.githubusercontent.com/schemaorg/schemaorg/main/data/releases/#{version}/schemaorg-all-https.ttl"
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/schema/rdf_parsing/ttl_graph_builder.rb

@@ -0,0 +1,142 @@
+require 'tsort'
+require 'rdf/turtle'
+require_relative '../schema_graph'
+require_relative '../schema_type'
+require_relative '../schema_property'
+
+module Chronicle
+  module Schema
+    module RDFParsing
+      class TTLGraphBuilder
+        attr_reader :ttl_str, :ttl_graph, :graph
+
+        def initialize(ttl_str, default_namespace: 'https://schema.org/')
+          @ttl_str = ttl_str
+          @default_namespace = default_namespace
+          @graph = Chronicle::Schema::SchemaGraph.new(default_namespace:)
+        end
+
+        def build
+          reader = RDF::Reader.for(:ttl).new(@ttl_str)
+          @ttl_graph = RDF::Graph.new << reader
+
+          @graph.version = get_version
+          @graph.types = build_type_graph
+          @graph.properties = build_property_graph
+          # build_datatype_graph
+          @graph.build_references!
+          @graph
+        end
+
+        def self.build_from_file(file_path, default_namespace:)
+          new(File.read(file_path), default_namespace:).build
+        end
+
+        def self.build_from_ttl(ttl_str, default_namespace:)
+          new(ttl_str, default_namespace:).build
+        end
+
+        private
+
+        # TODO: make this use proper schema
+        def get_version
+          @ttl_graph.query([nil, RDF::OWL.versionInfo, nil]).map(&:object).map(&:to_s).first
+        end
+
+        def build_type_graph
+          types = all_types.map do |type_id|
+            comment = comment_of_class(type_id)
+            Chronicle::Schema::SchemaType.new(type_id) do |t|
+              t.comment = comment
+              t.namespace = @default_namespace
+              t.see_also = see_also(type_id)
+            end
+          end
+
+          types.each do |schema_type|
+            schema_type.subtype_ids = subtypes_of_class(schema_type.id)
+          end
+
+          types
+        end
+
+        def build_property_graph
+          all_properties.map do |property_id|
+            Chronicle::Schema::SchemaProperty.new(property_id) do |p|
+              p.range = range_of_property(property_id)
+              p.domain = domain_of_property(property_id)
+              p.comment = comment_of_property(property_id)
+              p.required = property_required?(property_id)
+              p.many = property_many?(property_id)
+              p.namespace = @default_namespace
+              p.see_also = see_also(property_id)
+            end
+          end
+        end
+
+        def all_types
+          @ttl_graph.query([nil, RDF.type, RDF::RDFS.Class])
+            .map(&:subject)
+            .map(&:to_s)
+        end
+
+        def all_properties
+          @ttl_graph.query([nil, RDF.type, RDF::RDFV.Property])
+            .map(&:subject)
+            .map(&:to_s)
+        end
+
+        def all_datatypes; end
+
+        def subtypes_of_class(type_id)
+          @ttl_graph.query([nil, RDF::RDFS.subClassOf, RDF::URI.new(type_id)]).map(&:subject).map(&:to_s)
+        end
+
+        def parents_of_class(type_id)
+          @ttl_graph.query([RDF::URI.new(type_id), RDF::RDFS.subClassOf, nil]).map(&:object).map(&:to_s)
+        end
+
+        def comment_of_class(type_id)
+          @ttl_graph.query([RDF::URI.new(type_id), RDF::RDFS.comment, nil]).map(&:object).map(&:to_s).first
+        end
+
+        def comment_of_property(property_id)
+          @ttl_graph.query([RDF::URI.new(property_id), RDF::RDFS.comment, nil]).map(&:object).map(&:to_s).first
+        end
+
+        def range_of_property(property_id)
+          @ttl_graph.query([
+                             RDF::URI.new(property_id),
+                             RDF::URI.new("#{@default_namespace}rangeIncludes"),
+                             nil
+                           ]).map(&:object).map(&:to_s)
+        end
+
+        def domain_of_property(property_id)
+          @ttl_graph.query([
+                             RDF::URI.new(property_id),
+                             RDF::URI.new("#{@default_namespace}domainIncludes"),
+                             nil
+                           ]).map(&:object).map(&:to_s)
+        end
+
+        def property_required?(property_id)
+          min_cardinalities = @ttl_graph.query([RDF::URI.new(property_id), RDF::OWL.minCardinality,
+                                                nil]).map(&:object).map(&:to_i)
+          min_cardinalities.empty? ? false : min_cardinalities.first.positive?
+        end
+
+        def property_many?(property_id)
+          max_cardinalities = @ttl_graph.query([RDF::URI.new(property_id), RDF::OWL.maxCardinality,
+                                                nil]).map(&:object)
+
+          max_cardinalities.empty? ? true : max_cardinalities.map(&:to_i).first > 1
+        end
+
+        def see_also(id)
+          @ttl_graph.query([RDF::URI.new(id), RDF::RDFS.seeAlso, nil]).map(&:object).map(&:to_s).first
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/schema/rdf_parsing.rb

@@ -0,0 +1,11 @@
+module Chronicle
+  module Schema
+    module RDFParsing
+    end
+  end
+end
+
+require_relative 'rdf_parsing/ttl_graph_builder'
+require_relative 'rdf_parsing/graph_transformer'
+require_relative 'rdf_parsing/rdf_serializer'
+require_relative 'rdf_parsing/schemaorg'

data/lib/chronicle/schema/schema_graph.rb

@@ -0,0 +1,145 @@
+require 'tsort'
+
+module Chronicle
+  module Schema
+    # Represents a RDF graph as a DAG of types and their properties, built
+    # from a TTL string
+    class SchemaGraph
+      include TSort
+
+      attr_accessor :types, :properties, :datatypes, :version, :default_namespace
+
+      def initialize(default_namespace: 'https://schema.chronicle.app/')
+        @default_namespace = default_namespace
+        @types = []
+        @properties = []
+        @datatypes = []
+      end
+
+      def self.build_from_json(json)
+        graph = new
+        graph.version = json['version']
+        json['types'].each do |type_data|
+          id = graph.id_to_identifier(type_data['id'])
+          graph.add_type(id).tap do |klass|
+            klass.comment = type_data['comment']
+            klass.subtype_ids = type_data['subtype_ids']
+          end
+        end
+        json['properties'].each do |property_data|
+          id = graph.id_to_identifier(property_data['id'])
+          graph.add_property(id).tap do |property|
+            property.comment = property_data['comment']
+            property.domain = property_data['domain']
+            property.range = property_data['range']
+            property.many = property_data['many']
+            property.required = property_data['required']
+          end
+        end
+        graph.build_references!
+        graph
+      end
+
+      def pretty_print(pp)
+        pp.text('SchemaGraph')
+        pp.nest(2) do
+          pp.breakable
+          pp.text("Num types: #{types.size}")
+        end
+      end
+
+      def inspect
+        "#<SchemaGraph:#{object_id}>"
+      end
+
+      def to_h
+        {
+          version: @version,
+          types: types.map(&:to_h),
+          properties: properties.map(&:to_h)
+        }
+      end
+
+      def build_references!
+        @types.each do |klass|
+          klass.subtypes = klass.subtype_ids.map { |id| find_type_by_id(id) }
+          klass.supertypes = @types.select { |c| c.subtype_ids.include?(klass.id) }
+        end
+
+        @properties.each do |property|
+          property.domain.each do |type_id|
+            klass = find_type_by_id(type_id)
+            klass.properties << property if klass
+          end
+
+          # prune unknown range values from property
+          property.range = property.range.select do |range|
+            find_type_by_id(range)
+          end
+
+          property.range_types = property.range.map { |id| find_type_by_id(id) }
+        end
+      end
+
+      def find_type_by_id(id)
+        @types.find { |c| c.id == id }
+      end
+
+      def find_type(identifier)
+        @types.find { |c| c.id == identifier_to_uri(identifier) }
+      end
+
+      def find_property(identifier)
+        @properties.find { |p| p.id == identifier_to_uri(identifier) }
+      end
+
+      def find_property_by_id(id)
+        @properties.find { |p| p.id == id }
+      end
+
+      def add_type(identifier)
+        find_type(identifier) || add_new_type(identifier)
+      end
+
+      def add_property(identifier)
+        find_property(identifier) || add_new_property(identifier)
+      end
+
+      def id_to_identifier(id)
+        id.gsub(@default_namespace, '')
+      end
+
+      def identifier_to_uri(identifier)
+        "#{@default_namespace}#{identifier}"
+      end
+
+      private
+
+      def add_new_type(identifier)
+        new_type = SchemaType.new(identifier_to_uri(identifier)) do |t|
+          t.namespace = @default_namespace
+        end
+
+        @types << new_type unless @types.include?(new_type)
+        new_type
+      end
+
+      def add_new_property(identifier)
+        new_property = SchemaProperty.new(identifier_to_uri(identifier))
+        @properties << new_property unless @properties.include?(new_property)
+        new_property
+      end
+
+      def tsort_each_node(&block)
+        @types.each do |node|
+          block.call(node)
+        end
+      end
+
+      def tsort_each_child(node, &)
+        puts "tsort_each_child called with node: #{node&.id}"
+        node&.subtypes&.each(&)
+      end
+    end
+  end
+end

data/lib/chronicle/schema/schema_property.rb

@@ -0,0 +1,81 @@
+module Chronicle
+  module Schema
+    # Represents a property in the RDF graph
+    class SchemaProperty
+      attr_reader :id
+      attr_accessor :domain,
+        :range,
+        :comment,
+        :many,
+        :required,
+        :namespace,
+        :range_types, # FIXME
+        :see_also
+
+      def initialize(id)
+        @id = id
+        @domain = []
+        @range = []
+        @many = false
+        @required = false
+
+        yield self if block_given?
+      end
+
+      def pretty_print(pp)
+        pp.text("SchemaProperty: #{id}")
+        pp.nest(2) do
+          pp.breakable
+          pp.text("Range: #{range}")
+          pp.breakable
+          pp.text("Domain: #{domain}")
+        end
+      end
+
+      def to_h
+        output = {
+          id:,
+          domain:,
+          range:,
+          many: @many,
+          required: @required
+        }
+        output[:comment] = @comment if @comment
+        output[:see_also] = @see_also if @see_also
+        output
+      end
+
+      def ==(other)
+        id == other.id
+      end
+
+      def required?
+        @required
+      end
+
+      def many?
+        @many
+      end
+
+      def identifier
+        @id.split('/').last&.to_sym
+      end
+
+      # FIXME
+      def id_snakecase
+        @id.split('/').last.gsub(/([a-z\d])([A-Z])/, '\1_\2').downcase.to_sym
+      end
+
+      # FIXME: refactor this and the next
+      def range_identifiers
+        range.map do |r|
+          r.split('/').last&.to_sym
+        end
+      end
+
+      def full_range_identifiers
+        range_types.map(&:descendants).flatten.map { |x| x.id.split('/').last&.to_sym } + range_identifiers
+      end
+    end
+  end
+end