chronicle-core 0.2.2 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed6ae5d478596b65ee33d68c10862a87f351dd7711e49e9ac7b15237e878fbea
-  data.tar.gz: 33ed98e503065a2d5dfe21f4e6e551d3c6ba30dfcf052d0e621bf5d9750519be
+  metadata.gz: d953db1cde445a4dc7c887226864715ccc5dffd745fc3d04b2c463a54ecbc40a
+  data.tar.gz: f523823b5b681df1b278532b73cb36e89bcf1ec3a114bd8c9b680f6dfb4c97b3
 SHA512:
-  metadata.gz: ee6ff44d1b2318451073b38d38db1e015edf19d7df70c61f5e15406ff00483eeb7d0d131173fa61e233c8cf782d3a0345f12fdf9bcea31187a10a2143585a9b7
-  data.tar.gz: 4afd04abcd28e6a8792c8f947ad97473a452fe792e3ed4e1bac6d2e99f7fe4cfb5b614d7589d2e2d10dcd3684285c75a8510bf061d73cfcac976519ea02eaa66
+  metadata.gz: 1b332fe113f5db60c318aad8b28d22ddc95dfa2b6e7afc6ee4ffd15521c56282690a48a083b1ca1518c2c4f4c38afc015f95fd664ae15a9bc2bdd8de8538ff5c
+  data.tar.gz: '09e32019bd267e92d0ab873f3b098595fc82c40ae4c7ba89525112d89b428adb8ca66d1830cbc5d8caa83d87839e86a5fcef7d341e7f71b9a9c431715ca32d9f'

data/.github/workflows/rspec.yml

@@ -13,7 +13,7 @@ jobs:
 
     strategy:
       matrix:
-        ruby_version: ['2.7', '3.0']
+        ruby_version: ['3.1']
       fail-fast: false
 
     steps:

data/.gitignore

@@ -8,4 +8,6 @@
 /tmp/
 
 Gemfile.lock
-.DS_Store
+.DS_Store
+
+.ruby-version

data/.rubocop-plugin.yml

@@ -0,0 +1,4 @@
+inherit_from: ./.rubocop.yml
+
+Style/Documentation:
+  Enabled: false

data/.rubocop.yml

@@ -1,5 +1,19 @@
+AllCops:
+  NewCops: enable
+  TargetRubyVersion: 3.1
+  SuggestExtensions: false
+
 Style/FrozenStringLiteralComment:
   Enabled: false
 
-StringLiterals:
-  Enabled: false
+Metrics:
+  Enabled: false
+
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+Layout/MultilineMethodCallIndentation:
+  EnforcedStyle: indented
+
+Gemspec/DevelopmentDependencies:
+  EnforcedStyle: gemspec

data/Gemfile

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-source "https://rubygems.org"
+source 'https://rubygems.org'
 
 # Specify your gem's dependencies in chronicle-core.gemspec
 gemspec
 
-gem "rake", "~> 13.0"
+gem 'rake', '~> 13.0'

data/Guardfile

@@ -1,7 +1,7 @@
-guard :rspec, cmd: "bundle exec rspec" do
-  require "guard/rspec/dsl"
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
 
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')  { "spec" }
+  watch('spec/spec_helper.rb')  { 'spec' }
 end

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2023 Andrew Louis
+Copyright (c) 2024 Andrew Louis
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,3 +1,88 @@
-# Chronicle::Core
+[![Gem Version](https://badge.fury.io/rb/chronicle-core.svg)](https://badge.fury.io/rb/chronicle-core) [![Ruby](https://github.com/chronicle-app/chronicle-core/actions/workflows/rspec.yml/badge.svg)](https://github.com/chronicle-app/chronicle-core/actions/workflows/rspec.yml) [![Docs](https://img.shields.io/badge/docs-rubydoc.info-blue)](https://www.rubydoc.info/gems/chronicle-core/)
+
+Core schema-related code for [Chronicle](https://github.com/chronicle-app/)
+
+## Schema
+
+Chronicle Schema is a set of types and properties that are used to describe personal digital history. The schema is a derivative of [Schema.org](https://schema.org). For the most part, it is a subset of Schema.org types and properties. 
+
+There are a few top-level properties that are unique to Chronicle:
+- `source` - the source of the record (e.g. `spotify`, `twitter`, `instagram`)
+- `sourceId` - the unique identifier of the record in the source system
+- `slug` - a human-readable identifier for the record (e.g. `bridge-over-troubled-water`)
+
+To view all valid types and properties in Chronicle Schema, see the `schema/` diretory.
+
+TODO: generate doc files from schema automatically.
+
+To update the schema, edit `schema/chronicle_schema_VERSION.rb` and run `rake schema:generate`.
+
+## Models
+
+For each valid type in Chronicle Schema, there is a corresponding model class that can instantiated as an immutable Ruby object.
+
+### Usage
+
+```ruby
+require 'chronicle/models'
+
+song = Chronicle::Models::MusicRecording.new do |r|
+  r.name = 'Bridge Over Troubled Water'
+  r.in_album = 'Aretha Live at Fillmore West'
+  r.by_artist = [Chronicle::Models::MusicGroup.new(name: 'Aretha Franklin')]
+  r.duration = "PT5M49S"
+  r.source = 'spotify'
+  r.source_id = '0e0isrwFsu5W0KJqxkfPpX?si=3d2f08d6d63149bb'
+end
+
+puts song.to_json
+# => {"@type":"MusicRecording","name":"Bridge Over Troubled Water","in_album":"Aretha Live at Fillmore West","by_artist":[{"@type":"MusicGroup","name":"Aretha Franklin"}],"duration":"PT5M49S","source":"spotify","sourceId":"0e0isrwFsu5W0KJqxkfPpX?si=3d2f08d6d63149bb"}
+```
+
+## Validation
+
+chronicle-core provides a validator that checks if a given JSON object conforms to the Chronicle Schema. The validator assumes the JSON is serialized as JSON-LD.
+
+### Usage
+
+```ruby
+# TODO
+```
+
+## Serialization
+
+chronicle-core provides the following serializers for records:
+- JSON
+- JSONAPI
+- JSON-LD
+
+## Development
+
+To run the tests:
+
+```bash
+# Run once
+bundle exec rspec
+
+# Run continuously
+bundle exec guard
+```
+
+## Get in touch
+
+- [@hyfen](https://twitter.com/hyfen) on Twitter
+- [@hyfen](https://github.com/hyfen) on Github
+- Email: andrew@hyfen.net
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/chronicle-app/chronicle-core. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Chronicle::ETL project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/chronicle-app/chronicle-core/blob/main/CODE_OF_CONDUCT.md).
 
-Home of some shared code for [Chronicle](https://github.com/chronicle-app/). Currently, this means the models and a validator for Chronicle Schema.

data/Rakefile

@@ -1,4 +1,66 @@
 # frozen_string_literal: true
 
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
 task default: %i[]
+
+namespace :generate do
+  DEFAULT_SCHEMA_FILE = File.join(File.dirname(__FILE__), 'lib', 'chronicle', 'schema', 'data', 'schema.ttl')
+
+  desc 'Generate everything'
+  task :all, :version do |_t, args|
+    version = args[:version] || latest_version
+    Rake::Task['generate:generate'].invoke(version)
+    Rake::Task['generate:cache_schema'].invoke(version)
+  end
+
+  desc 'Generate schema'
+  task :generate, :version do |_t, args|
+    require 'pry'
+    require 'chronicle/schema'
+    require 'chronicle/schema/rdf_parsing'
+    require 'json'
+
+    version = args[:version] || latest_version
+    puts "Generating schema with version: #{version}"
+
+    schema_dsl_file = File.join(File.dirname(__FILE__), 'schema', "chronicle_schema_v#{version}.rb")
+
+    new_graph = Chronicle::Schema::RDFParsing::GraphTransformer.transform_from_file(schema_dsl_file)
+
+    ttl_str = Chronicle::Schema::RDFParsing::RDFSerializer.serialize(new_graph)
+    output_filename = schema_dsl_file.gsub(/\.rb$/, '.ttl')
+
+    File.write(output_filename, ttl_str)
+  end
+
+  desc 'Cache schema from ttl file'
+  task :cache_schema, :version do |_t, args|
+    require 'chronicle/schema'
+    require 'chronicle/schema/rdf_parsing'
+
+    version = args[:version] || latest_version
+    puts "Caching schema with version: #{version}"
+
+    ttl_file = File.join(File.dirname(__FILE__), 'schema', "chronicle_schema_v#{version}.ttl")
+    graph = Chronicle::Schema::RDFParsing::TTLGraphBuilder.build_from_file(ttl_file, default_namespace: 'https://schema.chronicle.app/')
+
+    output_filename = File.join(File.dirname(__FILE__), 'schema',
+      "chronicle_schema_v#{version}.json")
+
+    output_str = JSON.pretty_generate(graph.to_h)
+
+    File.write(output_filename, output_str)
+  end
+
+  # Highest version schema file in schema/ directory
+  def latest_version
+    schema_files = Dir.glob(File.join(File.dirname(__FILE__), 'schema', 'chronicle_schema_v*.rb'))
+    versions = schema_files.map { |f| f.match(/chronicle_schema_v(.*)\.rb/)&.captures&.first }
+    sorted_versions = versions.sort_by { |version| Gem::Version.new(version) }
+    sorted_versions.last
+  end
+end
+
+task :generate, :version do |_t, args|
+  Rake::Task['generate:all'].invoke(args[:version])
+end

data/bin/console

@@ -1,15 +1,15 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "bundler/setup"
-require "chronicle/core"
+require 'bundler/setup'
+require 'chronicle/core'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
 # (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
+require 'pry'
+Pry.start
 
-require "irb"
-IRB.start(__FILE__)
+# require "irb"
+# IRB.start(__FILE__)

data/chronicle-core.gemspec

@@ -1,42 +1,48 @@
 # frozen_string_literal: true
 
-require_relative "lib/chronicle/core/version"
+require_relative 'lib/chronicle/core/version'
 
 Gem::Specification.new do |spec|
-  spec.name = "chronicle-core"
+  spec.name = 'chronicle-core'
   spec.version = Chronicle::Core::VERSION
-  spec.authors = ["Andrew Louis"]
-  spec.email = ["andrew@hyfen.net"]
+  spec.authors = ['Andrew Louis']
+  spec.email = ['andrew@hyfen.net']
 
-  spec.summary       = "Core libraries for Chronicle"
-  spec.description   = "Core libraries for Chronicle including models and schema definitions."
-  spec.homepage      = "https://github.com/chronicle-app"
-  spec.license       = "MIT"
-  spec.required_ruby_version = ">= 2.7"
+  spec.summary       = 'Core libraries for Chronicle'
+  spec.description   = 'Core libraries for Chronicle including models and schema definitions.'
+  spec.homepage      = 'https://github.com/chronicle-app'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 3.1'
 
   if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
 
-    spec.metadata["homepage_uri"] = spec.homepage
-    spec.metadata["source_code_uri"] = "https://github.com/chronicle-app/chronicle-core"
-    spec.metadata["changelog_uri"] = "https://github.com/chronicle-app/chronicle-core/releases"
+    spec.metadata['homepage_uri'] = spec.homepage
+    spec.metadata['source_code_uri'] = 'https://github.com/chronicle-app/chronicle-core'
+    spec.metadata['changelog_uri'] = 'https://github.com/chronicle-app/chronicle-core/releases'
   else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+          'public gem pushes.'
   end
 
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir = "exe"
+  spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency "dry-schema", "~> 1.13"
-
-  spec.add_development_dependency "guard-rspec", "~> 4.7.3"
-  spec.add_development_dependency "pry-byebug", "~> 3.9"
-  spec.add_development_dependency "rubocop", "~> 1.25.1"
-  spec.add_development_dependency "rspec", "~> 3.9"
-  spec.add_development_dependency "simplecov", "~> 0.21"
+  spec.require_paths = ['lib']
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.add_dependency 'dry-struct', '~> 1.6'
+  spec.add_dependency 'dry-validation', '~> 1.10'
+
+  spec.add_development_dependency 'guard-rspec', '~> 4.7.3'
+  spec.add_development_dependency 'pry-byebug', '~> 3.9'
+  spec.add_development_dependency 'rake', '~> 13'
+  spec.add_development_dependency 'rdf-reasoner', '~> 0.9'
+  spec.add_development_dependency 'rdf-turtle', '~> 3.3'
+  spec.add_development_dependency 'rspec', '~> 3.9'
+  spec.add_development_dependency 'rubocop', '~> 1.57'
+  spec.add_development_dependency 'simplecov', '~> 0.21'
+  spec.add_development_dependency 'sparql', '~> 3.3'
 end

data/lib/chronicle/core/version.rb

@@ -1,7 +1,5 @@
-# frozen_string_literal: true
-
 module Chronicle
   module Core
-    VERSION = "0.2.2"
+    VERSION = '0.3.1'.freeze
   end
 end

data/lib/chronicle/core.rb

@@ -1,6 +1,4 @@
-# frozen_string_literal: true
-
-require_relative "core/version"
+require_relative 'core/version'
 
 begin
   require 'pry'

data/lib/chronicle/models/base.rb

@@ -0,0 +1,96 @@
+require 'dry-struct'
+require 'chronicle/schema/types'
+
+module Chronicle
+  module Models
+    # The base class for all generated models
+    class Base < Dry::Struct
+      transform_keys(&:to_sym)
+      schema schema.strict
+
+      class << self
+        attr_reader :type_id
+      end
+
+      def properties
+        # FIXME: think about this more. Does dedupe belong in serialization
+        attributes.except(:dedupe_on)
+      end
+
+      # TODO: remove this from attributes and just do custom getters/setters
+      attribute(:id, Chronicle::Schema::Types::String.optional.default(nil).meta(many: false, required: false))
+
+      # set of properties to dedupe on
+      # each set of properties is an array of symbols representing the properties to dedupe on
+      # example: [[:slug, :source], [:url]]
+      attribute(
+        :dedupe_on,
+        Chronicle::Schema::Types::Array.of(
+          Chronicle::Schema::Types::Array.of(
+            Chronicle::Schema::Types::Symbol
+          )
+        ).optional.default([].freeze).meta(
+          many: false, required: false
+        )
+      )
+
+      def self.new(attributes = {})
+        if block_given?
+          attribute_struct = Struct.new(*schema.map(&:name)).new
+          attribute_struct.dedupe_on = []
+
+          yield(attribute_struct)
+          attributes = attribute_struct.to_h
+        end
+
+        super(attributes)
+      rescue Dry::Struct::Error, NoMethodError => e
+        # TODO: be more clear about the attribute that's invalid
+        raise AttributeError, e.message
+      end
+
+      def meta
+        output = {}
+        output[:dedupe_on] = dedupe_on unless dedupe_on.empty?
+        output
+      end
+
+      def type_id
+        self.class.type_id
+      end
+
+      def to_h
+        super.merge({ type: type_id })
+          .except(:dedupe_on)
+          .compact # TODO: don't know about this one
+      end
+
+      # TODO: exclude dedupe_on from serialization
+      def to_h_flattened
+        require 'chronicle/utils/hash_utils'
+        Chronicle::Utils::HashUtils.flatten_hash(to_h)
+      end
+
+      # FIXME: this isn't working
+      def self.many_cardinality_attributes
+        schema.type.select do |type|
+          type.meta[:many]
+        end.map(&:name)
+      end
+
+      def self.one_cardinality_attributes
+        schema.type.map(&:name) - many_cardinality_attributes
+      end
+    end
+
+    def self.schema_type(types)
+      Chronicle::Schema::Types::Instance(Chronicle::Models::Base).constructor do |input|
+        unless input.type_id && [types].flatten.include?(input.type_id)
+          raise Dry::Types::ConstraintError.new(:type?, input)
+        end
+
+        input
+      end
+    end
+  end
+end

data/lib/chronicle/models/builder.rb

@@ -0,0 +1,35 @@
+module Chronicle
+  module Models
+    # Utility methods to build models from hash input
+    module Builder
+      # Build a nested a model with nested associated from a hash in json-ld format
+      def build(obj)
+        raise ArgumentError, 'Object must be a hash' unless obj.is_a?(Hash)
+
+        type = obj[:@type]
+        raise ArgumentError, 'Object must have a type' unless type
+
+        model_klass = const_get(type.to_sym)
+        raise ArgumentError, "Unknown model type: #{type}" unless model_klass
+
+        # recursively create nested models
+        # TODO: have a better way of detecting chronicle schema objects
+        begin
+          obj.each do |property, value|
+            if value.is_a?(Hash)
+              obj[property] = build(value)
+            elsif value.is_a?(Array)
+              obj[property] = value.map do |v|
+                v.is_a?(Hash) ? build(v) : v
+              end
+            end
+          end
+
+          model_klass.new(obj.except(:@type))
+        rescue Chronicle::Models::AttributeError => e
+          raise ArgumentError, e.message
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/models/generation.rb

@@ -0,0 +1,89 @@
+require 'chronicle/schema'
+require_relative 'model_factory'
+require_relative 'base'
+
+module Chronicle
+  module Models
+    module Generation
+      @models_generated = false
+      @benchmark_enabled = false
+
+      class << self
+        attr_accessor :models_generated, :benchmark_enabled
+      end
+
+      def self.included(base)
+        base.extend ClassMethods
+        base.generate_models unless base.models_generated?
+      end
+
+      # Methods for generating models from the schema
+      module ClassMethods
+        def models_generated?
+          Chronicle::Models::Generation.models_generated
+        end
+
+        def extant_models
+          constants.select do |constant_name|
+            constant = const_get(constant_name)
+            constant.is_a?(Class) && constant.superclass == Chronicle::Models::Base
+          end
+        end
+
+        def generate_models(graph = nil)
+          graph ||= begin
+            require 'json'
+            schema_path = File.join(File.dirname(__FILE__), '..', '..', '..', 'schema', 'chronicle_schema_v1.json')
+            Chronicle::Schema::SchemaGraph.build_from_json(JSON.parse(File.read(schema_path)))
+          end
+
+          start_time = Time.now
+          graph.types.each do |klass|
+            type_id = graph.id_to_identifier(klass.id)
+
+            new_model_klass = Chronicle::Models::ModelFactory.new(
+              type_id: type_id.to_sym,
+              properties: klass.all_properties
+            ).generate
+
+            const_set(type_id, new_model_klass)
+          end
+          end_time = Time.now
+          duration_ms = (end_time - start_time) * 1000
+          handle_benchmark_data(graph.types.length, duration_ms) if Chronicle::Models::Generation.benchmark_enabled
+
+          Chronicle::Models::Generation.models_generated = true
+        end
+
+        def unload_models
+          constants.each do |constant_name|
+            constant = const_get(constant_name)
+            if constant.is_a?(Class) && constant.superclass == Chronicle::Models::Base
+              send(:remove_const,
+                constant_name)
+            end
+          end
+          Chronicle::Models::Generation.models_generated = false
+        end
+
+        def enable_benchmarking
+          Chronicle::Models::Generation.benchmark_enabled = true
+        end
+
+        private
+
+        def handle_benchmark_data(number_of_models, duration)
+          puts "Generated #{number_of_models} models in #{duration.round(2)}ms"
+        end
+      end
+
+      def self.reset
+        Chronicle::Models::Generation.models_generated = false
+      end
+
+      def self.suppress_model_generation
+        Chronicle::Models::Generation.models_generated = true
+      end
+    end
+  end
+end

data/lib/chronicle/models/model_factory.rb

@@ -0,0 +1,65 @@
+module Chronicle
+  module Models
+    class ModelFactory
+      attr_reader :id
+
+      def initialize(type_id:, properties: [])
+        @type_id = type_id
+        @properties = properties
+      end
+
+      def generate
+        attribute_info = @properties.map do |property|
+          generate_attribute_info(property)
+        end
+
+        type_id = @type_id
+
+        Class.new(Chronicle::Models::Base) do
+          attribute_info.each do |a|
+            attribute(a[:name], a[:type])
+          end
+
+          @type_id = type_id
+        end
+      end
+
+      private
+
+      def generate_attribute_info(property)
+        range = build_type(property)
+        type = if property.many?
+                 Chronicle::Schema::Types::Array.of(range)
+               else
+                 range
+               end
+
+        type = type.optional.default(nil) unless property.required?
+        type = type.meta(required: property.required?, many: property.many?)
+
+        {
+          name: property.id_snakecase.to_sym,
+          type:
+        }
+      end
+
+      def build_type(property)
+        type_values = []
+
+        full_range_identifiers = property.full_range_identifiers
+
+        type_values << Chronicle::Schema::Types::Params::Time if full_range_identifiers.include? :DateTime
+        type_values << Chronicle::Schema::Types::String if %i[Text URL Distance Duration Energy
+                                                              Mass].intersect?(full_range_identifiers)
+
+        type_values << Chronicle::Schema::Types::Params::Float if full_range_identifiers.include? :Float
+        type_values << Chronicle::Schema::Types::Params::Integer if full_range_identifiers.include? :Integer
+        type_values << Chronicle::Models.schema_type(full_range_identifiers)
+
+        type_values.reduce do |memo, type|
+          memo | type
+        end
+      end
+    end
+  end
+end

data/lib/chronicle/models.rb

@@ -0,0 +1,17 @@
+require_relative 'models/generation'
+require_relative 'models/base'
+require_relative 'models/builder'
+
+module Chronicle
+  module Models
+    class Error < StandardError; end
+    class AttributeError < Error; end
+
+    # Automatically generate models from the default schema
+    # We will do this dynamically whenever 'chronicle/models' is required
+    # until the performance becomes an issue (currently takes about ~10ms)
+    include Chronicle::Models::Generation
+
+    extend Chronicle::Models::Builder
+  end
+end