glossarist 2.8.18 → 2.9.0

data/lib/glossarist/cli/export_command.rb

@@ -3,8 +3,12 @@
 module Glossarist
   class CLI
     class ExportCommand
+      AGGREGATE_FORMATS = %w[jsonld turtle tbx jsonl].freeze
+      PER_CONCEPT_FORMATS = %w[json jsonld turtle yaml].freeze
+
       EXTENSIONS = {
         "json" => "json",
+        "yaml" => "yaml",
         **GcrPackage::COMPILED_EXTENSIONS,
       }.freeze
 
@@ -14,20 +18,32 @@ module Glossarist
       end
 
       def run
-        format = @options[:format]
+        formats = parse_formats(@options[:format])
         output_dir = File.expand_path(@options[:output])
         FileUtils.mkdir_p(output_dir)
 
         concepts = load_concepts
         name = resolve_shortname
 
-        case format
-        when "json" then export_json(concepts, output_dir)
-        when "jsonld" then export_jsonld(concepts, name, output_dir)
-        when "turtle" then export_turtle(concepts, name, output_dir)
-        when "tbx" then export_tbx(concepts, name, output_dir)
-        when "jsonl" then export_jsonl(concepts, name, output_dir)
+        formats.each do |format|
+          if per_concept? && per_concept_supported?(format)
+            export_per_concept(concepts, format, output_dir)
+          end
+
+          if aggregate?
+            export_aggregate(format, concepts, name, output_dir)
+          elsif per_concept? && !per_concept_supported?(format)
+            raise ArgumentError,
+                  format(
+                    "Per-concept export is not supported for '%<format>s' " \
+                    "(only aggregate). Drop --per-concept or pick one of: %<formats>s",
+                    format: format,
+                    formats: PER_CONCEPT_FORMATS.join(", "),
+                  )
+          end
         end
+
+        validate_outputs!(formats, output_dir, name) if validate?
       rescue ArgumentError => e
         warn "Error: #{e.message}"
         exit 1
@@ -63,6 +79,51 @@ module Glossarist
         }.compact
       end
 
+      def parse_formats(raw)
+        Array(raw).flat_map { |value| value.to_s.split(",") }
+          .map(&:strip).reject(&:empty?)
+          .each { |f| validate_format!(f) }
+      end
+
+      def validate_format!(format)
+        return if EXTENSIONS.key?(format)
+
+        raise ArgumentError,
+              format("Unknown format '%<format>s'. Valid formats: %<valid>s",
+                     format: format,
+                     valid: EXTENSIONS.keys.join(", "))
+      end
+
+      def per_concept?
+        @options.fetch(:per_concept, false)
+      end
+
+      def aggregate?
+        !per_concept? || @options.fetch(:aggregate, false)
+      end
+
+      def validate?
+        @options.fetch(:validate, false)
+      end
+
+      def per_concept_supported?(format)
+        PER_CONCEPT_FORMATS.include?(format)
+      end
+
+      def aggregate_supported?(format)
+        AGGREGATE_FORMATS.include?(format) || format == "json"
+      end
+
+      def export_aggregate(format, concepts, name, output_dir)
+        case format
+        when "json" then export_json(concepts, output_dir)
+        when "jsonld" then export_jsonld(concepts, name, output_dir)
+        when "turtle" then export_turtle(concepts, name, output_dir)
+        when "tbx" then export_tbx(concepts, name, output_dir)
+        when "jsonl" then export_jsonl(concepts, name, output_dir)
+        end
+      end
+
       def export_json(concepts, output_dir)
         concepts.each do |concept|
           id = concept.data&.id || concept.identifier
@@ -104,6 +165,68 @@ module Glossarist
           end
         end
       end
+
+      def export_per_concept(concepts, format, output_dir)
+        dir = File.join(output_dir, "concepts")
+        FileUtils.mkdir_p(dir)
+
+        case format
+        when "json"  then per_concept_json(concepts, dir)
+        when "yaml"  then per_concept_yaml(concepts, dir)
+        when "jsonld" then per_concept_jsonld(concepts, dir)
+        when "turtle" then per_concept_turtle(concepts, dir)
+        end
+      end
+
+      def per_concept_json(concepts, dir)
+        concepts.each do |concept|
+          id = concept.data&.id || concept.identifier
+          File.write(File.join(dir, "#{id}.json"), concept.to_json)
+        end
+      end
+
+      def per_concept_yaml(concepts, dir)
+        concepts.each do |concept|
+          id = concept.data&.id || concept.identifier
+          File.write(File.join(dir, "#{id}.yaml"), concept.to_yaml)
+        end
+      end
+
+      def per_concept_jsonld(concepts, dir)
+        require "glossarist/transforms/concept_to_gloss_transform"
+        concepts.each do |concept|
+          id = concept.data&.id || concept.identifier
+          transform = Transforms::ConceptToGlossTransform.new(concept,
+                                                              transform_options)
+          File.write(File.join(dir, "#{id}.jsonld"), transform.to_jsonld)
+        end
+      end
+
+      def per_concept_turtle(concepts, dir)
+        require "glossarist/transforms/concept_to_gloss_transform"
+        concepts.each do |concept|
+          id = concept.data&.id || concept.identifier
+          transform = Transforms::ConceptToGlossTransform.new(concept,
+                                                              transform_options)
+          File.write(File.join(dir, "#{id}.ttl"), transform.to_turtle)
+        end
+      end
+
+      def validate_outputs!(formats, output_dir, name)
+        return unless formats.include?("turtle")
+
+        require "glossarist/validation/shacl_validator"
+        shapes_path = @options[:shapes]
+        validator = Validation::ShaclValidator.new(shapes_path:)
+        files = []
+        files << File.join(output_dir, "#{name}.ttl") if File.exist?(File.join(output_dir, "#{name}.ttl"))
+        files.concat(Dir.glob(File.join(output_dir, "concepts", "*.ttl")))
+        report = validator.validate_files(files)
+        unless report.conformant?
+          warn report.to_s
+          exit 1
+        end
+      end
     end
   end
 end

data/lib/glossarist/cli.rb

@@ -105,8 +105,7 @@ module Glossarist
 
     desc "export PATH", "Export concepts in machine-readable formats"
     option :format, type: :string, required: true,
-                    enum: %w[json jsonld turtle tbx jsonl],
-                    desc: "Output format"
+                    desc: "Output format(s), comma-separated (json, jsonld, turtle, tbx, jsonl, yaml)"
     option :output, aliases: :o, type: :string, required: true,
                     desc: "Output directory"
     option :shortname, type: :string,
@@ -117,6 +116,16 @@ module Glossarist
                       desc: "Base URL of the glossarist site"
     option :title, type: :string,
                    desc: "Dataset title for document header"
+    option :per_concept, type: :boolean, default: false,
+                         desc: "Emit one file per concept (in concepts/ subdirectory)"
+    option :aggregate, type: :boolean, default: nil,
+                       desc: "Emit aggregate files (default: true unless --per-concept only)"
+    option :validate, type: :boolean, default: false,
+                      desc: "Run SHACL validation on every emitted .ttl; fail on violation"
+    option :shapes, type: :string,
+                    desc: "Path to SHACL shapes .ttl (default: bundled concept-model shapes)"
+    option :context, type: :string,
+                     desc: "Path to custom JSON-LD context (default: concept-model context)"
     def export(path)
       CLI::ExportCommand.new(path, options).run
     end

data/lib/glossarist/concept_document.rb

@@ -35,5 +35,18 @@ module Glossarist
       localizations.each { |l10n| mc.add_localization(l10n) }
       mc
     end
+
+    # Set concept.uuid from the document's record key (+id+) when the YAML
+    # stream did not provide one. The YAML is the source of truth for the
+    # UUID; +id+ is a layout concern — for the grouped layout it is the
+    # filename stem, which may be a clause identifier (e.g. 3.1.1.1.yaml)
+    # rather than a UUID. Both the load path (ConceptStore#load_glossary)
+    # and the round-trip path (ConceptDocumentSerializer#deserialize) call
+    # this so the rule lives in one place.
+    def ensure_concept_uuid!
+      return unless concept && id
+
+      concept.uuid ||= id
+    end
   end
 end

data/lib/glossarist/concept_reference.rb

@@ -7,7 +7,13 @@ module Glossarist
   # Local references use +concept_id+ without +source+. External references
   # use +source+ (a registry URN prefix) and +concept_id+ to identify the
   # target concept, or a direct +urn+ field for formal URN references.
+  #
+  # Includes the {Reference} protocol. Overrides +cite?+, +local?+, and
+  # +external?+ because ConceptReference is the only reference kind whose
+  # predicates depend on runtime state (ref_type, source).
   class ConceptReference < Lutaml::Model::Serializable
+    include Reference
+
     attribute :term, :string
     attribute :concept_id, :string
     attribute :source, :string

data/lib/glossarist/concept_store.rb

@@ -15,8 +15,8 @@ module Glossarist
       def deserialize(data, model_class)
         doc = model_class.from_yamls(data["_yamls"])
         doc.id = data["_id"]
+        doc.ensure_concept_uuid!
         concept = doc.concept
-        concept.uuid = doc.id if doc.id && concept
         doc.localizations.each { |l10n| concept&.add_localization(l10n) }
         doc
       end
@@ -37,8 +37,7 @@ module Glossarist
       )
 
       documents.each do |doc|
-        concept = doc.concept
-        concept.uuid = doc.id
+        doc.ensure_concept_uuid!
         db.save(doc)
       end
 

data/lib/glossarist/dataset_register.rb

@@ -123,7 +123,7 @@ module Glossarist
     end
 
     def explicit_section_ids(concept)
-      domains = concept.respond_to?(:data) ? concept.data&.domains : nil
+      domains = concept.is_a?(ManagedConcept) ? concept.data&.domains : nil
       Array(domains).select { |d| d.ref_type == "section" }
         .filter_map(&:concept_id)
     end
@@ -133,7 +133,7 @@ module Glossarist
     # longest registered section prefix.
     # @example "103-01-01" with section "103" registered → ["103"]
     def derive_section_ids_from_id(concept)
-      concept_id = concept.respond_to?(:data) ? concept.data&.id : nil
+      concept_id = concept.is_a?(ManagedConcept) ? concept.data&.id : nil
       return [] unless concept_id
 
       all_section_ids = collect_all_section_ids

data/lib/glossarist/figure_reference.rb

@@ -10,9 +10,5 @@ module Glossarist
       map :entity_id, to: :entity_id
       map :display, to: :display
     end
-
-    def dedup_key
-      [self.class.name, entity_id]
-    end
   end
 end

data/lib/glossarist/formula_reference.rb

@@ -10,9 +10,5 @@ module Glossarist
       map :entity_id, to: :entity_id
       map :display, to: :display
     end
-
-    def dedup_key
-      [self.class.name, entity_id]
-    end
   end
 end

data/lib/glossarist/non_verbal_reference.rb

@@ -7,7 +7,13 @@ module Glossarist
   # entity ID and an optional display override. They are produced both by
   # structural arrays (`figures: [id]` on ManagedConceptData) and by inline
   # mentions (`{{fig:id}}` in text).
+  #
+  # Includes the {Reference} protocol so mixed-collection validation rules
+  # can iterate these alongside ConceptReference / BibliographicReference /
+  # AssetReference without type-checking. All predicates default to false.
   class NonVerbalReference < Lutaml::Model::Serializable
+    include Reference
+
     attribute :entity_id, :string
     attribute :display, :string
 
@@ -19,5 +25,9 @@ module Glossarist
         display: hash["display"] || hash[:display],
       )
     end
+
+    def dedup_key
+      [self.class.name, entity_id]
+    end
   end
 end

data/lib/glossarist/rdf/gloss_detailed_definition.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "digest"
 require "lutaml/model"
 
 module Glossarist
@@ -12,7 +13,7 @@ module Glossarist
       rdf do
         namespace Namespaces::GlossaristNamespace, Namespaces::RdfNamespace
 
-        subject { |d| "definition/#{d.content.hash.abs}" }
+        subject { |d| "definition/#{GlossDetailedDefinition.deterministic_id(d)}" }
 
         types "gloss:DetailedDefinition"
 
@@ -21,6 +22,10 @@ module Glossarist
         members :sources
         members :examples, link: "gloss:hasExample"
       end
+
+      def self.deterministic_id(definition)
+        Digest::MD5.hexdigest(definition.content.to_s)[0..11]
+      end
     end
   end
 end

data/lib/glossarist/rdf/gloss_localized_concept.rb

@@ -1,10 +1,20 @@
 # frozen_string_literal: true
 
 require "lutaml/model"
+require "rdf"
 
 module Glossarist
   module Rdf
     class GlossLocalizedConcept < Lutaml::Model::Serializable
+      include EmitsExtraTriples
+
+      SKOS_PREF_LABEL   = RDF::URI("http://www.w3.org/2004/02/skos/core#prefLabel")
+      SKOS_ALT_LABEL    = RDF::URI("http://www.w3.org/2004/02/skos/core#altLabel")
+      SKOS_HIDDEN_LABEL = RDF::URI("http://www.w3.org/2004/02/skos/core#hiddenLabel")
+      SKOS_DEFINITION   = RDF::URI("http://www.w3.org/2004/02/skos/core#definition")
+      SKOS_SCOPE_NOTE   = RDF::URI("http://www.w3.org/2004/02/skos/core#scopeNote")
+      SKOS_EXAMPLE      = RDF::URI("http://www.w3.org/2004/02/skos/core#example")
+
       attribute :concept_id, :string
       attribute :language_code, :string
       attribute :domain, :string
@@ -72,6 +82,61 @@ module Glossarist
         else "skosxl:altLabel"
         end
       end
+
+      # Hook invoked by Glossarist::Rdf::LutamlTurtleTransformExt.
+      # Emits direct SKOS predicates alongside the reified SKOS-XL / gloss
+      # forms, so consumers that only speak plain SKOS (no SKOS-XL) see the
+      # labels, definitions, notes, and examples as plain literals.
+      def emit_extra_triples(subject_uri, _mapping)
+        lang = language_code.to_s if language_code && !language_code.to_s.empty?
+        triples = []
+
+        Array(designations).each do |desig|
+          predicate = self.class.skos_label_predicate(desig)
+          next unless predicate
+
+          triples << RDF::Statement.new(subject_uri, predicate,
+                                        self.class.rdf_literal(desig.designation, lang))
+        end
+
+        Array(definitions).each do |d|
+          triples << RDF::Statement.new(subject_uri, SKOS_DEFINITION,
+                                        self.class.rdf_literal(d.content, lang))
+        end
+
+        Array(notes).each do |n|
+          triples << RDF::Statement.new(subject_uri, SKOS_SCOPE_NOTE,
+                                        self.class.rdf_literal(n.content, lang))
+        end
+
+        Array(examples).each do |e|
+          triples << RDF::Statement.new(subject_uri, SKOS_EXAMPLE,
+                                        self.class.rdf_literal(e.content, lang))
+        end
+
+        triples
+      end
+
+      class << self
+        def skos_label_predicate(designation)
+          status = designation.normative_status.to_s.split("/").last
+          case status
+          when "preferred" then SKOS_PREF_LABEL
+          when "deprecated" then SKOS_HIDDEN_LABEL
+          else SKOS_ALT_LABEL
+          end
+        end
+
+        def rdf_literal(value, lang)
+          return RDF::Literal.new("") if value.nil?
+
+          if lang
+            RDF::Literal.new(value.to_s, language: lang)
+          else
+            RDF::Literal.new(value.to_s)
+          end
+        end
+      end
     end
   end
 end

data/lib/glossarist/rdf/lutaml_ext.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "lutaml/turtle"
+
+# Bridge between lutaml-model's Transform hook and Serializable instances.
+#
+# lutaml-model defines `additional_resource_triples(instance, subject_uri,
+# mapping)` on Lutaml::Turtle::Transform and calls it from `build_graph`.
+# The Transform is a separate object from the model instance, so the model
+# cannot directly emit extra RDF statements.
+#
+# Models that want to emit extra RDF (e.g., direct SKOS alongside reified
+# SKOS-XL) include `EmitsExtraTriples` and override `emit_extra_triples`.
+module Glossarist
+  module Rdf
+    module EmitsExtraTriples
+      def emit_extra_triples(_subject_uri, _mapping)
+        []
+      end
+    end
+
+    module LutamlTurtleTransformExt
+      def additional_resource_triples(instance, subject_uri, mapping)
+        triples = super
+        return triples unless instance.is_a?(EmitsExtraTriples)
+
+        triples + Array(instance.emit_extra_triples(subject_uri, mapping))
+      end
+    end
+  end
+end
+
+Lutaml::Turtle::Transform.prepend(Glossarist::Rdf::LutamlTurtleTransformExt)

data/lib/glossarist/rdf.rb

@@ -4,6 +4,8 @@ require "lutaml/rdf"
 require "lutaml/turtle"
 require "lutaml/jsonld"
 
+require_relative "rdf/lutaml_ext"
+
 module Glossarist
   module Rdf
     autoload :Namespaces,             "#{__dir__}/rdf/namespaces"

data/lib/glossarist/reference.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Glossarist
+  # Shared protocol for all reference kinds produced by ReferenceExtractor.
+  #
+  # ConceptReference, BibliographicReference, AssetReference, and the
+  # NonVerbalReference family (Figure/Table/Formula) all participate in
+  # validation rules that iterate a mixed collection extracted from a
+  # concept's text fields. Rules such as CiteRefIntegrityRule call
+  # `select(&:cite?)` on these mixed collections and must not crash on
+  # any member.
+  #
+  # The defaults here represent the common case: most reference kinds are
+  # neither inline `{{cite:...}}` mentions, nor local/external concept
+  # cross-refs. ConceptReference overrides all three predicates because
+  # its semantics depend on ref_type and source.
+  #
+  # Including this module in a new reference class is sufficient to make
+  # it participate correctly in mixed-collection validation rules.
+  module Reference
+    def cite?
+      false
+    end
+
+    def local?
+      false
+    end
+
+    def external?
+      false
+    end
+  end
+end

data/lib/glossarist/table_reference.rb

@@ -10,9 +10,5 @@ module Glossarist
       map :entity_id, to: :entity_id
       map :display, to: :display
     end
-
-    def dedup_key
-      [self.class.name, entity_id]
-    end
   end
 end

data/lib/glossarist/tasks/shacl.rake

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "rdf/turtle"
+require "shacl"
+require "pathname"
+require "glossarist/validation/shacl_validator"
+
+namespace :glossarist do
+  desc "Validate all .ttl outputs against concept-model SHACL shapes. " \
+       "Pass path=root_dir or shapes=path/to/shapes.ttl."
+  task :shacl, [:path] do |_t, args|
+    shapes = args[:shapes] || ENV.fetch("SHAPES_PATH", nil)
+    root   = args[:path] || ENV.fetch("SHACL_PATH", "compiled")
+
+    files = Pathname.glob("#{root}/**/*.ttl")
+    if files.empty?
+      warn "No .ttl files found under #{root}"
+      exit 1
+    end
+
+    validator = Glossarist::Validation::ShaclValidator.new(shapes_path: shapes)
+    report = validator.validate_files(files.map(&:to_s))
+    if report.conformant?
+      puts "All #{files.length} .ttl file(s) conform to SHACL shapes."
+    else
+      warn report.to_s
+      exit 1
+    end
+  end
+end

data/lib/glossarist/tasks/sync.rake

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require_relative "../tasks/sync_model"
+
+namespace :glossarist do
+  namespace :sync do
+    desc "Sync vendored concept-model data from upstream. " \
+         "Pass ref=[tag|branch|sha] to pin a specific version."
+    task :model, [:ref] do |_t, args|
+      ref = args[:ref] || ENV.fetch("REF", nil)
+      Glossarist::Tasks::SyncModel.call(ref: ref)
+    end
+  end
+end

data/lib/glossarist/tasks/sync_model.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "net/http"
+require "uri"
+
+module Glossarist
+  module Tasks
+    # Syncs vendored concept-model data artifacts from glossarist/concept-model.
+    #
+    # concept-model is a data-only repo (TTL, JSON-LD, YAML schemas). It is
+    # not a gem. We vendor the small set of artifacts we need (shapes,
+    # context, ontology) and load them at runtime via ShaclValidator.
+    module SyncModel
+      REPO = "glossarist/concept-model"
+      OUT_DIR = File.expand_path("data/concept-model", File.join(__dir__, "..", "..", ".."))
+
+      TARGETS = {
+        "glossarist.context.jsonld" => %w[
+          ontologies/glossarist.context.jsonld
+          glossarist.context.jsonld
+        ].freeze,
+        "glossarist.ttl" => %w[
+          ontologies/glossarist.ttl
+          glossarist.ttl
+        ].freeze,
+        "shapes/glossarist.shacl.ttl" => %w[
+          ontologies/shapes/glossarist.shacl.ttl
+          shapes/glossarist.shacl.ttl
+        ].freeze,
+      }.freeze
+
+      class << self
+        def call(ref: nil)
+          ref ||= latest_tag
+          FileUtils.mkdir_p(File.join(OUT_DIR, "shapes"))
+
+          TARGETS.each do |out_rel, candidates|
+            content = fetch_any(ref, candidates)
+            out_path = File.join(OUT_DIR, out_rel)
+            FileUtils.mkdir_p(File.dirname(out_path))
+            File.write(out_path, content)
+            puts "  ✓ #{out_rel} (#{content.length} bytes)"
+          end
+
+          write_source_manifest(ref)
+          puts "\nSynced #{TARGETS.length} file(s) from #{REPO}@#{ref}."
+        end
+
+        private
+
+        def latest_tag
+          url = URI("https://api.github.com/repos/#{REPO}/releases/latest")
+          req = Net::HTTP::Get.new(url)
+          req["Accept"] = "application/vnd.github+json"
+          JSON.parse(Net::HTTP.start(url.hostname, url.port, use_ssl: true) { |http| http.request(req) }.body)
+            .fetch("tag_name")
+        rescue StandardError => e
+          warn "Could not determine latest concept-model tag: #{e.message}"
+          exit 1
+        end
+
+        def fetch_any(ref, candidates)
+          candidates.each do |path|
+            begin
+              return fetch_file(ref, path)
+            rescue StandardError
+              next
+            end
+          end
+          raise "Could not fetch any of: #{candidates.join(', ')}"
+        end
+
+        def fetch_file(ref, path)
+          url = URI("https://raw.githubusercontent.com/#{REPO}/#{ref}/#{path}")
+          Net::HTTP.get(url)
+        end
+
+        def write_source_manifest(ref)
+          manifest = {
+            "repo" => REPO,
+            "ref" => ref,
+            "syncedAt" => Time.now.utc.iso8601,
+          }
+          File.write(File.join(OUT_DIR, "SOURCE.json"),
+                     JSON.pretty_generate(manifest) + "\n")
+        end
+      end
+    end
+  end
+end

data/lib/glossarist/transforms/concept_to_gloss_transform.rb

@@ -122,10 +122,11 @@ module Glossarist
 
         dd_attrs = if data
                      data.class.detailed_definition_fields.to_h do |field|
-                       [field, build_gloss_definitions(data.public_send(field))]
+                       key = field == :definition ? :definitions : field
+                       [key, build_gloss_definitions(data.public_send(field))]
                      end
                    else
-                     { definition: [], notes: [], examples: [] }
+                     { definitions: [], notes: [], examples: [] }
                    end
 
         sources = build_gloss_sources(data&.sources)