glossarist 2.6.6 → 2.6.7

data/scripts/migrate_dataset.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# Migrate glossarist datasets to current schema format.
+#
+# Usage:
+#   bundle exec ruby scripts/migrate_dataset.rb SOURCE_DIR OUTPUT_DIR [--add-iev-domains]
+#
+# --add-iev-domains: Add domain ConceptReference objects for IEV-style identifiers
+#   (e.g. "426-24-74" → area-426, section-426-24)
+
+require "glossarist"
+require "fileutils"
+
+source_dir = ARGV[0]
+output_dir = ARGV[1]
+add_iev_domains = ARGV.include?("--add-iev-domains")
+
+unless source_dir && output_dir
+  abort "Usage: bundle exec ruby scripts/migrate_dataset.rb SOURCE_DIR OUTPUT_DIR [--add-iev-domains]"
+end
+
+source_dir = File.expand_path(source_dir)
+output_dir = File.expand_path(output_dir)
+
+unless File.directory?(source_dir)
+  abort "Error: #{source_dir} is not a directory"
+end
+
+def add_subject_area_concepts(collection)
+  areas = {}
+  sections = {}
+
+  collection.each do |concept|
+    next unless concept.data.domains
+
+    concept.data.domains.each do |ref|
+      next unless ref.is_a?(Glossarist::ConceptReference) && ref.concept_id
+
+      id = ref.concept_id
+      if id.start_with?("area-")
+        areas[id] = true
+      elsif id.start_with?("section-")
+        sections[id] = true
+      end
+    end
+  end
+
+  existing_ids = collection.map { |c| c.data.id }.to_set
+
+  areas.each_key do |area_id|
+    next if existing_ids.include?(area_id)
+
+    mc = Glossarist::ManagedConcept.new(
+      data: Glossarist::ManagedConceptData.new(
+        id: area_id,
+        domains: [Glossarist::ConceptReference.domain(area_id)],
+      ),
+    )
+
+    code = area_id.sub("area-", "")
+    narrower = sections.keys.select { |s| s.start_with?("section-#{code}-") }
+    mc.related = narrower.map { |s| Glossarist::RelatedConcept.new(type: "narrower", content: s) }
+
+    l10n = build_domain_localization(area_id, code, "eng")
+    mc.add_l10n(l10n)
+
+    collection.store(mc)
+  end
+
+  sections.each_key do |section_id|
+    next if existing_ids.include?(section_id)
+
+    parts = section_id.sub("section-", "").split("-")
+    area_id = "area-#{parts[0]}"
+
+    mc = Glossarist::ManagedConcept.new(
+      data: Glossarist::ManagedConceptData.new(
+        id: section_id,
+        domains: [
+          Glossarist::ConceptReference.domain(area_id),
+          Glossarist::ConceptReference.domain(section_id),
+        ],
+      ),
+    )
+
+    mc.related = [Glossarist::RelatedConcept.new(type: "broader", content: area_id)]
+
+    section_code = parts.length > 1 ? parts[0..1].join("-") : parts[0]
+    l10n = build_domain_localization(section_id, section_code, "eng")
+    l10n.data.domain = area_id
+    mc.add_l10n(l10n)
+
+    collection.store(mc)
+  end
+end
+
+def build_domain_localization(id, label, lang_code)
+  cd = Glossarist::ConceptData.new(
+    id: id,
+    language_code: lang_code,
+    terms: [
+      Glossarist::Designation::Expression.new(
+        type: "expression",
+        designation: id,
+        normative_status: "preferred",
+      ),
+    ],
+  )
+
+  l10n = Glossarist::LocalizedConcept.new
+  l10n.data = cd
+  l10n.entry_status = "valid"
+  l10n
+end
+
+# Detect format: managed (concept/ + localized_concept/) vs grouped (*.yaml)
+concept_subdir = File.join(source_dir, "concept")
+is_managed_format = File.directory?(concept_subdir)
+
+puts "Loading concepts from #{source_dir} (#{is_managed_format ? 'managed' : 'grouped'} format)..."
+
+collection = Glossarist::ManagedConceptCollection.new
+collection.load_from_files(source_dir)
+
+puts "Loaded #{collection.count} concepts"
+
+# Add IEV domain references if requested
+if add_iev_domains
+  puts "Adding IEV domain references..."
+
+  collection.each do |concept|
+    next if concept.data.domains && !concept.data.domains.empty?
+
+    identifier = concept.data.id.to_s
+    next if identifier.empty? || identifier.start_with?("area-", "section-")
+    parts = identifier.split("-")
+    next unless parts.length >= 2
+
+    area_uri = "area-#{parts[0]}"
+    section_uri = "section-#{parts[0]}-#{parts[1]}"
+
+    concept.data.domains = [
+      Glossarist::ConceptReference.domain(area_uri),
+      Glossarist::ConceptReference.domain(section_uri),
+    ]
+  end
+
+  puts "Adding subject area hierarchy concepts..."
+  add_subject_area_concepts(collection)
+
+  puts "Domains added. Total concepts: #{collection.count}"
+end
+
+# Save output
+puts "Saving to #{output_dir}..."
+
+if is_managed_format
+  concepts_out = File.join(output_dir, "concepts")
+  FileUtils.mkdir_p(concepts_out)
+  collection.save_to_files(concepts_out)
+else
+  concepts_out = File.join(output_dir)
+  FileUtils.mkdir_p(concepts_out)
+  collection.save_grouped_concepts_to_files(concepts_out)
+end
+
+# Copy register.yaml if present
+register_src = File.join(File.dirname(source_dir), "register.yaml")
+if File.exist?(register_src) && !File.exist?(File.join(output_dir, "..", "register.yaml"))
+  register_dst_dir = is_managed_format ? File.dirname(output_dir) : output_dir
+  register_dst = if File.exist?(File.join(File.dirname(source_dir), "register.yaml"))
+    File.join(is_managed_format ? File.dirname(output_dir) : File.dirname(output_dir), "register.yaml")
+  end
+  if register_dst
+    FileUtils.mkdir_p(File.dirname(register_dst))
+    FileUtils.cp(register_src, register_dst) unless register_src == register_dst
+  end
+end
+
+puts "Done. #{collection.count} concepts migrated."

data/scripts/migrate_isotc204_to_v3.rb

@@ -0,0 +1,134 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Migration script for isotc204-glossary: adds v3 fields to existing concepts.
+#
+# Adds:
+#   - status: "valid" at managed concept level
+#   - date_accepted at managed concept level
+#   - domains (section ConceptReference) at managed concept data level
+#   - related (broader to section) at managed concept level
+#   - Creates section hierarchy concepts with narrower relations
+#
+# Usage:
+#   cd /Users/mulgogi/src/glossarist/glossarist-ruby
+#   bundle exec ruby scripts/migrate_isotc204_to_v3.rb
+#
+# Safe to run multiple times (idempotent).
+
+require "glossarist"
+
+DIR = "/Users/mulgogi/src/geolexica/isotc204-glossary/concepts"
+
+ISO_TS_14812_SECTIONS = {
+  "3.1" => "General concepts",
+  "3.2" => "Transport information and control",
+  "3.3" => "ITS station",
+  "3.4" => "Communications",
+  "3.5" => "ITS services",
+  "3.6" => "Geospatial",
+  "3.7" => "Driving automation",
+}.freeze
+
+ISO_TS_14812_DATE = "2022-01-01T00:00:00+00:00"
+ISO_TS_14812_SOURCE = "urn:iso:std:iso:ts:14812"
+
+collection = Glossarist::ManagedConceptCollection.new
+collection.load_from_files(DIR)
+
+puts "Loaded #{collection.count} concepts"
+
+# Track which concepts belong to which section for narrower relations
+section_children = Hash.new { |h, k| h[k] = [] }
+
+collection.each do |concept|
+  identifier = concept.data.id.to_s
+  section_code = identifier.split(".")[0..1].join(".")
+  section_uri = "section-#{section_code.gsub('.', '-')}"
+  section_children[section_uri] << identifier
+
+  # Set status
+  concept.status = "valid" unless concept.status
+
+  # Set date_accepted
+  unless concept.date_accepted
+    concept.date_accepted = Glossarist::ConceptDate.new(
+      type: "accepted",
+      date: ISO_TS_14812_DATE,
+    )
+  end
+
+  # Add domain ConceptReference
+  concept.data.domains ||= []
+  unless concept.data.domains.any? { |d| d.concept_id == section_uri }
+    concept.data.domains << Glossarist::ConceptReference.new(
+      concept_id: section_uri,
+      source: ISO_TS_14812_SOURCE,
+      ref_type: "domain",
+    )
+  end
+
+  # Add broader relation to section
+  concept.related ||= []
+  unless concept.related.any? { |r| r.type == "broader" && r.ref&.id == section_uri }
+    concept.related << Glossarist::RelatedConcept.new(
+      type: "broader",
+      content: section_uri,
+      ref: Glossarist::Citation.new(source: "ISO/TS 14812", id: section_uri),
+    )
+  end
+end
+
+puts "Updated #{collection.count} concepts with status, date_accepted, domains, related"
+
+# Create section hierarchy concepts
+ISO_TS_14812_SECTIONS.each do |code, title|
+  section_uri = "section-#{code.gsub('.', '-')}"
+
+  mc = Glossarist::ManagedConcept.new(
+    data: Glossarist::ManagedConceptData.new(
+      id: section_uri,
+      domains: [Glossarist::ConceptReference.new(
+        concept_id: section_uri,
+        source: ISO_TS_14812_SOURCE,
+        ref_type: "domain",
+      )],
+    ),
+  )
+  mc.status = "valid"
+  mc.date_accepted = Glossarist::ConceptDate.new(
+    type: "accepted",
+    date: ISO_TS_14812_DATE,
+  )
+
+  l10n = Glossarist::LocalizedConcept.new
+  l10n.data = Glossarist::ConceptData.new(
+    id: section_uri,
+    language_code: "eng",
+    terms: [Glossarist::Designation::Expression.new(
+      type: "expression",
+      designation: title,
+      normative_status: "preferred",
+    )],
+  )
+  l10n.entry_status = "valid"
+  mc.add_l10n(l10n)
+
+  # Add narrower relations to child concepts
+  children = section_children[section_uri]
+  if children.any?
+    mc.related = children.sort.map do |child_id|
+      Glossarist::RelatedConcept.new(
+        type: "narrower",
+        content: child_id,
+        ref: Glossarist::Citation.new(source: "ISO/TS 14812", id: child_id),
+      )
+    end
+  end
+
+  collection.store(mc)
+  puts "Created section concept: #{section_uri} (#{title}) — #{children.length} narrower"
+end
+
+collection.save_grouped_concepts_to_files(DIR)
+puts "Saved #{collection.count} concepts to #{DIR}"

data/scripts/migrate_isotc211_to_v3.rb

@@ -0,0 +1,153 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Migration script for isotc211-glossary: adds v3 fields to existing concepts.
+#
+# Adds:
+#   - domains (ISO standard ConceptReference) at managed concept data level
+#   - related (broader to standard domain) at managed concept level
+#   - Creates domain concepts per ISO standard (with narrower relations)
+#
+# Idempotent: safe to run multiple times.
+#
+# Usage:
+#   cd /Users/mulgogi/src/glossarist/glossarist-ruby
+#   bundle exec ruby scripts/migrate_isotc211_to_v3.rb
+
+require "glossarist"
+
+DIR = "/Users/mulgogi/src/geolexica/isotc211-glossary/concepts"
+
+ISO_SOURCE_URN = "urn:iso:std:iso"
+
+# Extract a stable domain ID from an authoritative source reference string.
+# @param ref_text [String] e.g. "ISO 19136-1:2020", "ISO/IEC 19501:2005"
+# @return [String, nil] domain ID e.g. "iso-19136-1", "iso-iec-19501"
+def extract_domain_id(ref_text)
+  # Match various ISO reference patterns
+  patterns = [
+    %r{ISO/IEC/IEEE\s+([\d-]+)},    # ISO/IEC/IEEE 24765:2017
+    %r{ISO/IEC\s+([\d-]+)},          # ISO/IEC 19501:2005
+    %r{ISO/TS\s+([\d-]+)},           # ISO/TS 19130:2010
+    %r{ISO/TR\s+([\d-]+)},           # ISO/TR 19120:2001
+    %r{ISO/IEC\s+Guide\s+([\d-]+)},  # ISO/IEC Guide 98-3:2008
+    %r{ISO\s+DIS\s+([\d-]+)},        # ISO DIS 19123-1:2022
+    %r{ISO\s+([\d]+-?[\d]*)},        # ISO 19136-1:2020
+  ]
+
+  patterns.each do |pat|
+    if (m = ref_text.match(pat))
+      # Extract the full match, normalize
+      prefix = ref_text[m.begin(0)...m.begin(1)].strip
+      number = m[1]
+      domain = "#{prefix} #{number}"
+      return domain.downcase.gsub(/[\s\/]+/, "-")
+    end
+  end
+
+  nil
+end
+
+collection = Glossarist::ManagedConceptCollection.new
+collection.load_from_files(DIR)
+
+puts "Loaded #{collection.count} concepts"
+
+# Build index: domain_id -> [concept_id]
+domain_index = {}
+concepts_with_domain = 0
+concepts_without_domain = 0
+
+collection.each do |concept|
+  eng = concept.localization("eng")
+  next unless eng
+
+  sources = eng.data&.sources
+  next unless sources
+
+  auth = sources.find { |s| s.type == "authoritative" }
+  next unless auth&.origin
+
+  ref_text = auth.origin.text || auth.origin.ref
+  unless ref_text
+    concepts_without_domain += 1
+    next
+  end
+
+  domain_id = extract_domain_id(ref_text)
+  unless domain_id
+    concepts_without_domain += 1
+    next
+  end
+
+  (domain_index[domain_id] ||= []) << concept.data.id
+  concepts_with_domain += 1
+
+  # Add domain ConceptReference
+  concept.data.domains ||= []
+  unless concept.data.domains.any? { |d| d.concept_id == domain_id }
+    concept.data.domains << Glossarist::ConceptReference.new(
+      concept_id: domain_id,
+      source: ISO_SOURCE_URN,
+      ref_type: "domain",
+    )
+  end
+
+  # Add broader relation to domain concept
+  concept.related ||= []
+  unless concept.related.any? { |r| r.type == "broader" && r.ref&.id == domain_id }
+    concept.related << Glossarist::RelatedConcept.new(
+      type: "broader",
+      content: domain_id,
+      ref: Glossarist::Citation.new(source: "ISO", id: domain_id),
+    )
+  end
+end
+
+puts "Added domains to #{concepts_with_domain} concepts"
+puts "No domain extracted for #{concepts_without_domain} concepts"
+
+# Create domain hierarchy concepts
+domain_index.sort.each do |domain_id, child_ids|
+  mc = Glossarist::ManagedConcept.new(
+    data: Glossarist::ManagedConceptData.new(
+      id: domain_id,
+      domains: [Glossarist::ConceptReference.new(
+        concept_id: domain_id,
+        source: ISO_SOURCE_URN,
+        ref_type: "domain",
+      )],
+    ),
+  )
+  mc.status = "valid"
+
+  # Create a basic English localization with the domain ID as the term
+  l10n = Glossarist::LocalizedConcept.new
+  l10n.data = Glossarist::ConceptData.new(
+    id: domain_id,
+    language_code: "eng",
+    terms: [Glossarist::Designation::Expression.new(
+      type: "expression",
+      designation: domain_id,
+      normative_status: "preferred",
+    )],
+  )
+  l10n.entry_status = "valid"
+  mc.add_l10n(l10n)
+
+  # Add narrower relations to child concepts
+  narrower = child_ids.sort.map do |child_id|
+    Glossarist::RelatedConcept.new(
+      type: "narrower",
+      content: child_id.to_s,
+      ref: Glossarist::Citation.new(source: "ISO", id: child_id.to_s),
+    )
+  end
+  mc.related = narrower
+
+  collection.store(mc)
+  puts "Created domain: #{domain_id} — #{child_ids.size} narrower"
+end
+
+collection.save_grouped_concepts_to_files(DIR)
+puts "Saved #{collection.count} concepts to #{DIR}"

data/scripts/migrate_osgeo_to_v3.rb

@@ -0,0 +1,155 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Migration script for osgeo-glossary: adds v3 fields to existing concepts.
+#
+# Adds:
+#   - status: "valid" at managed concept level
+#   - date_accepted at managed concept level (set to 2011-01-01 as earliest
+#     known publication date for the OSGeo Lexicon)
+#   - domains for concepts with identifiable ISO standard sources
+#   - related (broader to domain) for those concepts
+#   - Creates domain concepts for identified ISO standards
+#
+# Idempotent: safe to run multiple times.
+#
+# Usage:
+#   cd /Users/mulgogi/src/glossarist/glossarist-ruby
+#   bundle exec ruby scripts/migrate_osgeo_to_v3.rb
+
+require "glossarist"
+
+DIR = "/Users/mulgogi/src/geolexica/osgeo-glossary/concepts"
+
+OSGEO_DATE = "2011-01-01T00:00:00+00:00"
+
+# Extract a stable domain ID from an authoritative source reference string.
+def extract_domain_id(ref_text)
+  return nil unless ref_text
+
+  patterns = [
+    %r{ISO/IEC/IEEE\s+([\d-]+)},
+    %r{ISO/IEC\s+([\d-]+)},
+    %r{ISO/TS\s+([\d-]+)},
+    %r{ISO/TR\s+([\d-]+)},
+    %r{ISO\s+([\d]+-?[\d]*)},
+  ]
+
+  patterns.each do |pat|
+    if (m = ref_text.match(pat))
+      return "iso-#{m[1]}"
+    end
+  end
+
+  nil
+end
+
+collection = Glossarist::ManagedConceptCollection.new
+collection.load_from_files(DIR)
+
+puts "Loaded #{collection.count} concepts"
+
+domain_index = {}
+concepts_with_domain = 0
+
+collection.each do |concept|
+  # Set status
+  concept.status = "valid" unless concept.status
+
+  # Set date_accepted
+  unless concept.date_accepted
+    concept.date_accepted = Glossarist::ConceptDate.new(
+      type: "accepted",
+      date: OSGEO_DATE,
+    )
+  end
+
+  # Extract domain from source
+  eng = concept.localization("eng")
+  next unless eng
+
+  sources = eng.data&.sources
+  next unless sources
+
+  auth = sources.find { |s| s.type == "authoritative" }
+  next unless auth&.origin
+
+  ref_text = auth.origin.text || auth.origin.ref
+  next unless ref_text
+
+  domain_id = extract_domain_id(ref_text)
+  next unless domain_id
+
+  (domain_index[domain_id] ||= []) << concept.data.id
+  concepts_with_domain += 1
+
+  # Add domain ConceptReference
+  concept.data.domains ||= []
+  unless concept.data.domains.any? { |d| d.concept_id == domain_id }
+    concept.data.domains << Glossarist::ConceptReference.new(
+      concept_id: domain_id,
+      source: "urn:iso:std:iso",
+      ref_type: "domain",
+    )
+  end
+
+  # Add broader relation to domain concept
+  concept.related ||= []
+  unless concept.related.any? { |r| r.type == "broader" && r.ref&.id == domain_id }
+    concept.related << Glossarist::RelatedConcept.new(
+      type: "broader",
+      content: domain_id,
+      ref: Glossarist::Citation.new(source: "ISO", id: domain_id),
+    )
+  end
+end
+
+puts "Added status and date_accepted to #{collection.count} concepts"
+puts "Added domains to #{concepts_with_domain} concepts with ISO sources"
+
+# Create domain hierarchy concepts
+domain_index.sort.each do |domain_id, child_ids|
+  mc = Glossarist::ManagedConcept.new(
+    data: Glossarist::ManagedConceptData.new(
+      id: domain_id,
+      domains: [Glossarist::ConceptReference.new(
+        concept_id: domain_id,
+        source: "urn:iso:std:iso",
+        ref_type: "domain",
+      )],
+    ),
+  )
+  mc.status = "valid"
+  mc.date_accepted = Glossarist::ConceptDate.new(
+    type: "accepted",
+    date: OSGEO_DATE,
+  )
+
+  l10n = Glossarist::LocalizedConcept.new
+  l10n.data = Glossarist::ConceptData.new(
+    id: domain_id,
+    language_code: "eng",
+    terms: [Glossarist::Designation::Expression.new(
+      type: "expression",
+      designation: domain_id,
+      normative_status: "preferred",
+    )],
+  )
+  l10n.entry_status = "valid"
+  mc.add_l10n(l10n)
+
+  narrower = child_ids.sort.map do |child_id|
+    Glossarist::RelatedConcept.new(
+      type: "narrower",
+      content: child_id.to_s,
+      ref: Glossarist::Citation.new(source: "OSGeo", id: child_id.to_s),
+    )
+  end
+  mc.related = narrower
+
+  collection.store(mc)
+  puts "Created domain: #{domain_id} — #{child_ids.size} narrower"
+end
+
+collection.save_grouped_concepts_to_files(DIR)
+puts "Saved #{collection.count} concepts to #{DIR}"

data/scripts/upgrade_dataset_to_v3.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Upgrades a dataset directory to v3 format.
+#
+# Usage: bundle exec ruby scripts/upgrade_dataset_to_v3.rb <concepts_dir> [--grouped|--separate]
+#
+# Formats:
+#   --grouped  (default): each concept + localizations in a single YAML file
+#   --separate: concept/ and localized_concept/ in separate files
+#
+# For each concept:
+#   1. Loads with ConceptManager (auto-detects v2/v3)
+#   2. Promotes data.related -> top-level related (if needed)
+#   3. Sets schema_version = "3"
+#   4. Re-saves in v3 format
+
+require "glossarist"
+
+dir = ARGV[0]
+mode = ARGV.include?("--separate") ? :separate : :grouped
+
+unless dir && Dir.exist?(dir)
+  abort "Usage: #{$PROGRAM_NAME} <concepts_dir> [--grouped|--separate]"
+end
+
+manager = Glossarist::ConceptManager.new(path: dir)
+collection = Glossarist::ManagedConceptCollection.new
+
+puts "Loading concepts from #{dir}..."
+manager.load_from_files(collection: collection)
+
+count = 0
+collection.each do |concept|
+  Glossarist::SchemaMigration.migrate_concept(concept)
+  count += 1
+end
+
+puts "Migrated #{count} concepts. Saving..."
+
+if mode == :separate
+  manager.save_to_files(collection)
+else
+  manager.save_grouped_concepts_to_files(collection)
+end
+
+puts "Done. All concepts now in v3 format."