unitsdb 1.0.0 → 2.1.0

data/lib/unitsdb/commands/si_updater.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "fileutils"
+
+module Unitsdb
+  module Commands
+    # Updater for SI references in YAML
+    module SiUpdater
+      SI_AUTHORITY = "si-digital-framework"
+
+      module_function
+
+      # Update references in YAML file (TTL → DB direction)
+      def update_references(entity_type, missing_matches, db_entities, output_file, include_potential = false)
+        # Use the database objects to access the data directly
+        original_yaml_file = db_entities.first.send(:yaml_file) if db_entities&.first.respond_to?(:yaml_file, true)
+
+        # If we can't get the path from the database object, use the output file path as a fallback
+        if original_yaml_file.nil? || !File.exist?(original_yaml_file)
+          puts "Warning: Could not determine original YAML file path. Using output file as template."
+          original_yaml_file = output_file
+
+          # Create an empty template if output file doesn't exist
+          unless File.exist?(original_yaml_file)
+            FileUtils.mkdir_p(File.dirname(original_yaml_file))
+            File.write(original_yaml_file, { entity_type => [] }.to_yaml)
+          end
+        end
+
+        # Load the original YAML file
+        yaml_content = File.read(original_yaml_file)
+        output_data = YAML.safe_load(yaml_content)
+
+        # Group by entity ID to avoid duplicates
+        grouped_matches = missing_matches.group_by { |match| match[:entity_id] }
+
+        # Process each entity that needs updating
+        grouped_matches.each do |entity_id, matches|
+          # Filter matches based on include_potential parameter
+          filtered_matches = matches.select do |match|
+            # Check if it's an exact match or if we're including potential matches
+            match_details = match[:match_details]
+            if match_details&.dig(:exact) == false || %w[symbol_match
+                                                         partial_match].include?(match_details&.dig(:match_desc) || "")
+              include_potential
+            else
+              true
+            end
+          end
+
+          # Skip if no matches after filtering
+          next if filtered_matches.empty?
+
+          # Find the entity in the array under the entity_type key
+          entity_index = output_data[entity_type].find_index do |e|
+            # Find entity with matching identifier
+            e["identifiers"]&.any? { |id| id["id"] == entity_id }
+          end
+
+          next unless entity_index
+
+          # Get the entity
+          entity = output_data[entity_type][entity_index]
+
+          # Initialize references array if it doesn't exist
+          entity["references"] ||= []
+
+          # Add new references
+          filtered_matches.each do |match|
+            # If this match has multiple SI references, add them all
+            if match[:multiple_si]
+              match[:multiple_si].each do |si_data|
+                # Check if reference already exists
+                next if entity["references"].any? do |ref|
+                  ref["uri"] == si_data[:uri] && ref["authority"] == SI_AUTHORITY
+                end
+
+                # Add new reference
+                entity["references"] << {
+                  "uri" => si_data[:uri],
+                  "type" => "normative",
+                  "authority" => SI_AUTHORITY
+                }
+              end
+            else
+              # Check if reference already exists
+              next if entity["references"].any? do |ref|
+                ref["uri"] == match[:si_uri] && ref["authority"] == SI_AUTHORITY
+              end
+
+              # Add new reference
+              entity["references"] << {
+                "uri" => match[:si_uri],
+                "type" => "normative",
+                "authority" => SI_AUTHORITY
+              }
+            end
+          end
+        end
+
+        write_yaml_file(output_file, output_data)
+      end
+
+      # Update references in YAML file (DB → TTL direction)
+      def update_db_references(entity_type, missing_refs, output_file, include_potential = false)
+        # Try to get the original YAML file from the first entity
+        first_entity = missing_refs.first&.dig(:db_entity)
+        original_yaml_file = first_entity.send(:yaml_file) if first_entity.respond_to?(:yaml_file, true)
+
+        # If we can't get the path from the database object, use the output file path as a fallback
+        if original_yaml_file.nil? || !File.exist?(original_yaml_file)
+          puts "Warning: Could not determine original YAML file path. Using output file as template."
+          original_yaml_file = output_file
+
+          # Create an empty template if output file doesn't exist
+          unless File.exist?(original_yaml_file)
+            FileUtils.mkdir_p(File.dirname(original_yaml_file))
+            File.write(original_yaml_file, { entity_type => [] }.to_yaml)
+          end
+        end
+
+        # Load the original YAML file
+        yaml_content = File.read(original_yaml_file)
+        output_data = YAML.safe_load(yaml_content)
+
+        # Group by entity ID to avoid duplicates
+        missing_refs_by_id = {}
+
+        missing_refs.each do |match|
+          entity_id = match[:entity_id] || match[:db_entity].short
+          ttl_entities = match[:ttl_entities]
+          match_types = match[:match_types] || {}
+
+          # Filter TTL entities based on include_potential parameter
+          filtered_ttl_entities = ttl_entities.select do |ttl_entity|
+            # Check if it's an exact match or if we're including potential matches
+            match_type = match_types[ttl_entity[:uri]] || "Exact match" # Default to exact match
+            match_pair_key = "#{entity_id}:#{ttl_entity[:uri]}"
+            match_details = Unitsdb::Commands::SiMatcher.instance_variable_get(:@match_details)&.dig(match_pair_key)
+
+            if match_details && %w[symbol_match partial_match].include?(match_details[:match_desc])
+              include_potential
+            else
+              match_type == "Exact match" || include_potential
+            end
+          end
+
+          # Skip if no entities after filtering
+          next if filtered_ttl_entities.empty?
+
+          missing_refs_by_id[entity_id] ||= []
+
+          # Add filtered matching TTL entities for this DB entity
+          filtered_ttl_entities.each do |ttl_entity|
+            missing_refs_by_id[entity_id] << {
+              uri: ttl_entity[:uri],
+              type: "normative",
+              authority: SI_AUTHORITY
+            }
+          end
+        end
+
+        # Update the YAML content
+        output_data[entity_type].each do |entity_yaml|
+          # Find entity by ID or short
+          entity_id = if entity_yaml["identifiers"]
+                        begin
+                          entity_yaml["identifiers"].first["id"]
+                        rescue StandardError
+                          nil
+                        end
+                      elsif entity_yaml["id"]
+                        entity_yaml["id"]
+                      end
+
+          next unless entity_id && missing_refs_by_id.key?(entity_id)
+
+          # Add references
+          entity_yaml["references"] ||= []
+
+          missing_refs_by_id[entity_id].each do |ref|
+            # Check if this reference already exists
+            next if entity_yaml["references"].any? do |existing_ref|
+              existing_ref["uri"] == ref[:uri] &&
+              existing_ref["authority"] == ref[:authority]
+            end
+
+            # Add the reference
+            entity_yaml["references"] << {
+              "uri" => ref[:uri],
+              "type" => ref[:type],
+              "authority" => ref[:authority]
+            }
+          end
+        end
+
+        write_yaml_file(output_file, output_data)
+      end
+
+      # Helper to write YAML file
+      def write_yaml_file(output_file, output_data)
+        # Ensure the output directory exists
+        output_dir = File.dirname(output_file)
+        FileUtils.mkdir_p(output_dir) unless Dir.exist?(output_dir)
+
+        # Write to YAML file
+        File.write(output_file, output_data.to_yaml)
+      end
+    end
+  end
+end

data/lib/unitsdb/commands/validate/identifiers.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Unitsdb
+  module Commands
+    module Validate
+      class Identifiers < Base
+        def run
+          db = load_database
+          all_dups = db.validate_uniqueness
+
+          display_results(all_dups)
+        rescue Unitsdb::Errors::DatabaseError => e
+          puts "Error: #{e.message}"
+          exit(1)
+        end
+
+        private
+
+        def display_results(all_dups)
+          %i[short id].each do |type|
+            dups = all_dups[type]
+            if dups.empty?
+              puts "No duplicate '#{type}' fields found."
+              next
+            end
+
+            puts "\nFound duplicate '#{type}' fields:"
+            dups.each do |file, items|
+              puts "  #{file}:"
+              items.each do |val, paths|
+                puts "    '#{val}':"
+                paths.each { |p| puts "      - #{p}" }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unitsdb/commands/validate/references.rb

@@ -0,0 +1,316 @@
+# frozen_string_literal: true
+
+module Unitsdb
+  module Commands
+    module Validate
+      class References < Base
+        def run
+          # Load the database
+          db = load_database(@options[:database])
+
+          # Build registry of all valid IDs
+          registry = build_id_registry(db)
+
+          # Check all references
+          invalid_refs = check_references(db, registry)
+
+          # Display results
+          display_reference_results(invalid_refs, registry)
+        rescue Unitsdb::Errors::DatabaseError => e
+          puts "Error: #{e.message}"
+          exit(1)
+        end
+
+        private
+
+        def build_id_registry(db)
+          registry = {}
+
+          # Add all unit identifiers to the registry
+          registry["units"] = {}
+          db.units.each_with_index do |unit, index|
+            unit.identifiers.each do |identifier|
+              next unless identifier.id && identifier.type
+
+              # Add the composite key (type:id)
+              composite_key = "#{identifier.type}:#{identifier.id}"
+              registry["units"][composite_key] = "index:#{index}"
+
+              # Also add just the ID for backward compatibility
+              registry["units"][identifier.id] = "index:#{index}"
+            end
+          end
+
+          # Add dimension identifiers
+          registry["dimensions"] = {}
+          db.dimensions.each_with_index do |dimension, index|
+            dimension.identifiers.each do |identifier|
+              next unless identifier.id && identifier.type
+
+              composite_key = "#{identifier.type}:#{identifier.id}"
+              registry["dimensions"][composite_key] = "index:#{index}"
+              registry["dimensions"][identifier.id] = "index:#{index}"
+            end
+
+            # Also track dimensions by short name
+            if dimension.respond_to?(:short) && dimension.short
+              registry["dimensions_short"] ||= {}
+              registry["dimensions_short"][dimension.short] = "index:#{index}"
+            end
+          end
+
+          # Add quantity identifiers
+          registry["quantities"] = {}
+          db.quantities.each_with_index do |quantity, index|
+            quantity.identifiers.each do |identifier|
+              next unless identifier.id && identifier.type
+
+              composite_key = "#{identifier.type}:#{identifier.id}"
+              registry["quantities"][composite_key] = "index:#{index}"
+              registry["quantities"][identifier.id] = "index:#{index}"
+            end
+          end
+
+          # Add prefix identifiers
+          registry["prefixes"] = {}
+          db.prefixes.each_with_index do |prefix, index|
+            prefix.identifiers.each do |identifier|
+              next unless identifier.id && identifier.type
+
+              composite_key = "#{identifier.type}:#{identifier.id}"
+              registry["prefixes"][composite_key] = "index:#{index}"
+              registry["prefixes"][identifier.id] = "index:#{index}"
+            end
+          end
+
+          # Add unit system identifiers
+          registry["unit_systems"] = {}
+          db.unit_systems.each_with_index do |unit_system, index|
+            unit_system.identifiers.each do |identifier|
+              next unless identifier.id && identifier.type
+
+              composite_key = "#{identifier.type}:#{identifier.id}"
+              registry["unit_systems"][composite_key] = "index:#{index}"
+              registry["unit_systems"][identifier.id] = "index:#{index}"
+            end
+
+            # Also track unit systems by short name
+            if unit_system.respond_to?(:short) && unit_system.short
+              registry["unit_systems_short"] ||= {}
+              registry["unit_systems_short"][unit_system.short] = "index:#{index}"
+            end
+          end
+
+          # Debug registry if requested
+          if @options[:debug_registry]
+            puts "Registry contents:"
+            registry.each do |type, ids|
+              puts "  #{type}:"
+              ids.each do |id, location|
+                puts "    #{id} => #{location}"
+              end
+            end
+          end
+
+          registry
+        end
+
+        def check_references(db, registry)
+          invalid_refs = {}
+
+          # Check unit references in dimensions
+          check_dimension_references(db, registry, invalid_refs)
+
+          # Check unit_system references
+          check_unit_system_references(db, registry, invalid_refs)
+
+          # Check quantity references
+          check_quantity_references(db, registry, invalid_refs)
+
+          # Check root unit references in units
+          check_root_unit_references(db, registry, invalid_refs)
+
+          invalid_refs
+        end
+
+        def check_dimension_references(db, registry, invalid_refs)
+          db.dimensions.each_with_index do |dimension, index|
+            next unless dimension.respond_to?(:dimension_reference) && dimension.dimension_reference
+
+            ref_id = dimension.dimension_reference
+            ref_type = "dimensions"
+            ref_path = "dimensions:index:#{index}:dimension_reference"
+
+            validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, "dimensions")
+          end
+        end
+
+        def check_unit_system_references(db, registry, invalid_refs)
+          db.units.each_with_index do |unit, index|
+            next unless unit.respond_to?(:unit_system_reference) && unit.unit_system_reference
+
+            unit.unit_system_reference.each_with_index do |ref_id, idx|
+              ref_type = "unit_systems"
+              ref_path = "units:index:#{index}:unit_system_reference[#{idx}]"
+
+              validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, "units")
+            end
+          end
+        end
+
+        def check_quantity_references(db, registry, invalid_refs)
+          db.units.each_with_index do |unit, index|
+            next unless unit.respond_to?(:quantity_references) && unit.quantity_references
+
+            unit.quantity_references.each_with_index do |ref_id, idx|
+              ref_type = "quantities"
+              ref_path = "units:index:#{index}:quantity_references[#{idx}]"
+
+              validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, "units")
+            end
+          end
+        end
+
+        def check_root_unit_references(db, registry, invalid_refs)
+          db.units.each_with_index do |unit, index|
+            next unless unit.respond_to?(:root_units) && unit.root_units
+
+            unit.root_units.each_with_index do |root_unit, idx|
+              next unless root_unit.respond_to?(:unit_reference) && root_unit.unit_reference
+
+              # Check unit reference
+              ref_id = root_unit.unit_reference
+              ref_type = "units"
+              ref_path = "units:index:#{index}:root_units.#{idx}.unit_reference"
+
+              validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, "units")
+
+              # Check prefix reference if present
+              next unless root_unit.respond_to?(:prefix_reference) && root_unit.prefix_reference
+
+              ref_id = root_unit.prefix_reference
+              ref_type = "prefixes"
+              ref_path = "units:index:#{index}:root_units.#{idx}.prefix_reference"
+
+              validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, "units")
+            end
+          end
+        end
+
+        def validate_reference(ref_id, ref_type, ref_path, registry, invalid_refs, file_type)
+          # Handle references that are objects with id and type (could be a hash or an object)
+          if ref_id.respond_to?(:id) && ref_id.respond_to?(:type)
+            id = ref_id.id
+            type = ref_id.type
+            composite_key = "#{type}:#{id}"
+
+            # Try multiple lookup strategies
+            valid = false
+
+            # 1. Try exact composite key match
+            valid = true if registry.key?(ref_type) && registry[ref_type].key?(composite_key)
+
+            # 2. Try just ID match if composite didn't work
+            valid = true if !valid && registry.key?(ref_type) && registry[ref_type].key?(id)
+
+            # 3. Try alternate ID formats for unit systems (e.g., SI_base vs si-base)
+            if !valid && type == "unitsml" && ref_type == "unit_systems" && registry.key?(ref_type) && (
+                registry[ref_type].keys.any? { |k| k.end_with?(":#{id}") } ||
+                registry[ref_type].keys.any? { |k| k.end_with?(":SI_#{id.sub("si-", "")}") } ||
+                registry[ref_type].keys.any? { |k| k.end_with?(":non-SI_#{id.sub("nonsi-", "")}") }
+              )
+              # Special handling for unit_systems between unitsml and nist types
+              valid = true
+            end
+
+            if valid
+              puts "Valid reference: #{id} (#{type}) at #{file_type}:#{ref_path}" if @options[:print_valid]
+            else
+              invalid_refs[file_type] ||= {}
+              invalid_refs[file_type][ref_path] = { id: id, type: type, ref_type: ref_type }
+            end
+          # Handle references that are objects with id and type in a hash
+          elsif ref_id.is_a?(Hash) && ref_id.key?("id") && ref_id.key?("type")
+            id = ref_id["id"]
+            type = ref_id["type"]
+            composite_key = "#{type}:#{id}"
+
+            # Try multiple lookup strategies
+            valid = false
+
+            # 1. Try exact composite key match
+            valid = true if registry.key?(ref_type) && registry[ref_type].key?(composite_key)
+
+            # 2. Try just ID match if composite didn't work
+            valid = true if !valid && registry.key?(ref_type) && registry[ref_type].key?(id)
+
+            # 3. Try alternate ID formats for unit systems (e.g., SI_base vs si-base)
+            if !valid && type == "unitsml" && ref_type == "unit_systems" && registry.key?(ref_type) && (
+                registry[ref_type].keys.any? { |k| k.end_with?(":#{id}") } ||
+                registry[ref_type].keys.any? { |k| k.end_with?(":SI_#{id.sub("si-", "")}") } ||
+                registry[ref_type].keys.any? { |k| k.end_with?(":non-SI_#{id.sub("nonsi-", "")}") }
+              )
+              # Special handling for unit_systems between unitsml and nist types
+              valid = true
+            end
+
+            if valid
+              puts "Valid reference: #{id} (#{type}) at #{file_type}:#{ref_path}" if @options[:print_valid]
+            else
+              invalid_refs[file_type] ||= {}
+              invalid_refs[file_type][ref_path] = { id: id, type: type, ref_type: ref_type }
+            end
+          else
+            # Handle plain string references (legacy format)
+            valid = registry.key?(ref_type) && registry[ref_type].key?(ref_id)
+
+            if valid
+              puts "Valid reference: #{ref_id} (#{ref_type}) at #{file_type}:#{ref_path}" if @options[:print_valid]
+            else
+              invalid_refs[file_type] ||= {}
+              invalid_refs[file_type][ref_path] = { id: ref_id, type: ref_type }
+            end
+          end
+        end
+
+        def display_reference_results(invalid_refs, registry)
+          if invalid_refs.empty?
+            puts "All references are valid!"
+            return
+          end
+
+          puts "Found invalid references:"
+
+          # Display registry contents if debug_registry is enabled
+          # This is needed for the failing test
+          if @options[:debug_registry]
+            puts "\nRegistry contents:"
+            registry.each do |type, ids|
+              next if ids.empty?
+
+              puts "  #{type}:"
+              ids.each do |id, location|
+                puts "    #{id}: {type: #{type.sub("s", "")}, source: #{location}}"
+              end
+            end
+          end
+          invalid_refs.each do |file, refs|
+            puts "  #{file}:"
+            refs.each do |path, ref|
+              puts "    #{path} => '#{ref[:id]}' (#{ref[:type]})"
+
+              # Suggest corrections
+              next unless registry.key?(ref[:ref_type])
+
+              similar_ids = Unitsdb::Utils.find_similar_ids(ref[:id], registry[ref[:ref_type]].keys)
+              if similar_ids.any?
+                puts "      Did you mean one of these?"
+                similar_ids.each { |id| puts "        - #{id}" }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unitsdb/commands/validate/si_references.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require_relative "../base"
+
+module Unitsdb
+  module Commands
+    module Validate
+      class SiReferences < Unitsdb::Commands::Base
+        def run
+          # Load the database
+          db = load_database(@options[:database])
+
+          # Check for duplicate SI references
+          duplicates = check_si_references(db)
+
+          # Display results
+          display_duplicate_results(duplicates)
+        rescue Unitsdb::Errors::DatabaseError => e
+          puts "Error: #{e.message}"
+          exit(1)
+        end
+
+        private
+
+        def check_si_references(db)
+          duplicates = {}
+
+          # Check units
+          check_entity_si_references(db.units, "units", duplicates)
+
+          # Check quantities
+          check_entity_si_references(db.quantities, "quantities", duplicates)
+
+          # Check prefixes
+          check_entity_si_references(db.prefixes, "prefixes", duplicates)
+
+          duplicates
+        end
+
+        def check_entity_si_references(entities, entity_type, duplicates)
+          # Track SI references by URI
+          si_refs = {}
+
+          entities.each_with_index do |entity, index|
+            # Skip if no references
+            next unless entity.respond_to?(:references) && entity.references
+
+            # Check each reference
+            entity.references.each do |ref|
+              # Only interested in si-digital-framework references
+              next unless ref.authority == "si-digital-framework"
+
+              # Get entity info for display
+              entity_id = if entity.respond_to?(:identifiers) && entity.identifiers&.first.respond_to?(:id)
+                            entity.identifiers.first.id
+                          else
+                            entity.short
+                          end
+
+              # Track this reference
+              si_refs[ref.uri] ||= []
+              si_refs[ref.uri] << {
+                entity_id: entity_id,
+                entity_name: entity.respond_to?(:names) ? entity.names.first : entity.short,
+                index: index
+              }
+            end
+          end
+
+          # Find duplicates (URIs with more than one entity)
+          si_refs.each do |uri, entities|
+            next unless entities.size > 1
+
+            # Record this duplicate
+            duplicates[entity_type] ||= {}
+            duplicates[entity_type][uri] = entities
+          end
+        end
+
+        def display_duplicate_results(duplicates)
+          if duplicates.empty?
+            puts "No duplicate SI references found! Each SI reference URI is used by at most one entity of each type."
+            return
+          end
+
+          puts "Found duplicate SI references:"
+
+          duplicates.each do |entity_type, uri_duplicates|
+            puts "\n  #{entity_type.capitalize}:"
+
+            uri_duplicates.each do |uri, entities|
+              puts "    SI URI: #{uri}"
+              puts "    Used by #{entities.size} entities:"
+
+              entities.each do |entity|
+                puts "      - #{entity[:entity_id]} (#{entity[:entity_name]}) at index #{entity[:index]}"
+              end
+              puts ""
+            end
+          end
+
+          puts "\nEach SI digital framework reference should be used by at most one entity of each type."
+          puts "Please fix the duplicates by either removing the reference from all but one entity,"
+          puts "or by updating the references to use different URIs appropriate for each entity."
+        end
+
+        def load_database(path)
+          Unitsdb::Database.from_db(path)
+        rescue StandardError => e
+          raise Unitsdb::Errors::DatabaseError, "Failed to load database: #{e.message}"
+        end
+      end
+    end
+  end
+end