opencdd 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c582aa6a0bf79bedf583a586f97c1c308852b53d4efc9972b3bb883a2be425fc
-  data.tar.gz: 1195b6615ee045c3c1ac63f237ccdb930d80838db9dbe33677216e271748accc
+  metadata.gz: 907feca38dccee34a74c306983cdbe506bf3b18843640000501e7482cea75414
+  data.tar.gz: 2581c9dfac24e7bb1b8fcb973ef311854d8096947daf9a6d01ca726e672d2612
 SHA512:
-  metadata.gz: 1cbdf2e8832ccd86b95fcfc6789d8d9dfe4be656a27f7b4cac4cb854ea6f224153f5aa8cca521362f364954e123e0c50d3424da3daf031cd73f1974823794cb9
-  data.tar.gz: 072c326fba5a5807249b7ea79eda95c31fc42384c80de750dba20c741de0601c8c67afe61e9b58a27e594797c68b12cbb38b4166c0f56ce547130e610625c80c
+  metadata.gz: eb42bba181fa6e5802a6c3ed1ea46dce2d7ee9197dea2f929bd8b2e9565826cdd19dc503ba58a231b5555208244e5d8e084b92199944d8b7d30aa1e2e6879c2f
+  data.tar.gz: 74ab77f45f8365a433a638414c982d092140b07f9084fbb97ea94d846cc60a0c3787a29a2180d729a424715896514008c6ec0b2d5e9060ca8b3c60197d09f2fb

data/bin/lint-no-raw-mdc

@@ -1,4 +1,6 @@
-#!/usr/bin/env bash
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+#
 # lint-no-raw-mdc — fail if any raw MDC_P### / MDC_C### / EXT_P### /
 # EXT_C### / CIM_P### string literal appears in lib/opencdd/ outside
 # the designated SSOT files.
@@ -14,29 +16,50 @@
 #
 # Comment lines (lines whose first non-whitespace character is '#')
 # are also exempt.
-#
-set -euo pipefail
-
-excludes=(
-  ':!lib/opencdd/property_ids.rb'
-  ':!lib/opencdd/meta_class.rb'
-  ':!lib/opencdd/alias_table.rb'
-  ':!lib/opencdd/parcel/sheet_schema.rb'
-  ':!lib/opencdd/cddal/generated_parser.rb'
-)
-
-# Collect hits, filtering out Ruby comment-only lines.
-hits=$(git grep -nE '"(MDC_[CP][0-9]+(_[0-9]+)?|EXT_[CP][0-9]+|CIM_P[0-9]+)"' \
-  -- 'lib/opencdd/**/*.rb' "${excludes[@]}" 2>/dev/null \
-  | grep -vE '^[^:]+:[0-9]+:\s*#' \
-  || true)
-
-if [ -n "$hits" ]; then
-  echo "ERROR: raw MDC/EXT/CIM literal found outside designated SSOT files." >&2
-  echo "Use Opencdd::PropertyIds::REGISTRY / Opencdd::MetaClasses::REGISTRY instead." >&2
-  echo "" >&2
-  echo "$hits" >&2
+
+require "open3"
+
+EXCLUDED_PATHS = %w[
+  lib/opencdd/property_ids.rb
+  lib/opencdd/meta_class.rb
+  lib/opencdd/alias_table.rb
+  lib/opencdd/parcel/sheet_schema.rb
+  lib/opencdd/cddal/generated_parser.rb
+].freeze
+
+# Match "MDC_P###", "MDC_C###", "EXT_P###", "EXT_C###", "CIM_P###"
+# with optional _### sub-ids, as a double-quoted Ruby string literal.
+# Source string is POSIX ERE (not Ruby regex) — git grep -E requires it.
+LITERAL_PATTERN_SOURCE = '"(MDC_[CP][0-9]+(_[0-9]+)?|EXT_[CP][0-9]+|CIM_P[0-9]+)"'.freeze
+
+# Each git grep hit looks like: <path>:<lineno>:<content>
+# Skip comment-only lines (content's first non-space char is '#').
+HIT_LINE_PATTERN = /\A(?<path>[^:]+):(?<lineno>\d+):(?<content>.*)\z/m.freeze
+
+# lib/opencdd/**/*.rb minus the excluded SSOT files.
+pathspecs = ["lib/opencdd/**/*.rb"].concat(EXCLUDED_PATHS.map { |p| ":!#{p}" })
+
+cmd = ["git", "grep", "-n", "-E", LITERAL_PATTERN_SOURCE] + ["--"] + pathspecs
+stdout, _stderr, status = Open3.capture3(*cmd)
+
+# git grep returns 1 when there are no matches, which is the success
+# case. Treat anything other than 0/1 as a real failure.
+unless [0, 1].include?(status.exitstatus)
+  warn "ERROR: git grep failed (exit #{status.exitstatus})"
+  exit 2
+end
+
+hits = stdout.each_line.map(&:chomp).reject do |line|
+  match = HIT_LINE_PATTERN.match(line)
+  match && match[:content].lstrip.start_with?("#")
+end
+
+if hits.any?
+  warn "ERROR: raw MDC/EXT/CIM literal found outside designated SSOT files."
+  warn "Use Opencdd::PropertyIds::REGISTRY / Opencdd::MetaClasses::REGISTRY instead."
+  warn ""
+  hits.each { |hit| warn hit }
   exit 1
-fi
+end
 
 exit 0

data/lib/cdd.rb

@@ -6,6 +6,6 @@
 require "opencdd"
 
 # Alias for backward compatibility. The module was renamed from
-# Cdd to Opencdd, but many callers (cdd-data Rakefile, specs, etc.)
+# Cdd to Opencdd, but many callers (data-private Rakefile, specs, etc.)
 # still reference Cdd::. This alias lets them keep working.
 Cdd = Opencdd unless defined?(Cdd)

data/lib/opencdd/cddal/serializer.rb

@@ -174,7 +174,6 @@ module Opencdd
       def format_value(value)
         s = value.to_s
         return quote_string(s) if string_literal?(s)
-        return format_set(s) if set_like?(s)
         s
       end
 
@@ -194,15 +193,6 @@ module Opencdd
         "\"#{escaped}\""
       end
 
-      def set_like?(s)
-        s.start_with?("(") && s.end_with?(")")
-      end
-
-      def format_set(s)
-        elements = Opencdd::StructuredValues.unwrap_and_split(s)
-        "{ #{elements.join(', ')} }"
-      end
-
       def symbol_name_for(entity)
         code = entity.code&.to_s
         return code if code && !code.empty?

data/lib/opencdd/languages.rb

@@ -12,6 +12,12 @@ module Opencdd
   # "what languages does this dictionary have?" — used by the browser
   # to render a language switcher and by the data pipeline to report
   # language coverage.
+  #
+  # This is a model object. It does not normalize language codes:
+  # source-format quirks (e.g. IEC CDD .xls files using "jp" for
+  # Japanese) are normalized at the SheetSchema ingestion boundary
+  # via +Opencdd::Parcel::LanguageAliases+, so every consumer of
+  # this class can assume ISO 639-1 codes unconditionally.
   class Languages
     attr_reader :source, :translations
 
@@ -58,7 +64,7 @@ module Opencdd
     def self.from_properties(properties, default_source: "en")
       langs = properties.keys.each_with_object(Set.new) do |key, acc|
         next unless key.include?(".")
-        prefix, lang = key.split(".", 2)
+        _, lang = key.split(".", 2)
         next unless lang =~ /\A[a-z]{2}(-[a-z0-9]+)?\z/i
         acc << lang
       end
@@ -67,4 +73,4 @@ module Opencdd
       new(source: source, translations: translations)
     end
   end
-end
+end

data/lib/opencdd/parcel/language_aliases.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Maps non-conformant language codes found in IEC CDD source .xls
+    # exports to their ISO 639-1 equivalents.
+    #
+    # This is a Parcel-layer concern, not a model concern: the aliases
+    # exist because one specific source format (IEC CDD XLS) uses
+    # non-standard codes in its property ID suffixes and directive
+    # rows. The +Languages+ value object — and every downstream
+    # consumer (JSON wire format, browser, TS model) — speaks ISO
+    # 639-1 unconditionally.
+    #
+    # The normalization happens at the SheetSchema ingestion
+    # boundary so downstream code never sees an alias. SheetSchema
+    # also exposes the set of original-to-canonical mappings it
+    # applied via +#normalized_language_codes+ for data-quality
+    # reporting.
+    module LanguageAliases
+      ALIASES = {
+        "jp" => "ja",
+      }.freeze
+
+      # Returns the ISO 639-1 form of +code+, or the input unchanged
+      # if it is already standard. Pure function: no I/O, no side
+      # effects.
+      def self.normalize(code)
+        return code if code.nil?
+        s = code.to_s.strip
+        return s if s.empty?
+        ALIASES.fetch(s, s)
+      end
+
+      # True when +code+ would be rewritten by +normalize+.
+      def self.alias?(code)
+        return false if code.nil?
+        ALIASES.key?(code.to_s.strip)
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/sheet_schema.rb

@@ -22,6 +22,8 @@ module Opencdd
       # Canonicalize a Parcel column ID. Splits language tags
       # (<id>.<lang>) so they round-trip cleanly. Returns the
       # canonical ID (or the input unchanged if no mapping exists).
+      # Language codes are normalized to ISO 639-1 via
+      # +Opencdd::Parcel::LanguageAliases.normalize+.
       def self.canonical_id(raw_id)
         return nil if raw_id.nil?
         s = raw_id.to_s.strip
@@ -30,7 +32,7 @@ module Opencdd
         base = match ? match.pre_match : s
         lang = match && match[:lang]
         canonical = VARIANT_TO_CANONICAL[base] || base
-        lang ? "#{canonical}.#{lang}" : canonical
+        lang ? "#{canonical}.#{Opencdd::Parcel::LanguageAliases.normalize(lang)}" : canonical
       end
 
       DIRECTIVE_ROWS = %w[
@@ -110,7 +112,7 @@ module Opencdd
 
       DIRECTIVE_ROW_PREFIX = "#".freeze
 
-      attr_reader :columns, :columns_by_id, :column_directives
+      attr_reader :columns, :columns_by_id, :column_directives, :normalized_language_codes
 
       def initialize
         @columns = []
@@ -194,6 +196,7 @@ module Opencdd
           @columns_by_id[col.property_id] = col
         end
 
+        @normalized_language_codes = compute_normalized_language_codes.freeze
         freeze
       end
 
@@ -260,11 +263,34 @@ module Opencdd
           next if v.nil?
           s = v.to_s.strip
           next if s.empty?
+          lang = Opencdd::Parcel::LanguageAliases.normalize(lang)
           lang = "en" if lang.nil? || lang.empty?
           h[lang] = s
         end
         h
       end
+
+      # Audit trail of language-code normalizations applied at this
+      # schema's ingestion boundary. Returns a frozen Hash mapping each
+      # original (non-conformant) language code seen in the source to
+      # its ISO 639-1 equivalent. Empty when the source spoke ISO.
+      def compute_normalized_language_codes
+        seen = {}
+        @column_directives.each_key do |key|
+          _, lang = key.to_s.split(".", 2)
+          next if lang.nil? || lang.empty?
+          normalized = Opencdd::Parcel::LanguageAliases.normalize(lang)
+          seen[lang] = normalized if normalized != lang
+        end
+        @columns.each do |col|
+          next if col.raw_property_id.nil?
+          _, lang = col.raw_property_id.split(".", 2)
+          next if lang.nil? || lang.empty?
+          normalized = Opencdd::Parcel::LanguageAliases.normalize(lang)
+          seen[lang] = normalized if normalized != lang
+        end
+        seen
+      end
     end
   end
 end

data/lib/opencdd/parcel/writer.rb

@@ -147,7 +147,11 @@ module Opencdd
           rows = emitter.emit(built.sheet, built.entities)
           workbook.add_worksheet(name: built.sheet.name) do |ws|
             rows.each do |row|
-              emitted = ws.add_row(row)
+              # All property values are strings per IEC 61360 wire format.
+              # Force string cell type so values like "001" survive round-trip
+              # (otherwise caxlsx auto-types numeric-looking strings and roo
+              # reads them back as integers, breaking semantically_equal?).
+              emitted = ws.add_row(row, types: :string)
               emitted.hidden = true if row_hidden?(row, hidden_directives)
             end
           end

data/lib/opencdd/parcel.rb

@@ -7,6 +7,7 @@ module Opencdd
 
     autoload :Metadata,        "opencdd/parcel/metadata"
     autoload :SheetSchema,     "opencdd/parcel/sheet_schema"
+    autoload :LanguageAliases, "opencdd/parcel/language_aliases"
     autoload :Sheet,           "opencdd/parcel/sheet"
     autoload :Workbook,        "opencdd/parcel/workbook"
     autoload :WorkbookReader,  "opencdd/parcel/workbook_reader"

data/lib/opencdd/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Opencdd
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opencdd
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - OpenCDD contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-24 00:00:00.000000000 Z
+date: 2026-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: roo
@@ -263,6 +263,7 @@ files:
 - lib/opencdd/parcel/csv_writer.rb
 - lib/opencdd/parcel/entity_manifest.rb
 - lib/opencdd/parcel/flat_dir_reader.rb
+- lib/opencdd/parcel/language_aliases.rb
 - lib/opencdd/parcel/layout_detector.rb
 - lib/opencdd/parcel/metadata.rb
 - lib/opencdd/parcel/referenced_irdis.rb