opencdd 0.1.1

data/lib/opencdd/model/yaml_entity.rb

@@ -0,0 +1,196 @@
+# frozen_string: true
+
+require "lutaml/model"
+
+module Opencdd
+  module Model
+    # CDD-native YAML entity. Every attribute uses the semantic name
+    # from the CDD ontology (preferred_name, superclass, class_type)
+    # rather than the IEC 62656-1 wire-format key (MDC_P004, MDC_P010,
+    # MDC_P011). Multilingual fields are Hash<String, String>;
+    # collection fields are Array<String>.
+    #
+    # This is the canonical YAML shape for a CDD entity:
+    #
+    #   ---
+    #   irdi: 0112/2///61360_4#AAA001
+    #   type: class
+    #   code: AAA001
+    #   preferred_name:
+    #     en: Vehicle
+    #     fr: Véhicule
+    #   class_type: ITEM_CLASS
+    #   superclass: UNIVERSE
+    #   applicable_properties:
+    #     - vehicle_length
+    #     - vehicle_weight
+    #
+    # Conversion between YamlEntity and the existing Entity model
+    # is via #from_entity and #to_entity, which bridge the semantic
+    # attributes to the raw properties Hash.
+    class YamlEntity < Lutaml::Model::Serializable
+      # ── Identity ────────────────────────────────────────────
+      attribute :irdi, :string
+      attribute :type, :string               # :class, :property, :unit, ...
+      attribute :code, :string
+      attribute :guid, :string
+
+      # ── Multilingual text (Hash<lang, text>) ───────────────
+      attribute :preferred_name, :hash
+      attribute :short_name, :hash
+      attribute :definition, :hash
+      attribute :note, :hash
+      attribute :remark, :hash
+
+      # ── Version ─────────────────────────────────────────────
+      attribute :version, :string
+      attribute :revision, :string
+
+      # ── Class-specific ──────────────────────────────────────
+      attribute :class_type, :string          # ITEM_CLASS, CATEGORICAL_CLASS
+      attribute :superclass, :string           # IRDI or symbolic name
+      attribute :is_case_of, :string, collection: true
+      attribute :applicable_properties, :string, collection: true
+      attribute :imported_properties, :string, collection: true
+      attribute :sub_class_selection, :string, collection: true
+
+      # ── Property-specific ───────────────────────────────────
+      attribute :data_type, :string           # REAL_TYPE, CLASS_REFERENCE(X)
+      attribute :value_format, :string
+      attribute :definition_class, :string    # IRDI or symbolic name
+      attribute :unit, :string
+      attribute :condition, :string
+      attribute :property_data_element_type, :string
+
+      # ── Unit-specific ───────────────────────────────────────
+      attribute :unit_structure, :string
+      attribute :unit_text, :string
+
+      # ── Value-list-specific ─────────────────────────────────
+      attribute :enumerated_values, :string, collection: true
+      attribute :list_type, :string
+
+      # ── Relation-specific ───────────────────────────────────
+      attribute :relation_type, :string
+      attribute :domain, :string, collection: true
+      attribute :codomain, :string
+
+      # ── Catch-all for unknown properties (lossless) ────────
+      attribute :extra, :hash
+
+      # ── Conversion: Entity → YamlEntity ────────────────────
+      def self.from_entity(entity)
+        attrs = {
+          irdi: entity.irdi&.to_s,
+          type: entity.type&.to_s,
+          code: entity.code,
+          guid: entity[Opencdd::PropertyIds::MDC_P066],
+          version: entity[Opencdd::PropertyIds::MDC_P002_1],
+          revision: entity[Opencdd::PropertyIds::MDC_P002_2],
+        }
+
+        # Multilingual fields
+        attrs[:preferred_name] = extract_ml(entity, Opencdd::PropertyIds::MDC_P004)
+        attrs[:short_name]     = extract_ml(entity, Opencdd::PropertyIds::MDC_P005)
+        attrs[:definition]     = extract_ml(entity, Opencdd::PropertyIds::MDC_P006)
+        attrs[:note]           = extract_ml(entity, Opencdd::PropertyIds::MDC_P008)
+        attrs[:remark]         = extract_ml(entity, Opencdd::PropertyIds::MDC_P009)
+
+        # Class-specific
+        if entity.type == :class
+          ct = entity.read_field(:class_type)
+          attrs[:class_type] = ct&.to_s
+          attrs[:superclass] = entity.read_field(:superclass_irdi)&.to_s
+          attrs[:is_case_of] = entity.read_field(:is_case_of_irdis)&.map(&:to_s) || []
+          attrs[:applicable_properties] = entity.read_field(:applicable_property_irdis)&.map(&:to_s) || []
+          attrs[:imported_properties] = entity.read_field(:imported_property_irdis)&.map(&:to_s) || []
+          attrs[:sub_class_selection] = entity.read_field(:sub_class_selection_irdis)&.map(&:to_s) || []
+        end
+
+        # Property-specific
+        if entity.type == :property
+          dt = entity.read_field(:parsed_data_type)
+          attrs[:data_type] = dt&.to_s
+          attrs[:value_format] = entity.read_field(:parsed_value_format)&.to_s
+          dc = entity.read_field(:definition_class_irdi)
+          attrs[:definition_class] = dc&.to_s
+          ui = entity.read_field(:unit_irdi)
+          attrs[:unit] = ui&.to_s
+          attrs[:condition] = entity.read_field(:condition)&.to_s
+          attrs[:property_data_element_type] = entity.read_field(:property_data_element_type)&.to_s
+        end
+
+        new(**attrs.compact)
+      end
+
+      # ── Conversion: YamlEntity → Entity ────────────────────
+      def to_entity(database = nil)
+        props = {}
+
+        props[Opencdd::PropertyIds::MDC_P066] = guid if guid
+        props[Opencdd::PropertyIds::MDC_P002_1] = version if version
+        props[Opencdd::PropertyIds::MDC_P002_2] = revision if revision
+
+        # Multilingual
+        merge_ml_into(props, Opencdd::PropertyIds::MDC_P004, preferred_name)
+        merge_ml_into(props, Opencdd::PropertyIds::MDC_P005, short_name)
+        merge_ml_into(props, Opencdd::PropertyIds::MDC_P006, definition)
+        merge_ml_into(props, Opencdd::PropertyIds::MDC_P008, note)
+        merge_ml_into(props, Opencdd::PropertyIds::MDC_P009, remark)
+
+        # Class-specific
+        props[Opencdd::PropertyIds::MDC_P011] = class_type if class_type
+        props[Opencdd::PropertyIds::MDC_P010] = superclass if superclass
+        props[Opencdd::PropertyIds::MDC_P013] = "{#{is_case_of.join(",")}}" if is_case_of&.any?
+        props[Opencdd::PropertyIds::MDC_P014] = "{#{applicable_properties.join(",")}}" if applicable_properties&.any?
+        props[Opencdd::PropertyIds::MDC_P090] = "{#{imported_properties.join(",")}}" if imported_properties&.any?
+        props[Opencdd::PropertyIds::MDC_P016] = "{#{sub_class_selection.join(",")}}" if sub_class_selection&.any?
+
+        # Property-specific
+        props[Opencdd::PropertyIds::MDC_P022] = data_type if data_type
+        props[Opencdd::PropertyIds::MDC_P024] = value_format if value_format
+        props[Opencdd::PropertyIds::MDC_P021] = definition_class if definition_class
+        props[Opencdd::PropertyIds::MDC_P041] = unit if unit
+        props[Opencdd::PropertyIds::MDC_P028] = condition if condition
+        props[Opencdd::PropertyIds::MDC_P020] = property_data_element_type if property_data_element_type
+
+        # Determine entity class from type
+        entity_class = Opencdd::MetaClasses.entity_class_for_type(type&.to_sym) || Opencdd::Klass
+        meta_code = Opencdd::MetaClasses.meta_class_for_type(type&.to_sym) || "MDC_C002"
+
+        parsed_irdi = irdi ? Opencdd::IRDI.parse(irdi) : nil
+        parsed_meta = Opencdd::IRDI.parse(meta_code)
+
+        entity_class.new(
+          irdi: parsed_irdi,
+          properties: props,
+          meta_class_irdi: parsed_meta,
+        )
+      end
+
+      # ── Helpers ─────────────────────────────────────────────
+
+      def self.extract_ml(entity, pid)
+        result = {}
+        entity.properties.each do |key, val|
+          next unless key.to_s.start_with?("#{pid}.")
+          lang = key.to_s.split(".", 2).last
+          result[lang] = val if val
+        end
+        result.empty? ? nil : result
+      end
+      private_class_method :extract_ml
+
+      def self.merge_ml(props, pid, hash)
+        return unless hash&.any?
+        hash.each { |lang, val| props["#{pid}.#{lang}"] = val }
+      end
+
+      private
+
+      def merge_ml_into(props, pid, hash)
+        self.class.merge_ml(props, pid, hash)
+      end
+    end
+  end
+end

data/lib/opencdd/model.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Opencdd
+  # CDD-native model layer backed by Lutaml::Model. Provides typed
+  # attributes with semantic CDD names (preferred_name, superclass,
+  # class_type) — not wire-format keys (MDC_P004, MDC_P010). YAML
+  # is the canonical persistence format per user decision (2026-07-13).
+  module Model
+    autoload :YamlEntity,   "opencdd/model/yaml_entity"
+    autoload :YamlDatabase, "opencdd/model/yaml_database"
+    autoload :EntityStore,  "opencdd/model/entity_store"
+  end
+end

data/lib/opencdd/parcel/csv_reader.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module Opencdd
+  module Parcel
+    module CsvReader
+      DEFAULT_ENCODING = "UTF-8".freeze
+
+      def self.read(path, meta_class_irdi:, name: nil, col_sep: ",", encoding: DEFAULT_ENCODING)
+        rows = ::CSV.read(path, col_sep: col_sep, encoding: encoding)
+        rows = rows.reject { |r| r.nil? || r.all? { |c| c.nil? || c.to_s.strip.empty? } }
+        meta_code = extract_meta_code(meta_class_irdi)
+        scaffold = Opencdd::Parcel::Sheet.scaffold(
+          meta_class_irdi: meta_code,
+          parcel_id: "IMPORT",
+        )
+        raw_rows = rows.map { |r| [nil, *r] }
+        Opencdd::Parcel::Sheet.new(
+          name: name || File.basename(path.to_s, ".*"),
+          metadata: scaffold.metadata,
+          schema: scaffold.schema,
+          raw_rows: raw_rows,
+        )
+      end
+
+      def self.extract_meta_code(value)
+        return value if value.nil?
+        s = value.to_s
+        s.include?("#") ? s.split("#").last : s
+      end
+      private_class_method :extract_meta_code
+    end
+  end
+end

data/lib/opencdd/parcel/csv_writer.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "csv"
+require "stringio"
+
+module Opencdd
+  module Parcel
+    module CsvWriter
+      DEFAULT_ENCODING = "UTF-8".freeze
+
+      def self.write_sheet(sheet, entities, target, source_language: SheetEmitter::DEFAULT_SOURCE_LANGUAGE, encoding: DEFAULT_ENCODING, write_bom: false)
+        emitter = SheetEmitter.new(source_language: source_language)
+        rows = emitter.emit_data_rows(sheet, entities)
+        data_rows = rows.map { |row| row.drop(1) }
+        write_csv(target, data_rows, encoding: encoding, write_bom: write_bom)
+        target
+      end
+
+      def self.write_workbook(database, dir, parcel_id:, source_language: SheetEmitter::DEFAULT_SOURCE_LANGUAGE, encoding: DEFAULT_ENCODING, write_bom: false)
+        require "fileutils"
+        FileUtils.mkdir_p(dir)
+
+        built_sheets = build_sheets(database, parcel_id, source_language)
+        paths = []
+        built_sheets.each do |built|
+          filename = "#{parcel_id}_#{type_label(built.type)}.csv"
+          path = File.join(dir, filename)
+          write_sheet(built.sheet, built.entities, path,
+                      source_language: source_language, encoding: encoding, write_bom: write_bom)
+          paths << path
+        end
+        paths
+      end
+
+      class BuiltSheet < Struct.new(:sheet, :entities, :type, keyword_init: true); end
+
+      def self.build_sheets(database, parcel_id, source_language)
+        if database.workbooks.any?
+          source_workbook = database.workbooks.first
+          sheets_by_type = source_workbook.sheets.select { |s| s.type }.group_by(&:type)
+          sheets_by_type.map do |type, sheets|
+            primary = sheets.first
+            entities = database.entities_of_type(primary.type)
+            BuiltSheet.new(sheet: primary, entities: entities, type: type)
+          end
+        else
+          types = %i[class property value_list value_term unit relation view_control]
+          types.filter_map do |type|
+            meta_code = Opencdd::MetaClasses.meta_class_for_type(type)
+            next nil unless meta_code
+            sheet = Opencdd::Parcel::Sheet.scaffold(
+              meta_class_irdi: meta_code,
+              parcel_id: parcel_id,
+              source_language: source_language,
+            )
+            entities = database.entities_of_type(type)
+            BuiltSheet.new(sheet: sheet, entities: entities, type: type)
+          end
+        end
+      end
+      private_class_method :build_sheets
+
+      def self.type_label(type)
+        {
+          class:        "CLASS",
+          property:     "PROPERTY",
+          value_list:   "ENUM",
+          value_term:   "TERMINOLOGY",
+          unit:         "UoM",
+          relation:     "RELATION",
+          view_control: "VIEWCONTROL",
+        }.fetch(type, type.to_s.upcase)
+      end
+      private_class_method :type_label
+
+      def self.write_csv(target, rows, encoding:, write_bom:)
+        case target
+        when String then write_csv_to_path(target, rows, encoding, write_bom)
+        when Pathname then write_csv_to_path(target.to_s, rows, encoding, write_bom)
+        when IO, StringIO then write_csv_to_io(target, rows, encoding, write_bom)
+        else
+          raise ArgumentError, "unsupported CSV target: #{target.inspect} (expected String, Pathname, IO, or StringIO)"
+        end
+      end
+      private_class_method :write_csv
+
+      def self.write_csv_to_path(path, rows, encoding, write_bom)
+        File.open(path, "w:#{encoding}") do |f|
+          write_bom_prefix(f, encoding) if write_bom
+          write_csv_to_io(f, rows, encoding, write_bom)
+        end
+      end
+      private_class_method :write_csv_to_path
+
+      def self.write_csv_to_io(io, rows, _encoding, _write_bom)
+        rows.each do |row|
+          line = CSV.generate_line(row || [])
+          io.write(line)
+        end
+      end
+      private_class_method :write_csv_to_io
+
+      def self.write_bom_prefix(io, encoding)
+        bom = case encoding.upcase
+              when "UTF-8", "UTF8" then "\xEF\xBB\xBF"
+              when "UTF-16LE" then "\xFF\xFE"
+              when "UTF-16BE" then "\xFE\xFF"
+              end
+        io.write(bom) if bom
+      end
+      private_class_method :write_bom_prefix
+    end
+  end
+end

data/lib/opencdd/parcel/entity_manifest.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Opencdd
+  module Parcel
+    # Reads the harvester's <code>_entity.json</code> sidecar format
+    # and converts it into Opencdd::Entity::VersionHistory entries
+    # and stub entities for cross-reference resolution.
+    #
+    # Extracted from ShardedDirReader so the JSON-shape concern has
+    # its own home and can be tested with a hash fixture instead of
+    # requiring a full directory tree on disk.
+    #
+    # The sidecar schema (produced by harvest/download_versions.py):
+    #
+    #   {
+    #     "irdi": "0112/2///62656_1#AAA001",
+    #     "entity_type": "class",                 # Symbol
+    #     "current_version_dir": "ABC123...",     # UNID subfolder name
+    #     "versions": [
+    #       { "version": 1, "revision": 0, "status": "Released",
+    #         "timestamp": "2024-01-15T10:23:00Z", "user": "...",
+    #         "change_request_id": "...", "unid": "...",
+    #         "is_current": true }
+    #     ]
+    #   }
+    class EntityManifest
+      attr_reader :data, :path
+
+      # Returns a manifest for +entity_dir+, or +nil+ when no
+      # +_entity.json+ lives there or the file is unparseable.
+      def self.read(entity_dir)
+        path = File.join(entity_dir, "_entity.json")
+        return nil unless File.file?(path)
+        data = JSON.parse(File.read(path))
+        new(data, path)
+      rescue JSON::ParserError
+        nil
+      end
+
+      def initialize(data, path = nil)
+        @data = data
+        @path = path
+      end
+
+      # IRDI of the entity this manifest describes, or +nil+.
+      def irdi
+        raw = @data["irdi"]
+        return nil if raw.nil?
+        Opencdd::IRDI.parse(raw.to_s)
+      end
+
+      # Symbol type (:class, :property, ...) or +nil+.
+      def entity_type
+        @data["entity_type"]&.to_sym
+      end
+
+      # Meta-class IRDI code (e.g. "MDC_C002"), or +nil+ if the
+      # type isn't registered.
+      def meta_class_code
+        Opencdd::MetaClasses.meta_class_for_type(entity_type)
+      end
+
+      # Name of the per-version subfolder that holds the current
+      # version's XLS exports. +nil+ for flat-layout manifests.
+      def current_version_dir
+        @data["current_version_dir"]
+      end
+
+      # Version history compiled from the +versions+ array.
+      # Empty when no versions are listed.
+      def version_history
+        vh = Opencdd::Entity::VersionHistory.new
+        Array(@data["versions"]).each do |v|
+          vh.attach(build_entry(v))
+        end
+        vh
+      end
+
+      def versions_empty?
+        Array(@data["versions"]).empty?
+      end
+
+      # Constructs a minimal stub entity from this manifest, so
+      # cross-references to entities that weren't loaded from a
+      # +.xls+ still resolve. Returns +nil+ if the manifest lacks
+      # an IRDI, type, or known meta-class.
+      def to_stub_entity
+        return nil unless irdi && meta_class_code
+        meta_class = Opencdd::MetaClasses.for(meta_class_code)
+        return nil unless meta_class&.entity_class
+
+        code_property_id = Opencdd::MetaClasses.code_property_id_for(meta_class_code)
+        props = code_property_id ? { code_property_id => @data["irdi"].to_s } : {}
+        meta_class.entity_class.new(
+          irdi: irdi,
+          properties: props,
+          meta_class_irdi: Opencdd::IRDI.parse(meta_class_code),
+        )
+      end
+
+      private
+
+      def build_entry(raw)
+        Opencdd::Entity::VersionHistory::Entry.new(
+          version:           raw["version"],
+          revision:          raw["revision"],
+          status:            raw["status"],
+          timestamp:         raw["timestamp"],
+          user:              raw["user"],
+          change_request_id: raw["change_request_id"],
+          unid:              raw["unid"],
+          is_current:        raw["is_current"],
+        )
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/flat_dir_reader.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Reads the legacy "flat directory" Parcel layout: one directory
+    # containing 1-6 +export_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|
+    # VALUETERMS)_*.xls+ files at the top level. This is the format
+    # cdd.iec.ch's "EXCEL format" export produces when downloaded
+    # per-type without sharding by class.
+    #
+    # For per-class sharded subdirs see +Opencdd::Parcel::ShardedDirReader+.
+    # For single .xlsx workbooks see +Opencdd::Parcel::WorkbookReader+.
+    class FlatDirReader
+      # FILE_PATTERN moved to Opencdd::Parcel::LayoutDetector (SSOT).
+      # Kept here as an alias for back-compat with callers that
+      # reference FlatDirReader::FILE_PATTERN directly.
+      FILE_PATTERN = Opencdd::Parcel::LayoutDetector::FILE_PATTERN
+
+      TYPE_BY_PREFIX = {
+        "CLASS"      => :class,
+        "PROPERTY"   => :property,
+        "RELATION"   => :relation,
+        "UNIT"       => :unit,
+        "VALUELIST"  => :value_list,
+        "VALUETERMS" => :value_term,
+      }.freeze
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def read_workbook
+        files = legacy_files
+        raise "No legacy export_*.xls files found in #{@path.inspect}" if files.empty?
+
+        sheets = files.map { |f| read_one(f) }.compact
+        sheetmap = files.map { |f| sheetmap_entry_for(f) }.compact
+
+        parcel_id = files.first && File.basename(files.first).split("_").last&.sub(/\.\w+\z/, "")
+        project = Opencdd::Parcel::Workbook::ProjectInfo.new(
+          project_id: "LOCAL",
+          parcel_id: parcel_id,
+          multi_language: "",
+          base_language: "en",
+        )
+
+        Opencdd::Parcel::Workbook.new(
+          sheets: sheets,
+          sheetmap: sheetmap,
+          project: project,
+          source_path: @path.to_s,
+        )
+      end
+
+      def load_into(database)
+        workbook = read_workbook
+        database.add_workbook(workbook)
+        database.finalize!
+        database
+      end
+
+      private
+
+      def legacy_files
+        if File.directory?(@path)
+          Dir.children(@path).sort.map { |f| File.join(@path, f) }
+            .select { |f| File.file?(f) && File.basename(f) =~ FILE_PATTERN }
+        elsif File.file?(@path) && File.basename(@path) =~ FILE_PATTERN
+          [@path.to_s]
+        else
+          []
+        end
+      end
+
+      def read_one(file)
+        prefix = File.basename(file).sub(/\Aexport_/, "").split("_", 2).first
+        type = TYPE_BY_PREFIX[prefix]
+        warn "Unknown legacy file prefix: #{file}" unless type
+        return nil unless type
+
+        sheet_name = File.basename(file, ".*")
+        rows = read_rows(file)
+        Opencdd::Parcel::Sheet.from_rows(rows.each, name: sheet_name)
+      end
+
+      def read_rows(file)
+        ext = File.extname(file).downcase
+        if ext == ".xls"
+          read_xls_rows(file)
+        else
+          read_xlsx_rows(file)
+        end
+      end
+
+      def read_xls_rows(file)
+        require "spreadsheet"
+        book = Spreadsheet.open(file)
+        sheet = book.worksheet(0) || book.worksheets.first
+        sheet.rows.map do |row|
+          row.to_a
+        end
+      end
+
+      def read_xlsx_rows(file)
+        require "roo"
+        roo = Roo::Spreadsheet.open(file, extension: :xlsx)
+        target = roo.sheets.first || "Export"
+        sheet = roo.sheet(target)
+        (1..sheet.last_row.to_i).map { |i| sheet.row(i) }
+      end
+
+      def sheetmap_entry_for(file)
+        prefix = File.basename(file).sub(/\Aexport_/, "").split("_", 2).first
+        type = TYPE_BY_PREFIX[prefix]
+        return nil unless type
+        meta_class_code = Opencdd::Parcel::TYPE_TO_META_CLASS[type]
+        return nil unless meta_class_code
+
+        Opencdd::Parcel::Workbook::SheetMapEntry.new(
+          project_id: "LOCAL",
+          parcel_id: nil,
+          class_irdi: Opencdd::IRDI.parse("0112/2///62656_1##{meta_class_code}"),
+          content_no: 0,
+          sheet_no: nil,
+          sheet_name: File.basename(file, ".*"),
+          type: prefix,
+          target: "",
+        )
+      end
+    end
+  end
+end