opencdd 0.1.1

data/lib/opencdd/parcel/layout_detector.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Single source of truth for Parcel on-disk layout detection.
+    #
+    # Three layouts exist in the wild:
+    #
+    # [legacy single]   one <code>export_*.xls</code> file per type
+    #                   (CLASS, PROPERTY, RELATION, UNIT, VALUELIST,
+    #                   VALUETERMS).
+    # [flat dir]        a directory containing those six files at the
+    #                   top level.
+    # [sharded dir]     a directory containing one subdir per class
+    #                   code, each subdir holding 1-6 +export_*.xls+
+    #                   files either directly (flat sharded) or under
+    #                   a UNID-named version subfolder (per-version
+    #                   sharded, identified by <code>_entity.json</code>).
+    #
+    # Previously <code>Reader</code>, <code>FlatDirReader</code>,
+    # <code>ShardedDirReader</code>, and <code>ScrapeVerifier</code>
+    # each held their own copies of the regexes and detection
+    # helpers. Layout drift meant patching N files. Now they all
+    # delegate here.
+    module LayoutDetector
+      # Class-code shape per IEC CDD conventions: 2-6 uppercase
+      # letters, 1-6 digits, optional alnum suffix.
+      CLASS_CODE_PATTERN = /\A[A-Z]{2,6}[0-9]{1,6}[A-Z0-9-]*\z/.freeze
+
+      # Harvester per-version subfolder name (Lotus Notes UNID).
+      UNID_PATTERN = /\A[0-9A-F]{32}\z/i.freeze
+
+      # Per-type export filename produced by cdd.iec.ch's download
+      # endpoint.
+      FILE_PATTERN = /\Aexport_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|VALUETERMS)_[^\.]+\.(xls|xlsx)\z/i.freeze
+
+      module_function
+
+      # True if +name+ looks like a class code (AAA001, UAC696, ...).
+      def class_code?(name)
+        return false if name.nil?
+        name.match?(CLASS_CODE_PATTERN)
+      end
+
+      # True if +filename+ is one of the harvester's export_* files.
+      def legacy_export?(filename)
+        return false if filename.nil?
+        filename.match?(FILE_PATTERN)
+      end
+
+      # True if +name+ is a UNID-named subfolder.
+      def unid?(name)
+        return false if name.nil?
+        name.match?(UNID_PATTERN)
+      end
+
+      # Returns the directory holding the active +export_*.xls+ files
+      # for a given class-code dir, or +nil+ when no XLS is present.
+      #
+      # Resolution order:
+      #   1. Per-version: +_entity.json+ names a UNID subfolder.
+      #   2. Per-version (no manifest): a single UNID subfolder with XLS.
+      #   3. Flat sharded: XLS files directly under +code_dir+.
+      def active_xls_dir(code_dir)
+        return nil unless File.directory?(code_dir)
+
+        # Per-version with manifest.
+        manifest = File.join(code_dir, "_entity.json")
+        if File.file?(manifest)
+          current = read_current_version_dir(manifest)
+          if current && unid?(current)
+            candidate = File.join(code_dir, current)
+            return candidate if File.directory?(candidate)
+          end
+        end
+
+        # Per-version without manifest: unique UNID subfolder with XLS.
+        unid_subdirs = Dir.children(code_dir)
+                          .select { |n| unid?(n) }
+                          .map { |n| File.join(code_dir, n) }
+                          .select { |p| File.directory?(p) }
+        if unid_subdirs.size == 1
+          active = unid_subdirs.first
+          return active if Dir.children(active).any? { |f| legacy_export?(f) }
+        end
+
+        # Flat sharded.
+        return code_dir if Dir.children(code_dir).any? { |f| legacy_export?(f) }
+
+        nil
+      end
+
+      # True if +path+ is a sharded class-code subdir (contains XLS
+      # directly or via the per-version layout).
+      def sharded_class_subdir?(path)
+        return false unless File.directory?(path)
+        return false unless class_code?(File.basename(path))
+        !active_xls_dir(path).nil?
+      end
+
+      # ── Internal ─────────────────────────────────────────────
+
+      def read_current_version_dir(manifest_path)
+        require "json"
+        data = JSON.parse(File.read(manifest_path))
+        data["current_version_dir"]
+      rescue JSON::ParserError
+        nil
+      rescue Errno::ENOENT
+        nil
+      end
+      private_class_method :read_current_version_dir
+    end
+  end
+end

data/lib/opencdd/parcel/metadata.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    class Metadata
+      DIRECTIVE_REGEX = /\A#([^:=\s]+)\s*:=\s*(.*)\z/
+
+      attr_reader :directives
+
+      def self.from_directive(cell)
+        return nil if cell.nil?
+        s = cell.to_s.strip
+        return nil if s.empty?
+        return nil unless DIRECTIVE_REGEX.match?(s)
+        new(s)
+      end
+
+      def initialize(source = nil)
+        @directives = {}
+        @meta_class_raw = nil
+        add(source) if source
+      end
+
+      def add(cell)
+        s = cell.to_s
+        m = DIRECTIVE_REGEX.match(s.strip)
+        return nil unless m
+        key = m[1]
+        val = m[2]
+        @directives[key] = val
+        if key == "CLASS_ID"
+          @meta_class_raw = val
+        end
+        val
+      end
+
+      def [](key)
+        @directives[key.to_s]
+      end
+
+      def key?(key)
+        @directives.key?(key.to_s)
+      end
+
+      def fetch(key, default = nil, &block)
+        if @directives.key?(key.to_s)
+          @directives[key.to_s]
+        elsif block
+          yield
+        else
+          default
+        end
+      end
+
+      def meta_class_irdi
+        return @meta_class_irdi if defined?(@meta_class_irdi)
+        @meta_class_irdi =
+          if @meta_class_raw.nil? || @meta_class_raw.empty?
+            nil
+          else
+            Opencdd::IRDI.parse(synthesize_meta_class_irdi)
+          end
+      end
+
+      def synthesize_meta_class_irdi
+        raw = @meta_class_raw
+        return raw if raw.include?("#")
+        supplier = (@directives["DEFAULT_SUPPLIER"] || "").strip
+        return raw if supplier.empty?
+        "#{supplier}##{raw}"
+      end
+
+      def meta_class_code
+        meta_class_irdi&.code
+      end
+
+      def type
+        return @type if defined?(@type)
+        code = meta_class_code
+        @type = code ? Opencdd::Parcel::META_CLASS_TYPES[code] : nil
+      end
+
+      def class_name(lang = :en)
+        @directives["CLASS_NAME.#{lang}"]
+      end
+
+      def class_definition(lang = :en)
+        @directives["CLASS_DEFINITION.#{lang}"]
+      end
+
+      def class_note(lang = :en)
+        @directives["CLASS_NOTE.#{lang}"]
+      end
+
+      def source_language
+        @directives["SOURCE_LANGUAGE"]
+      end
+
+      def default_supplier
+        @directives["DEFAULT_SUPPLIER"]
+      end
+
+      def default_version
+        @directives["DEFAULT_VERSION"]
+      end
+
+      def each(&block)
+        @directives.each(&block)
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/referenced_irdis.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Walks a sharded downloads directory and collects every IRDI
+    # referenced by a class .xls that points to a non-class entity
+    # (property, unit, value-list, value-term). The output manifest
+    # drives the per-entity scraper Pass 2 — see
+    # +TODO.cdd-editor/43-scraper-pass-2-per-entity.md+.
+    #
+    # Source columns on the CLASS .xls (parsed by +Opencdd::Klass+):
+    #   applicable_property_irdis   (MDC_P014) → property
+    #   imported_property_irdis     (MDC_P090) → property (cross-dict)
+    #   is_case_of_irdis            (MDC_P013) → class (cross-dict)
+    #   sub_class_selection_irdis   (MDC_P016) → class
+    #
+    # Cross-dictionary IRDIs are partitioned by their source scheme so
+    # the scraper can pick the right +Dictionary+ config. Properties
+    # referenced in +applicable_property_irdis+ of a class in dict A
+    # that point at dict B's IRDI space are emitted with
+    # +source_scheme: B+.
+    class ReferencedIrdis
+      Entry = Struct.new(:irdi, :entity_type, :source_scheme, :referenced_by, keyword_init: true) do
+        def to_h
+          { irdi: irdi.to_s, entity_type: entity_type, source_scheme: source_scheme,
+            referenced_by: referenced_by.dup }
+        end
+      end
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      # Returns +Hash{ source_scheme => Array<Entry> }+.
+      def collect
+        reader = Opencdd::Parcel::ShardedDirReader.new(@path)
+        database = Opencdd::Database.new
+        reader.load_into(database)
+
+        by_irdi = {}
+        database.classes.each do |klass|
+          record_each(by_irdi, klass, klass.applicable_property_irdis, :property)
+          record_each(by_irdi, klass, klass.imported_property_irdis,   :property)
+          record_each(by_irdi, klass, klass.is_case_of_irdis,          :class)
+          record_each(by_irdi, klass, klass.sub_class_selection_irdis, :class)
+        end
+
+        by_scheme = {}
+        by_irdi.each_value do |entry|
+          bucket = by_scheme[entry.source_scheme] ||= []
+          bucket << entry
+        end
+        by_scheme.transform_values { |entries| entries.sort_by { |e| e.irdi.to_s } }
+      end
+
+      def as_json
+        collect.transform_values { |entries| entries.map(&:to_h) }
+      end
+
+      def to_json(*args)
+        require "json"
+        JSON.pretty_generate(as_json, *args)
+      end
+
+      private
+
+      def record_each(acc, klass, irdis, entity_type)
+        Array(irdis).each do |raw|
+          irdi = Opencdd::IRDI.parse(raw.to_s)
+          next unless irdi
+          scheme = irdi.scheme
+          next unless scheme
+          full = irdi.to_s
+          entry = acc[full]
+          if entry
+            entry.referenced_by << klass.irdi.to_s unless entry.referenced_by.include?(klass.irdi.to_s)
+          else
+            acc[full] = Entry.new(
+              irdi: irdi,
+              entity_type: entity_type,
+              source_scheme: scheme,
+              referenced_by: [klass.irdi.to_s],
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/scrape_verifier.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Verifies that scrape output produced populated .xls files, not
+    # just schema stubs. Catches the silent failure mode discovered
+    # on 2026-07-08: iec61360/iec63213/iec61360-7 scrapes appeared
+    # complete (10,338 PROPERTY .xls files for iec61360) but every
+    # file had 0 data rows. The build pipeline exported 574 classes
+    # + 0 properties, and the browser showed "No declared properties"
+    # for every class. Nothing failed loudly.
+    #
+    # This verifier reads every export_*.xls under a directory and
+    # reports which ones are empty. Use via `rake verify_scrape[dict]`
+    # or directly:
+    #
+    #   results = Opencdd::Parcel::ScrapeVerifier.verify("downloads/iec61360")
+    #   empty = results.select(&:empty?)
+    #   abort "#{empty.size} empty files" unless empty.empty?
+    class ScrapeVerifier
+      Result = Struct.new(:path, :entity_type, :data_row_count, :status,
+                          keyword_init: true) do
+        def ok?    = status == :ok
+        def empty? = status == :empty
+      end
+
+      FILE_PATTERN = /\Aexport_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|VALUETERMS)_[^\.]+\.(xls|xlsx)\z/i
+
+      class << self
+        def verify(directory)
+          new(directory).verify
+        end
+      end
+
+      attr_reader :directory
+
+      def initialize(directory)
+        @directory = directory
+      end
+
+      def verify
+        xls_files.map { |f| check_file(f) }
+      end
+
+      # Summary counts — useful for CLI output.
+      def summary
+        results = verify
+        {
+          total:    results.size,
+          ok:       results.count(&:ok?),
+          empty:    results.count(&:empty?),
+          by_type:  results.group_by(&:entity_type).transform_values(&:size),
+          empty_by_type: results.select(&:empty?).group_by(&:entity_type).transform_values(&:size),
+        }
+      end
+
+      # A "data row" in the Parcel wire format is a row with content
+      # in column 1+ whose first cell isn't a `#` directive or comment.
+      # Column 0 may be empty (the IEC CDD .xls layout leaves it blank;
+      # data starts in column 1). Public so tests can verify the rule
+      # without send.
+      def data_row?(row)
+        return false unless row.is_a?(Array) && row.size > 1
+        first = row.first.to_s.strip
+        return false if first.start_with?("#")
+        row[1..].any? { |c| !c.nil? && !c.to_s.strip.empty? }
+      end
+
+      private
+
+      def xls_files
+        return [] unless File.directory?(@directory)
+        Dir.glob(File.join(@directory, "**", "export_*.{xls,xlsx}")).sort
+      end
+
+      def check_file(path)
+        rows = read_rows(path)
+        data = rows.count { |r| data_row?(r) }
+        Result.new(
+          path: path,
+          entity_type: type_from_filename(path),
+          data_row_count: data,
+          status: data.positive? ? :ok : :empty,
+        )
+      end
+
+      def read_rows(path)
+        ext = File.extname(path).downcase
+        if ext == ".xls"
+          read_xls_rows(path)
+        else
+          read_xlsx_rows(path)
+        end
+      rescue StandardError
+        # Corrupt or unreadable files count as empty for verification
+        # purposes — the user can investigate via the path.
+        []
+      end
+
+      def read_xls_rows(path)
+        require "spreadsheet"
+        book = Spreadsheet.open(path)
+        sheet = book.worksheet(0) || book.worksheets.first
+        return [] unless sheet
+        sheet.rows.map { |row| row.to_a }
+      end
+
+      def read_xlsx_rows(path)
+        require "roo"
+        roo = Roo::Spreadsheet.open(path, extension: :xlsx)
+        target = roo.sheets.first || "Export"
+        sheet = roo.sheet(target)
+        (1..sheet.last_row.to_i).map { |i| sheet.row(i).to_a }
+      end
+
+      def type_from_filename(path)
+        match = File.basename(path).match(FILE_PATTERN)
+        match ? match[1] : "UNKNOWN"
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/selector.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Pure-data value object describing a parcel entity selection.
+    #
+    # The Selector captures the intent of an export:
+    #   - +entities:+ — an Array of IRDIs (or entities) to include.
+    #     +nil+ means "every entity in the database".
+    #   - +closure:+ — whether to expand the entity set along the
+    #     class hierarchy. One of +:none+, +:ancestors+,
+    #     +:descendants+, or +:tree+ (both).
+    #
+    # Resolution against a +Opencdd::Database+ returns a flat
+    # +Array<Opencdd::Entity>+ ordered by type (class, property,
+    # value_list, value_term, unit, relation, view_control). The
+    # caller (Writer, split, etc.) iterates and groups by type.
+    #
+    # Cross-type lifting (declared properties, value_lists,
+    # relations, units) is +Database+'s responsibility; the selector
+    # only walks the class hierarchy and asks the database for
+    # cross-type dependencies.
+    #
+    # Example:
+    #   sel = Opencdd::Parcel::Selector.new(entities: ["0112/2///61360_4#AAA001"], closure: :tree)
+    #   entities = sel.resolve(database)
+    #
+    Selector = Struct.new(:entities, :closure, keyword_init: true) do
+      CLOSURES = %i[none ancestors descendants tree].freeze
+
+      def initialize(entities: nil, closure: :none)
+        raise ArgumentError, "invalid closure: #{closure.inspect}" unless CLOSURES.include?(closure)
+        super(entities: entities, closure: closure)
+      end
+
+      def resolve(database)
+        return database.entities.to_a if entities.nil? && closure == :none
+
+        seed = expand_seed(database)
+        expanded = apply_closure(database, seed)
+        lift_cross_type(database, expanded)
+      end
+
+      private
+
+      def expand_seed(database)
+        Array(entities).flat_map { |e| resolve_entity(database, e) }
+      end
+
+      def resolve_entity(database, candidate)
+        case candidate
+        when Opencdd::Entity then [candidate]
+        when Opencdd::IRDI, String then
+          entity = database.find(candidate)
+          entity ? [entity] : []
+        when nil then []
+        else
+          raise TypeError, "Selector entities must be Opencdd::Entity, Opencdd::IRDI, or String; got #{candidate.class}"
+        end
+      end
+
+      def apply_closure(database, seed)
+        return seed if closure == :none || seed.empty?
+        classes = seed.select { |e| e.is_a?(Opencdd::Klass) }
+        return seed if classes.empty?
+
+        out = seed.dup
+        if closure == :ancestors || closure == :tree
+          classes.each { |k| add_ancestors(out, k) }
+        end
+        if closure == :descendants || closure == :tree
+          classes.each { |k| add_descendants(out, k) }
+        end
+        out
+      end
+
+      def add_ancestors(out, klass)
+        klass.ancestors.each do |a|
+          out << a unless out.include?(a)
+        end
+      end
+
+      def add_descendants(out, klass)
+        klass.descendants.each do |d|
+          out << d unless out.include?(d)
+        end
+      end
+
+      # Lifts cross-type dependencies for the resolved classes: their
+      # declared properties, value_lists those properties reference,
+      # units those properties use, and relations where the class is
+      # in domain or codomain. This makes the selected parcel
+      # self-sufficient when written.
+      def lift_cross_type(database, entities)
+        out = entities.dup
+        classes = entities.select { |e| e.is_a?(Opencdd::Klass) }
+        classes.each do |klass|
+          database.properties_of(klass).each { |p| out << p unless out.include?(p) }
+          database.relations_for(domain: klass.irdi).each { |r| out << r unless out.include?(r) }
+          database.relations_for(codomain: klass.irdi).each { |r| out << r unless out.include?(r) }
+          database.properties_of(klass).each do |prop|
+            next unless prop.is_a?(Opencdd::Property)
+            if prop.unit_irdi
+              unit = database.find(prop.unit_irdi)
+              out << unit if unit.is_a?(Opencdd::Unit) && !out.include?(unit)
+            end
+            vl = database.value_list_of(prop)
+            out << vl if vl && !out.include?(vl)
+          end
+        end
+        out
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/sharded_dir_reader.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Reads the per-class sharded Parcel layout emitted by cdd.iec.ch
+    # downloads: a top-level directory containing one subdir per
+    # class code, each subdir holding 1-6 +export_*_<code>.xls+
+    # files (CLASS, PROPERTY, RELATION, UNIT, VALUELIST, VALUETERMS).
+    #
+    # Two on-disk layouts are supported:
+    #
+    # [flat (legacy)]   +<CODE>/export_*.xls+ directly under the code dir.
+    # [per-version]     +<CODE>/<UNID>/export_*.xls+ nested under the
+    #   current version's UNID, with +<CODE>/_entity.json+ identifying
+    #   which +<UNID>+ folder is the current version. Emitted by
+    #   +harvest/download_versions.py+.
+    #
+    # The reader is a thin orchestrator over three deep modules:
+    #
+    #   - +Opencdd::Parcel::LayoutDetector+ — single source of truth
+    #     for "is this a class-code dir? where do the XLS live?"
+    #   - +Opencdd::Parcel::EntityManifest+ — parses the
+    #     +_entity.json+ sidecar and produces VersionHistory /
+    #     stub entities.
+    #   - +Opencdd::Parcel::FlatDirReader+ — actually reads the
+    #     +.xls+ files into a Workbook.
+    #
+    # Example:
+    #   reader = Opencdd::Parcel::ShardedDirReader.new("downloads/iec63213")
+    #   db = Opencdd::Database.new
+    #   reader.load_into(db)
+    #
+    class ShardedDirReader
+      # Detection patterns owned by Opencdd::Parcel::LayoutDetector
+      # (SSOT). Aliased here for back-compat with external callers
+      # that reference these constants by class.
+      CLASS_CODE_PATTERN = Opencdd::Parcel::LayoutDetector::CLASS_CODE_PATTERN
+      UNID_PATTERN       = Opencdd::Parcel::LayoutDetector::UNID_PATTERN
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def load_into(database)
+        class_subdirs.each do |subdir|
+          active = active_xls_dir(subdir)
+          next unless active
+          workbook = Opencdd::Parcel::FlatDirReader.new(active).read_workbook
+          database.add_workbook(workbook)
+        end
+        attach_unloaded_entities(database)
+        attach_version_histories(database)
+        database.finalize!
+        database
+      end
+
+      # Walks each class subdir's +_entity.json+ and attaches its
+      # +versions+ array to the corresponding entity.
+      def attach_version_histories(database)
+        class_subdirs.each do |subdir|
+          manifest = Opencdd::Parcel::EntityManifest.read(subdir)
+          next unless manifest
+          next if manifest.versions_empty?
+          target = database.find_by_code(File.basename(subdir))
+          target&.attach_version_history(manifest.version_history)
+        end
+      end
+
+      # Back-compat: external callers may reach the per-class version
+      # history directly. New code should use EntityManifest.
+      def parse_version_history(code_dir)
+        manifest = Opencdd::Parcel::EntityManifest.read(code_dir)
+        manifest&.version_history || Opencdd::Entity::VersionHistory.new
+      end
+
+      # Scans +_entities/+ for manifests whose entity wasn't loaded
+      # by the XLS round (e.g. a property whose .xls export only had
+      # CLASS rows). Creates minimal stub entities so cross-references
+      # resolve.
+      def attach_unloaded_entities(database)
+        entities_root = File.join(@path, "_entities")
+        return unless File.directory?(entities_root)
+
+        Dir.children(entities_root).sort.each do |code|
+          entity_dir = File.join(entities_root, code)
+          next unless File.directory?(entity_dir)
+
+          manifest = Opencdd::Parcel::EntityManifest.read(entity_dir)
+          next unless manifest
+
+          irdi = manifest.irdi
+          next unless irdi
+          next if database.find(irdi)
+
+          stub = manifest.to_stub_entity
+          database.add_entity(stub) if stub
+        end
+      end
+
+      def read_workbook
+        sub_workbooks = class_subdirs.filter_map do |subdir|
+          active = active_xls_dir(subdir)
+          next nil unless active
+          Opencdd::Parcel::FlatDirReader.new(active).read_workbook
+        end
+
+        Opencdd::Parcel::Workbook.new(
+          sheets: sub_workbooks.flat_map(&:sheets),
+          sheetmap: sub_workbooks.flat_map(&:sheetmap),
+          project: sub_workbooks.first&.project,
+          source_path: @path.to_s,
+        )
+      end
+
+      def class_subdirs
+        return [] unless File.directory?(@path)
+        dirs = list_class_subdirs(@path)
+
+        # Also scan _entities/ for manifest-driven entity dirs.
+        entities_root = File.join(@path, "_entities")
+        dirs.concat(list_class_subdirs(entities_root)) if File.directory?(entities_root)
+
+        dirs
+      end
+
+      # Delegates to LayoutDetector — single source of truth for
+      # "where do the active XLS files live for this class-code dir?"
+      def active_xls_dir(code_dir)
+        Opencdd::Parcel::LayoutDetector.active_xls_dir(code_dir)
+      end
+
+      private
+
+      def list_class_subdirs(root)
+        Dir.children(root).sort
+           .map { |n| File.join(root, n) }
+           .select { |p| File.directory?(p) && Opencdd::Parcel::LayoutDetector.class_code?(File.basename(p)) }
+      end
+
+      def looks_like_class_code?(name)
+        Opencdd::Parcel::LayoutDetector.class_code?(name)
+      end
+
+      def legacy?(filename)
+        Opencdd::Parcel::LayoutDetector.legacy_export?(filename)
+      end
+    end
+  end
+end