opencdd 0.1.1

data/lib/opencdd/parcel/workbook.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Opencdd
+  module Parcel
+    class Workbook
+      SheetMapEntry = Struct.new(:project_id, :parcel_id, :class_irdi, :content_no,
+                                 :sheet_no, :sheet_name, :type, :target, keyword_init: true)
+      ProjectInfo = Struct.new(:project_id, :parcel_id, :multi_language, :base_language,
+                               keyword_init: true)
+
+      HEADER_ROW_NAMES = %i[
+        class_id class_name source_language translation_language
+        property_id property_name datatype value_format pattern
+        default_value requirement unit
+      ].freeze
+
+      MANDATORY_HEADER_ROWS = %i[class_id property_id].freeze
+
+      attr_reader :sheets, :sheetmap, :project, :source_path
+
+      def initialize(sheets:, sheetmap: [], project: nil, source_path: nil,
+                     hidden_header_rows: nil)
+        @sheets = sheets
+        @sheetmap = sheetmap
+        @project = project
+        @source_path = source_path
+        @hidden_header_rows = validate_hidden(hidden_header_rows)
+        reindex
+        freeze
+      end
+
+      def hidden_header_rows
+        @hidden_header_rows.dup
+      end
+
+      def hide_header_row!(name)
+        sym = name.to_sym
+        raise ArgumentError, "unknown header row: #{sym.inspect}" unless HEADER_ROW_NAMES.include?(sym)
+        raise ArgumentError, "cannot hide mandatory header row: #{sym.inspect}" if MANDATORY_HEADER_ROWS.include?(sym)
+        @hidden_header_rows << sym
+        self
+      end
+
+      def show_header_row!(name)
+        @hidden_header_rows.delete(name.to_sym)
+        self
+      end
+
+      def register_sheet(sheet)
+        raise ArgumentError, "sheet already registered: #{sheet.name}" if @sheets_by_name.key?(sheet.name.to_s)
+        @sheets << sheet
+        @sheets_by_name[sheet.name.to_s] = sheet
+        @sheetmap << SheetMapEntry.new(
+          project_id: @project&.project_id || "LOCAL",
+          parcel_id:  @project&.parcel_id,
+          class_irdi: sheet.meta_class_irdi,
+          content_no: 0,
+          sheet_no:   @sheets.size,
+          sheet_name: sheet.name,
+          type:       parcel_type_label_for(sheet.type),
+          target:     "",
+        )
+        self
+      end
+
+      def import_into(sheet_name, file_path, format: nil)
+        target = sheet(sheet_name)
+        raise ArgumentError, "unknown sheet: #{sheet_name.inspect}" unless target
+
+        format ||= detect_format(file_path)
+        temp = read_external_sheet(file_path, format, target.meta_class_irdi)
+        unless meta_codes_match?(target.meta_class_irdi, temp.meta_class_irdi)
+          raise ArgumentError,
+                "meta-class mismatch: target=#{target.meta_class_irdi} source=#{temp.meta_class_irdi}"
+        end
+        target.merge_rows_from(temp)
+        self
+      end
+
+      def sheet(name)
+        @sheets_by_name[name.to_s]
+      end
+
+      def sheets_of_type(type)
+        type = type.to_sym
+        @sheets.select { |s| s.type == type }
+      end
+
+      def class_sheet      ; sheets_of_type(:class).first      ; end
+      def property_sheet   ; sheets_of_type(:property).first   ; end
+      def unit_sheet       ; sheets_of_type(:unit).first       ; end
+      def value_list_sheet ; sheets_of_type(:value_list).first ; end
+      def value_term_sheet ; sheets_of_type(:value_term).first ; end
+      def relation_sheet   ; sheets_of_type(:relation).first   ; end
+      def view_control_sheet; sheets_of_type(:view_control).first; end
+
+      def each_sheet(&block)
+        @sheets.each(&block)
+      end
+
+      def parcel_id
+        @project&.parcel_id
+      end
+
+      def project_id
+        @project&.project_id
+      end
+
+      def base_language
+        @project&.base_language || "en"
+      end
+
+      def merge(other)
+        merged_sheets = @sheets + other.sheets
+        merged_map    = @sheetmap + other.sheetmap
+        merged_proj   = @project || other.project
+        Workbook.new(sheets: merged_sheets, sheetmap: merged_map, project: merged_proj,
+                     source_path: @source_path)
+      end
+
+      private
+
+      def validate_hidden(rows)
+        return Set.new if rows.nil?
+        set = rows.to_set
+        invalid = set - HEADER_ROW_NAMES.to_set
+        raise ArgumentError, "unknown header rows: #{invalid.to_a.inspect}" unless invalid.empty?
+        mandatory = set & MANDATORY_HEADER_ROWS.to_set
+        raise ArgumentError, "cannot hide mandatory rows: #{mandatory.to_a.inspect}" unless mandatory.empty?
+        set
+      end
+
+      def detect_format(path)
+        case File.extname(path.to_s).downcase
+        when ".csv" then :csv
+        when ".txt" then :csv_tab
+        when ".xlsx", ".xlsm" then :xlsx
+        when ".xls" then :xls
+        else raise ArgumentError, "cannot detect format for: #{path.inspect}"
+        end
+      end
+
+      def read_external_sheet(path, format, meta_class_irdi)
+        case format
+        when :csv      then Opencdd::Parcel::CsvReader.read(path, meta_class_irdi: meta_class_irdi)
+        when :csv_tab  then Opencdd::Parcel::CsvReader.read(path, meta_class_irdi: meta_class_irdi, col_sep: "\t")
+        when :xlsx, :xls
+          Opencdd::Parcel::WorkbookReader.new(path).read_workbook.sheets.first
+        else raise ArgumentError, "unknown format: #{format.inspect}"
+        end
+      end
+
+      def parcel_type_label_for(type)
+        return nil if type.nil?
+        Opencdd::MetaClasses.sheet_type_for_type(type) || type.to_s.upcase
+      end
+
+      def meta_codes_match?(a, b)
+        return false if a.nil? || b.nil?
+        a_code = a.to_s.split("#").last
+        b_code = b.to_s.split("#").last
+        a_code == b_code
+      end
+
+      def reindex
+        @sheets_by_name = @sheets.to_h { |s| [s.name.to_s, s] }
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/workbook_reader.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module Opencdd
+  module Parcel
+    # Reads a single Parcel workbook file (.xlsx, .xlsm, .xltx, or
+    # legacy single-sheet .xls) and produces a +Opencdd::Parcel::Workbook+.
+    #
+    # This is the canonical IEC 62656-1 Parcel container: one file
+    # with Project / sheetmap / pcls_LOCAL / data sheets. Use
+    # +Opencdd::Parcel::FlatDirReader+ for the legacy 6-file-per-directory
+    # layout, and +Opencdd::Parcel::ShardedDirReader+ for the per-class
+    # sharded layout.
+    class WorkbookReader
+      LEGACY_TYPE_BY_PREFIX = {
+        "CLASS"      => :class,
+        "PROPERTY"   => :property,
+        "RELATION"   => :relation,
+        "UNIT"       => :unit,
+        "VALUELIST"  => :value_list,
+        "VALUETERMS" => :value_term,
+      }.freeze
+
+      LEGACY_TYPE_TO_PARCEL_NAME = {
+        class:       "CLASS",
+        property:    "PROPERTY",
+        value_list:  "ENUM",
+        value_term:  "TERMINOLOGY",
+        unit:        "UoM",
+        relation:    "RELATION",
+      }.freeze
+
+      META_CLASS_BY_LEGACY_TYPE = Opencdd::MetaClasses::TYPE_BY_META_CLASS.invert.freeze
+
+      attr_reader :path
+
+      def initialize(path)
+        @path = path
+      end
+
+      def read_workbook
+        sheets = []
+        sheetmap = []
+        project = nil
+
+        workbook = open_workbook
+
+        if workbook.sheet_names.include?("sheetmap")
+          sheetmap = parse_sheetmap(workbook.rows_for("sheetmap"))
+        end
+
+        if workbook.sheet_names.include?("Project")
+          project = parse_project(workbook.rows_for("Project"))
+        end
+
+        workbook.sheet_names.each do |sheet_name|
+          next if ["Project", "sheetmap", "pcls_LOCAL"].include?(sheet_name)
+          sheet = Opencdd::Parcel::Sheet.from_rows(workbook.rows_for(sheet_name).each, name: sheet_name)
+          sheets << sheet if sheet
+        end
+
+        if sheets.empty? && legacy?
+          sheets = [Opencdd::Parcel::Sheet.from_rows(
+            workbook.rows_for(workbook.sheet_names.first).each,
+            name: File.basename(@path.to_s, ".*"),
+          )].compact
+          sheetmap = sheets.map { |s|
+            type = s.type
+            next unless type
+            Workbook::SheetMapEntry.new(
+              project_id: project&.project_id || "LOCAL",
+              parcel_id:  project&.parcel_id,
+              class_irdi: Opencdd::IRDI.parse("0112/2///62656_1##{META_CLASS_BY_LEGACY_TYPE[type]}"),
+              content_no: 0,
+              sheet_no:   nil,
+              sheet_name: s.name,
+              type:       LEGACY_TYPE_TO_PARCEL_NAME[type] || type.to_s.upcase,
+              target:     "",
+            )
+          }.compact
+        end
+
+        Opencdd::Parcel::Workbook.new(
+          sheets: sheets,
+          sheetmap: sheetmap,
+          project: project,
+          source_path: @path.to_s,
+        )
+      end
+
+      def load_into(database)
+        workbook = read_workbook
+        database.add_workbook(workbook)
+        database.finalize!
+        database
+      end
+
+      private
+
+      def open_workbook
+        ext = File.extname(@path.to_s).downcase
+        if ext == ".xls"
+          SpreadsheetSource.new(@path.to_s)
+        else
+          RooSource.new(@path.to_s, ext)
+        end
+      end
+
+      def legacy?
+        File.basename(@path.to_s) =~ /\Aexport_(CLASS|PROPERTY|RELATION|UNIT|VALUELIST|VALUETERMS)_.*\z/i
+      end
+
+      def parse_sheetmap(rows)
+        header = rows.first&.map { |c| c.to_s.strip.downcase } || []
+
+        rows[1..].map do |r|
+          vals = r.to_a
+          h = {}
+          header.each_with_index do |k, i|
+            next if k.nil? || k.empty?
+            v = vals[i]
+            h[k.to_sym] = v.nil? || v.to_s.strip.empty? ? nil : v.to_s.strip
+          end
+          next unless h[:sheetname]
+          Workbook::SheetMapEntry.new(
+            project_id: h[:projectid],
+            parcel_id:  h[:parcelid],
+            class_irdi: h[:classid] && Opencdd::IRDI.parse(h[:classid]),
+            content_no: h[:contentno]&.to_i,
+            sheet_no:   h[:sheetno]&.to_i,
+            sheet_name: h[:sheetname],
+            type:       h[:type],
+            target:     h[:target] || "",
+          )
+        end.compact
+      end
+
+      def parse_project(rows)
+        header = rows.first&.map(&:to_s) || []
+        vals = rows[1]&.to_a || []
+        h = header.each_with_index.to_h { |k, i| [k.to_s.strip, vals[i]] }
+        Workbook::ProjectInfo.new(
+          project_id: h["Project ID"].to_s,
+          parcel_id:  h["Parcel ID"].to_s,
+          multi_language: h["Multi language"].to_s,
+          base_language: (h["Base language"].to_s.empty? ? "en" : h["Base language"].to_s),
+        )
+      end
+
+      class RooSource < Struct.new(:path, :extension)
+        def initialize(path, ext)
+          require "roo"
+          roo_ext =
+            case ext
+            when ".xlsx" then :xlsx
+            when ".xlsm" then :xlsm
+            when ".xls"  then :xls
+            when ".csv"  then :csv
+            else :xlsx
+            end
+          super(path, roo_ext)
+          @roo = Roo::Spreadsheet.open(path, extension: extension)
+        end
+
+        def sheet_names
+          @sheet_names ||= @roo.sheets
+        end
+
+        def rows_for(name)
+          sheet = @roo.sheet(name)
+          (1..sheet.last_row.to_i).map { |i| sheet.row(i) }
+        end
+      end
+
+      class SpreadsheetSource < Struct.new(:path)
+        def initialize(path)
+          require "spreadsheet"
+          super(path)
+          @book = Spreadsheet.open(path)
+        end
+
+        def sheet_names
+          @sheet_names ||= @book.worksheets.map { |ws| decode_name(ws.name) }
+        end
+
+        def rows_for(name)
+          ws = @book.worksheets.find { |w| decode_name(w.name) == name } || @book.worksheet(0)
+          ws.rows.map(&:to_a)
+        end
+
+        private
+
+        def decode_name(s)
+          s.to_s.encode("UTF-8", "UTF-16LE", invalid: :replace, undef: :replace).scrub("")
+        rescue Encoding::ConverterNotFoundError
+          s.to_s
+        end
+      end
+    end
+  end
+end

data/lib/opencdd/parcel/writer.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require "stringio"
+require "set"
+require "caxlsx"
+
+module Opencdd
+  module Parcel
+    class Writer
+      DEFAULT_PROJECT_ID = "LOCAL"
+      DEFAULT_SOURCE_LANGUAGE = "en"
+
+      attr_reader :database
+
+      def initialize(database)
+        @database = database
+      end
+
+      def write(target, parcel_id:, project_id: DEFAULT_PROJECT_ID,
+                source_language: DEFAULT_SOURCE_LANGUAGE, translation_languages: [],
+                sheet_types: :all, selector: nil)
+        package = build_package(
+          parcel_id: parcel_id, project_id: project_id,
+          source_language: source_language, translation_languages: translation_languages,
+          sheet_types: sheet_types, selector: selector,
+        )
+        serialize(package, target)
+        target
+      end
+
+      def write_sheet(sheet, entities, target, source_language: DEFAULT_SOURCE_LANGUAGE)
+        emitter = SheetEmitter.new(source_language: source_language)
+        rows = emitter.emit(sheet, entities)
+        package = Axlsx::Package.new
+        package.workbook.add_worksheet(name: sheet.name) do |ws|
+          rows.each { |row| ws.add_row(row) }
+        end
+        serialize(package, target)
+        target
+      end
+
+      private
+
+      def build_package(parcel_id:, project_id:, source_language:, translation_languages:, sheet_types:, selector: nil)
+        package = Axlsx::Package.new
+        workbook = package.workbook
+
+        effective_entities = selector ? selector.resolve(@database) : nil
+
+        built_sheets = build_sheets(
+          parcel_id: parcel_id, source_language: source_language,
+          translation_languages: translation_languages, sheet_types: sheet_types,
+          effective_entities: effective_entities,
+        )
+        hidden_directives = hidden_header_rows_from_database
+
+        add_project_sheet(workbook, parcel_id, project_id, source_language, translation_languages)
+        add_sheetmap_sheet(workbook, built_sheets, parcel_id, project_id)
+        add_data_sheets(workbook, built_sheets, source_language, hidden_directives)
+
+        package
+      end
+
+      ALL_SHEET_TYPES = %i[class property value_list value_term unit relation view_control].freeze
+
+      def resolve_types(sheet_types)
+        return ALL_SHEET_TYPES if sheet_types == :all
+        Array(sheet_types).map(&:to_sym)
+      end
+
+      def build_sheets(parcel_id:, source_language:, translation_languages:, sheet_types:, effective_entities: nil)
+        types = resolve_types(sheet_types)
+        if @database.workbooks.any?
+          build_sheets_from_workbook(types, parcel_id, source_language, translation_languages, effective_entities)
+        else
+          build_scaffolds(types, parcel_id, source_language, translation_languages, effective_entities)
+        end
+      end
+
+      def build_sheets_from_workbook(types, parcel_id, source_language, translation_languages, effective_entities)
+        source_workbook = @database.workbooks.first
+        picked = source_workbook.sheets.select { |s| s.type && types.include?(s.type) }
+        picked.group_by(&:type).flat_map do |_type, sheets|
+          primary = sheets.first
+          entities = filtered_entities_of_type(primary.type, effective_entities)
+          [BuiltSheet.new(sheet: primary, entities: entities, type: primary.type)]
+        end
+      end
+
+      def build_scaffolds(types, parcel_id, source_language, translation_languages, effective_entities)
+        types.filter_map do |type|
+          meta_code = Opencdd::MetaClasses.meta_class_for_type(type)
+          next nil unless meta_code
+          sheet = Sheet.scaffold(
+            meta_class_irdi: meta_code,
+            parcel_id: parcel_id,
+            source_language: source_language,
+            translation_languages: translation_languages,
+          )
+          entities = filtered_entities_of_type(type, effective_entities)
+          BuiltSheet.new(sheet: sheet, entities: entities, type: type)
+        end
+      end
+
+      def filtered_entities_of_type(type, effective_entities)
+        all = @database.entities_of_type(type)
+        return all if effective_entities.nil?
+        allowed = effective_entities.select { |e| e.type == type }.to_set
+        all.select { |e| allowed.include?(e) }
+      end
+
+      def add_project_sheet(workbook, parcel_id, project_id, source_language, translation_languages)
+        workbook.add_worksheet(name: "Project") do |ws|
+          ws.add_row(["Project ID", "Parcel ID", "Multi language", "Base language"])
+          multi = translation_languages.empty? ? "" : translation_languages.join(",")
+          ws.add_row([project_id, parcel_id, multi, source_language])
+        end
+      end
+
+      def add_sheetmap_sheet(workbook, built_sheets, parcel_id, project_id)
+        workbook.add_worksheet(name: "sheetmap") do |ws|
+          ws.add_row(%w[projectid parcelid classid contentno sheetno sheetname type target])
+          ws.add_row([project_id, "", "", 0, 2, "pcls_LOCAL", "PARCEL_LIST", ""])
+          built_sheets.each_with_index do |built, idx|
+            sheet = built.sheet
+            ws.add_row([
+              project_id,
+              parcel_id,
+              "#{sheet.meta_class_irdi&.to_s}##1",
+              0,
+              idx + 3,
+              sheet.name,
+              parcel_type_label(built.type),
+              "",
+            ])
+          end
+        end
+      end
+
+      def parcel_type_label(type)
+        Opencdd::MetaClasses.sheet_type_for_type(type) || type.to_s.upcase
+      end
+
+      def add_data_sheets(workbook, built_sheets, source_language, hidden_directives = Set.new)
+        emitter = SheetEmitter.new(source_language: source_language)
+        built_sheets.each do |built|
+          rows = emitter.emit(built.sheet, built.entities)
+          workbook.add_worksheet(name: built.sheet.name) do |ws|
+            rows.each do |row|
+              emitted = ws.add_row(row)
+              emitted.hidden = true if row_hidden?(row, hidden_directives)
+            end
+          end
+        end
+      end
+
+      def hidden_header_rows_from_database
+        wb = @database.workbooks.first
+        return Set.new unless wb
+        wb.hidden_header_rows
+      end
+
+      def row_hidden?(row, hidden_directives)
+        return false if hidden_directives.empty?
+        first = row.first.to_s
+        return false unless first.start_with?("#")
+        base = first.sub(/\A#/, "").split(/[:=.]/).first
+        return false if base.nil? || base.empty?
+        hidden_directives.include?(base.downcase.to_sym)
+      end
+
+      def serialize(package, target)
+        case target
+        when String then package.serialize(target)
+        when Pathname then package.serialize(target.to_s)
+        when IO, StringIO then target.write(package.to_stream.read)
+        else
+          raise ArgumentError, "unsupported write target: #{target.inspect}"
+        end
+      end
+
+      BuiltSheet = Struct.new(:sheet, :entities, :type, keyword_init: true)
+    end
+  end
+end