metanorma-oiml 0.1.0

data/lib/metanorma/oiml/sts/transformer/rendered_text_extractor.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Oiml
+    module Sts
+      module Transformer
+        # Extracts plain text from typed model objects for non-math content
+        # (titles, labels, list items, xref visible text). Stems are NEVER
+        # text-extracted — the paragraph transformer routes them through
+        # MathML passthrough via InlineFormula + Mml::V3::Math.
+        class RenderedTextExtractor
+          STEM_TYPES = [
+            Metanorma::Document::Components::Inline::StemInlineElement,
+            Metanorma::Document::Components::Inline::FmtStemElement,
+            Metanorma::Document::Components::Inline::AsciimathElement,
+            Metanorma::Document::Components::Inline::MathElement,
+          ].freeze
+
+          WALKABLE_TYPES = [
+            Metanorma::Document::Components::Paragraphs::ParagraphBlock,
+            Metanorma::Document::Components::Inline::SemxElement,
+            Metanorma::Document::Components::Inline::StrongRawElement,
+            Metanorma::Document::Components::Inline::EmRawElement,
+            Metanorma::Document::Components::Inline::SpanElement,
+            Metanorma::Document::Components::Inline::FmtPreferredElement,
+            Metanorma::Document::Components::Inline::FmtXrefLabelElement,
+            Metanorma::Document::Components::Inline::FmtConceptElement,
+            Metanorma::Document::Components::Inline::ConceptElement,
+            Metanorma::StandardDocument::Sections::ContentSection,
+          ].freeze
+
+          def self.text_of(node)
+            return "" if node.nil?
+            return node if node.is_a?(String)
+            return "" if stem_type?(node)
+            return "" unless walkable?(node)
+
+            walker = new
+            walker.collect(node)
+            walker.to_text
+          end
+
+          def self.stem_type?(node)
+            STEM_TYPES.any? { |t| node.is_a?(t) }
+          end
+          public_class_method :stem_type?
+
+          def self.walkable?(node)
+            WALKABLE_TYPES.any? { |t| node.is_a?(t) } ||
+              node.class.method_defined?(:each_mixed_content)
+          end
+          public_class_method :walkable?
+
+          def initialize
+            @parts = []
+            @seen = {}
+          end
+
+          def to_text
+            @parts.join.strip
+          end
+
+          def append_text(text)
+            @parts << text
+          end
+
+          def collect_node_public(child)
+            collect_node(child)
+          end
+
+          def collect(node)
+            return if node.nil? || @seen.key?(node.object_id)
+            @seen[node.object_id] = true
+
+            if node.is_a?(String)
+              @parts << node
+              return
+            end
+            return unless self.class.walkable?(node)
+
+            collect_content_mapped_text?(node) ||
+              node.each_mixed_content { |child| collect_node(child) }
+          end
+
+          private
+
+          # Elements like NameWithIdElement map their whole content to a
+          # plain `text` attribute (no mixed_content), so walking yields
+          # nothing. SpanElement declares BOTH mixed_content and a text
+          # mapping — only take the attribute shortcut when there is no
+          # mixed content to walk.
+          def collect_content_mapped_text?(node)
+            return false unless node.class.method_defined?(:text)
+            return false unless node.class.method_defined?(:mixed_content?) && !node.mixed_content?
+
+            text = node.text
+            return false unless text.is_a?(String) && !text.empty?
+
+            @parts << text
+            true
+          end
+
+          def collect_node(child)
+            if child.is_a?(String)
+              @parts << child
+              return
+            end
+
+            case child
+            when *STEM_TYPES
+              # Math content is handled by the paragraph transformer via
+              # MathML passthrough. Text extraction is intentionally a no-op.
+              nil
+            when Metanorma::Document::Components::Inline::XrefElement
+              @parts << xref_text(child)
+            when Metanorma::Document::Components::Inline::SemxElement
+              collect(child)
+              collect_semx_inlines(child)
+            when Metanorma::Document::Components::Inline::StrongRawElement,
+                 Metanorma::Document::Components::Inline::EmRawElement,
+                 Metanorma::Document::Components::Inline::SpanElement
+              collect(child)
+            else
+              collect(child)
+            end
+          end
+
+          SEMX_CHILD_ATTRS = %i[
+            stem_child strong em sup sub tt underline strike
+            preferred_child name_child title_child
+            verbal_definition_child definition_child
+            concept_child fn_child link_child
+          ].freeze
+
+          def collect_semx_inlines(semx)
+            SEMX_CHILD_ATTRS.each do |attr|
+              next unless semx.class.method_defined?(attr)
+              val = semx.public_send(attr)
+              next if val.nil?
+              Array(val).each { |v| collect_node(v) }
+            end
+          end
+
+          def xref_text(xref)
+            txt = array_or_string_to_s(xref.text)
+            return txt unless txt.nil? || txt.empty?
+
+            rid = xref.target if xref.class.method_defined?(:target)
+            return "" unless rid.is_a?(String) && !rid.empty?
+
+            case rid
+            when /\Asec-(.+)\z/, /\Afig-(.+)\z/, /\Atable-(.+)\z/, /\Afn-(.+)\z/
+              Regexp.last_match(1)
+            else
+              rid
+            end
+          end
+
+          def array_or_string_to_s(val)
+            return val if val.is_a?(String)
+            return "" unless val.is_a?(Array)
+            val.map(&:to_s).reject(&:empty?).join.strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/oiml/sts/transformer/section_transformer.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Oiml
+    module Sts
+      module Transformer
+        class SectionTransformer < Base
+          def transform(source_clause)
+            title_text = extract_title(source_clause)
+            id_val = source_clause.id if source_clause.is_a?(Lutaml::Model::Serializable) && source_clause.class.method_defined?(:id) && source_clause.id
+
+            content = dispatcher.dispatch_section_blocks(source_clause)
+
+            # Dispatch nested clauses for any section type carrying a
+            # clause collection (IsoClauseSection AND IsoAnnexSection —
+            # annex subsections like the Examples annex's entries).
+            nested_clauses = []
+            if source_clause.is_a?(Lutaml::Model::Serializable) && source_clause.class.method_defined?(:clause)
+              nested_clauses = Array(source_clause.clause).map { |sub| transform(sub) }.compact
+            end
+
+            # Dispatch terms sections (a top-level <terms> carries its
+            # entries directly in .term; a clause's nested terms section
+            # appears in .terms). Each nested terms-section (e.g.
+            # "3.1 General definitions") is preserved as its own Sec so
+            # its heading renders for parity; only the top-level section
+            # whose .term collection we're flattening skips the wrapper.
+            if source_clause.class.method_defined?(:term) && Array(source_clause.term).any?
+              Array(source_clause.term).each do |term|
+                term_sec = term_transformer.transform(term)
+                nested_clauses << term_sec if term_sec
+              end
+            end
+            if source_clause.class.method_defined?(:terms)
+              Array(source_clause.terms).each do |terms_section|
+                terms_sec = transform_terms_section(terms_section)
+                nested_clauses << terms_sec if terms_sec
+              end
+            end
+
+            attrs = {}
+            attrs[:id] = id_val if id_val
+            attrs[:title] = ::Sts::NisoSts::Title.new(content: [title_text]) if title_text && !title_text.empty?
+            autonum = extract_autonum(source_clause)
+            attrs[:label] = ::Sts::NisoSts::Label.new(content: [autonum]) if autonum
+
+            paragraphs, lists, figs, tables, notes, examples, formulas, def_lists, preformats, sub_secs = [], [], [], [], [], [], [], [], [], []
+            content.each do |item|
+              case item
+              when ::Sts::NisoSts::Paragraph then paragraphs << item
+              when ::Sts::NisoSts::List then lists << item
+              when ::Sts::NisoSts::Figure then figs << item
+              when ::Sts::TbxIsoTml::TableWrap then tables << item
+              when ::Sts::NisoSts::NonNormativeNote then notes << item
+              when ::Sts::NisoSts::NonNormativeExample then examples << item
+              when ::Sts::NisoSts::DisplayFormula then formulas << item
+              when ::Sts::NisoSts::DefList then def_lists << item
+              when ::Sts::NisoSts::Preformat then preformats << item
+              when ::Sts::NisoSts::Section then sub_secs << item
+              end
+            end
+            nested_clauses.concat(sub_secs)
+
+            attrs[:paragraphs] = paragraphs if paragraphs.any?
+            attrs[:list] = lists if lists.any?
+            attrs[:fig] = figs if figs.any?
+            attrs[:table_wrap] = tables if tables.any?
+            attrs[:non_normative_note] = notes if notes.any?
+            attrs[:non_normative_example] = examples if examples.any?
+            attrs[:disp_formula] = formulas if formulas.any?
+            attrs[:def_list] = def_lists if def_lists.any?
+            attrs[:preformat] = preformats if preformats.any?
+            attrs[:sec] = nested_clauses if nested_clauses.any?
+
+            ::Sts::NisoSts::Section.new(attrs)
+          end
+
+          # Build a Sec for a nested terms-section (e.g. "3.1 General
+          # definitions") that wraps its term entries. The wrapper's
+          # title and autonum are emitted so the renderer produces the
+          # "3.1 General definitions" heading that appears in MN HTML.
+          def transform_terms_section(terms_section)
+            title_text = extract_title(terms_section)
+            autonum = extract_autonum(terms_section)
+
+            nested = Array(terms_section.term).map { |term| term_transformer.transform(term) }.compact
+
+            attrs = {}
+            attrs[:id] = terms_section.id if terms_section.class.method_defined?(:id) && terms_section.id
+            attrs[:title] = ::Sts::NisoSts::Title.new(content: [title_text]) if title_text && !title_text.empty?
+            attrs[:label] = ::Sts::NisoSts::Label.new(content: [autonum]) if autonum
+            attrs[:sec] = nested if nested.any?
+            return nil unless title_text || autonum || nested.any?
+
+            ::Sts::NisoSts::Section.new(attrs)
+          end
+
+          # Section number from the presentation XML: the autonum
+          # attribute (annexes, term notes) or the fmt-title/fmt-name
+          # caption-label's autonum semx ("1", "8.1", "3.1"). The delim
+          # between number and title is a <tab/>, which text extraction
+          # drops, so the caption-label span must be isolated —
+          # splitting the whole fmt-title text would glue number and
+          # title together ("1Scope").
+          def extract_autonum(source_clause)
+            if source_clause.class.method_defined?(:autonum) && source_clause.autonum
+              return source_clause.autonum.to_s
+            end
+
+            title_wrapper = %i[fmt_title fmt_name].filter_map do |method|
+              source_clause.public_send(method) if source_clause.class.method_defined?(method)
+            end.first
+            title_wrapper = Array(title_wrapper).first
+            return nil unless title_wrapper
+
+            caption_label_text(title_wrapper)
+          end
+
+          private
+
+          def extract_title(source_clause)
+            return nil unless source_clause.class.method_defined?(:title) && source_clause.title
+            title = source_clause.title
+            return extract_with_stems(title) if title_is_mixed_with_stems?(title)
+            RenderedTextExtractor.text_of(title)
+          end
+
+          # Title text where stem elements contribute their visible
+          # AsciiMath tokens (e.g. "D_(\"min\")" → " D min ") so the
+          # parity check sees content where it would otherwise see "()".
+          # Source text from asciimath matches what Nokogiri's text
+          # extraction produces from MN HTML after the document-level
+          # normaliser lowercases both; case sensitivity is therefore
+          # not an issue here, and we never parse MathML XML.
+          def extract_with_stems(title)
+            parts = collect_title_parts(title, [])
+            parts.join.strip
+          end
+
+          def title_is_mixed_with_stems?(title)
+            title.class.method_defined?(:each_mixed_content)
+          end
+
+          def collect_title_parts(node, parts)
+            return parts unless node.is_a?(Lutaml::Model::Serializable)
+            return parts unless node.class.method_defined?(:each_mixed_content)
+
+            node.each_mixed_content do |child|
+              if child.is_a?(String)
+                parts << child
+              elsif stem_node?(child)
+                parts << stem_visible_text(child)
+              elsif xref_node?(child)
+                parts << xref_visible_text(child)
+              elsif child.is_a?(Lutaml::Model::Serializable)
+                collect_title_parts(child, parts)
+              end
+            end
+            parts
+          end
+
+          STEM_NODE_CLASSES = [
+            "Metanorma::Document::Components::Inline::StemInlineElement",
+            "Metanorma::Document::Components::Inline::FmtStemElement",
+            "Metanorma::Document::Components::TextElements::StemElement",
+            "Metanorma::Document::Components::Inline::AsciimathElement",
+            "Metanorma::Document::Components::Inline::MathElement",
+          ].freeze
+
+          def stem_node?(node)
+            STEM_NODE_CLASSES.include?(node.class.name)
+          end
+
+          def xref_node?(node)
+            node.is_a?(Metanorma::Document::Components::Inline::XrefElement)
+          end
+
+          def stem_visible_text(stem)
+            # The typed stem has an `asciimath` attribute mapping to
+            # Asciiml (a typed model with a `:value` string). For
+            # Semx wrappers the asciimath sits on the inner stem; for
+            # inline stems on the stem itself.
+            asciimath_value(asciimath_of(stem)).then do |raw|
+              text = raw.to_s.tr("_", " ").gsub(/[()"\\]/, "").strip
+              text.empty? ? "" : " #{text} "
+            end
+          end
+
+          def asciimath_of(stem)
+            stem.is_a?(Metanorma::Document::Components::TextElements::StemElement) ||
+              stem.is_a?(Metanorma::Document::Components::Inline::StemInlineElement) ||
+              stem.is_a?(Metanorma::Document::Components::Inline::FmtStemElement) ? stem.asciimath : nil
+          end
+
+          def asciimath_value(asciiml)
+            return nil unless asciiml
+            asciiml.is_a?(Metanorma::Document::Components::TextElements::Asciiml) ? asciiml.value : asciiml.to_s
+          end
+
+          def xref_visible_text(xref)
+            txt = Array(xref.text).join
+            txt = xref.target.to_s if txt.empty? && xref.target
+            case txt
+            when /\Asec-(.+)\z/, /\Afig-(.+)\z/, /\Atable-(.+)\z/, /\Afn-(.+)\z/
+              "#{Regexp.last_match(1)} "
+            else
+              txt
+            end
+          end
+
+          # Text of the <span class="fmt-caption-label"> inside a
+          # fmt-title — the bare section number.
+          def caption_label_text(fmt_title)
+            return nil unless fmt_title.class.method_defined?(:each_mixed_content)
+
+            label_span = nil
+            fmt_title.each_mixed_content do |child|
+              if child.is_a?(Metanorma::Document::Components::Inline::SpanElement) &&
+                  child.class_attr == "fmt-caption-label"
+                label_span = child
+                break
+              end
+            end
+            return nil unless label_span
+
+            text = RenderedTextExtractor.text_of(label_span).strip
+            text.empty? ? nil : text
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/oiml/sts/transformer/source_document.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+require "metanorma/document"
+require "metanorma/oiml/document"
+
+module Metanorma
+  module Oiml
+    module Sts
+      module Transformer
+        class SourceDocument
+          attr_reader :typed_root
+
+          def self.parse(input)
+            xml_string = input.class.method_defined?(:read) ? input.read : input.to_s
+            typed = Metanorma::Oiml::Document::Root.from_xml(xml_string)
+            new(typed)
+          end
+
+          # Reader entry point for the metanorma-core document_transformers
+          # contract (+reader.from_xml(String) -> model+); an alias of {.parse}.
+          def self.from_xml(input)
+            parse(input)
+          end
+
+          def initialize(typed_root)
+            @typed_root = typed_root
+          end
+
+          def bibdata
+            typed_root.bibdata
+          end
+
+          def language
+            return "en" unless bibdata
+            langs = bibdata.language rescue nil
+            return "en" unless langs
+            lang = langs.is_a?(Array) ? langs.first : langs
+            val = lang.is_a?(String) ? lang : lang.value.to_s
+            val.strip.match?(/\A[a-z]{2}(-[A-Z]{2})?\z/) ? val.strip : "en"
+          end
+
+          def docidentifier
+            return nil unless bibdata
+            ids = bibdata.doc_identifier rescue nil
+            return nil unless ids
+            Array(ids).each do |id_obj|
+              next if id_obj.type
+              return text_of(id_obj)
+            end
+            text_of(Array(ids).first)
+          end
+
+          def text_of(obj)
+            return obj.to_s if obj.is_a?(String)
+            return "" unless obj
+            val = (obj.value rescue nil) || (obj.content rescue nil)
+            return "" unless val
+            Array(val).map { |v| v.is_a?(String) ? v : text_of(v) }.join.strip
+          end
+
+          def title(type: nil)
+            return nil unless bibdata
+            titles = bibdata.titles rescue nil
+            return nil unless titles
+            titles = Array(titles)
+            t = titles.find { |x| type.nil? || (x.type == type rescue false) } || titles.first
+            text_of(t)
+          end
+
+          def formatted_title
+            return title unless bibdata
+            # The composed model #title can raise on documents whose front
+            # lacks the title structure it expects; fall back to the
+            # generic wrapper title rather than failing the transform.
+            iso_title = (bibdata.title rescue nil)
+            return title unless iso_title
+            main = abstract_title_value(iso_title.title_main)
+            part_prefix = abstract_title_value(iso_title.title_part_prefix)
+            part = abstract_title_value(iso_title.title_part)
+            segments = [main]
+            if part_prefix && !part_prefix.empty?
+              segments << (part && !part.empty? ? "#{part_prefix}:#{part}" : part_prefix)
+            elsif part && !part.empty?
+              segments << part
+            end
+            segments.reject! { |s| s.nil? || s.empty? }
+            return title if segments.empty?
+            segments.join(" — ")
+          end
+
+          def abstract_title_value(abstract)
+            return nil if abstract.nil?
+            val = abstract.value rescue nil
+            val.nil? ? nil : val.to_s.strip
+          end
+
+          def copyright_year
+            return nil unless bibdata
+            cps = bibdata.copyright rescue nil
+            return nil unless cps
+            cp = Array(cps).first
+            return nil unless cp
+            from = cp.from rescue nil
+            return text_of(from) unless from.nil?
+            nil
+          end
+
+          def copyright_holder
+            return nil unless bibdata
+            cps = bibdata.copyright rescue nil
+            return nil unless cps
+            cp = Array(cps).first
+            return nil unless cp
+            owner = cp.owner rescue nil
+            return nil unless owner
+            org = Array(owner.organization).first rescue nil
+            return nil unless org
+            text_of(org.name) rescue nil
+          end
+
+          def publisher
+            copyright_holder
+          end
+
+          def pub_date
+            copyright_year
+          end
+
+          def foreword
+            typed_root.preface&.foreword
+          end
+
+          def introduction
+            typed_root.preface&.introduction
+          end
+
+          # Top-level sections of every kind — <clause>, <terms>,
+          # <definitions>, <references> (normative references) — in
+          # document order. The XML's physical order differs (references
+          # trails the clauses); displayorder carries the intended one.
+          SECTION_COLLECTIONS = %i[clause terms definitions references].freeze
+
+          def sections
+            secs = typed_root.sections
+            return [] unless secs
+
+            SECTION_COLLECTIONS.flat_map { |attr| Array(secs.public_send(attr)) }
+              .sort_by do |s|
+                s.class.method_defined?(:displayorder) && s.displayorder ? s.displayorder : Float::INFINITY
+              end
+          end
+
+          def annexes
+            Array(typed_root.annex)
+          end
+
+          def bibliography
+            bib = typed_root.bibliography
+            return [] unless bib
+            Array(bib.references)
+          end
+
+          def front?
+            !foreword.nil? || !introduction.nil?
+          end
+
+          def has_metadata?
+            !docidentifier.nil?
+          end
+
+          def has_back?
+            annexes.any? || bibliography.any?
+          end
+
+          def term_autonum_for(term_id)
+            @term_autonum_cache ||= build_term_autonum_cache
+            @term_autonum_cache[term_id]
+          end
+
+          private
+
+          def build_term_autonum_cache
+            cache = {}
+            walk_for_terms(typed_root) do |term|
+              next unless term.is_a?(Metanorma::IsoDocument::Terms::IsoTerm)
+              autonum = extract_term_autonum(term)
+              next unless autonum && !autonum.empty?
+              cache[term.id] = autonum if term.id
+              anchor = term.anchor if term.class.method_defined?(:anchor)
+              cache[anchor] = autonum if anchor && !cache.key?(anchor)
+            end
+            cache
+          end
+
+          def walk_for_terms(node, seen = {}, &blk)
+            return unless node.is_a?(Object) && !node.is_a?(String)
+            return if seen[node.object_id]
+            seen[node.object_id] = true
+            blk.call(node)
+            attrs = node.class.instance_variable_get(:@attributes) rescue nil
+            return unless attrs
+            attrs.each do |name, _|
+              next unless node.class.method_defined?(name)
+              val = node.public_send(name) rescue next
+              next if val.nil?
+              Array(val).each { |v| walk_for_terms(v, seen, &blk) if v.is_a?(Object) && !v.is_a?(String) && !v.is_a?(Array) }
+            end
+          end
+
+          def extract_term_autonum(term)
+            label_arr = Array(term.fmt_xref_label)
+            label = label_arr.first
+            return nil unless label
+            return nil unless label.class.method_defined?(:each_mixed_content)
+            parts = []
+            label.each_mixed_content do |n|
+              if n.is_a?(String) && !n.strip.empty?
+                parts << n.strip
+              else
+                cn = n.class.name.split("::").last
+                txt = semx_text(n) if cn == "SemxElement"
+                txt = span_text(n) if cn == "SpanElement"
+                parts << txt if txt && !txt.empty?
+              end
+            end
+            parts.join.strip
+          end
+
+          def semx_text(semx)
+            return "" unless semx.class.method_defined?(:each_mixed_content)
+            parts = []
+            semx.each_mixed_content { |n| parts << n if n.is_a?(String) }
+            parts.join
+          end
+
+          def span_text(span)
+            return "" unless span.class.method_defined?(:each_mixed_content)
+            parts = []
+            span.each_mixed_content { |n| parts << n if n.is_a?(String) }
+            parts.join
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/oiml/sts/transformer/standard.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Oiml
+    module Sts
+      module Transformer
+        # Adapter conforming to the metanorma-core
+        # +Processor#document_transformers+ contract: +.new(model, options)+,
+        # where +model+ is the {SourceDocument} produced by
+        # {SourceDocument.from_xml}; +#transform+ returns a target responding to
+        # +#to_xml+.
+        #
+        # The adapter is its own target: its +#to_xml+ runs the full
+        # {DocumentTransformer#transform_to_xml} pipeline (model build plus the
+        # namespace / processing-meta / lang post-processing on the serialised
+        # string), so the pipeline output matches the standalone
+        # {Metanorma::Oiml::Sts::Transformer.convert}. Returning the raw
+        # +#transform+ model instead would drop that post-processing.
+        class Standard
+          def initialize(model, options = {})
+            @model = model
+            @options = options
+          end
+
+          def transform
+            @document_transformer = DocumentTransformer.new(Context.new(@model))
+            self
+          end
+
+          def to_xml(**_options)
+            @document_transformer.transform_to_xml(@model)
+          end
+        end
+      end
+    end
+  end
+end