metanorma-document 0.5.0 → 0.5.2.pre.alpha.1

data/lib/metanorma/document/components/text_elements/underline_element.rb

@@ -10,6 +10,8 @@ module Metanorma
           attribute :text, :string, collection: true
           attribute :em, "Metanorma::Document::Components::Inline::EmRawElement",
                     collection: true
+          attribute :ruby, "Metanorma::Document::Components::TextElements::RubyElement",
+                    collection: true
           attribute :strong, "Metanorma::Document::Components::Inline::StrongRawElement",
                     collection: true
           attribute :eref, "Metanorma::Document::Components::Inline::ErefElement",
@@ -22,6 +24,7 @@ module Metanorma
             mixed_content
             map_content to: :text
             map_element "em", to: :em
+            map_element "ruby", to: :ruby
             map_element "strong", to: :strong
             map_element "eref", to: :eref
             map_element "link", to: :link

data/lib/metanorma/document/components/text_elements.rb

@@ -12,6 +12,8 @@ module Metanorma
 
         autoload :MonospaceElement, "#{__dir__}/text_elements/monospace_element"
         autoload :RubyElement, "#{__dir__}/text_elements/ruby_element"
+        autoload :RubyPronunciationElement, "#{__dir__}/text_elements/ruby_pronunciation_element"
+        autoload :RubyAnnotationElement, "#{__dir__}/text_elements/ruby_annotation_element"
         autoload :SmallCapsElement,
                  "#{__dir__}/text_elements/small_caps_element"
         autoload :StemElement, "#{__dir__}/text_elements/stem_element"

data/lib/metanorma/document/components.rb

@@ -19,6 +19,7 @@ module Metanorma
       autoload :ReferenceElements,
                "metanorma/document/components/reference_elements"
       autoload :Sections, "metanorma/document/components/sections"
+      autoload :Semantic, "metanorma/document/components/semantic"
       autoload :Tables, "metanorma/document/components/tables"
       autoload :Inline, "metanorma/document/components/inline"
       autoload :TextElements, "metanorma/document/components/text_elements"

data/lib/metanorma/document/grammar_vocabulary.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Document
+    # The standoc-grammar element vocabulary this gem is accountable
+    # for (#2): every element defined by standoc-models (standoc.rnc +
+    # basicdoc.rnc, plus the standoc-presentation.rnc overlay) is
+    # either mapped by a model class here, delegated to its owning
+    # tree, or tracked as a gap. The grammar-coverage spec asserts the
+    # classification; `rake grammar_coverage` diffs it against a
+    # standoc-models checkout when the grammars change.
+    module GrammarVocabulary
+      ELEMENT_PATTERN = /element\s+([a-zA-Z0-9_.:-]+)\s*\{/
+
+      # Regenerate the vocabulary from a standoc-models grammars/
+      # directory: { semantic: [...], presentation: [...] }
+      def self.scan(grammars_dir)
+        semantic = ["#{grammars_dir}/standoc.rnc"]
+          .concat(Dir["#{grammars_dir}/basicdoc-models/grammars/*.rnc"])
+          .flat_map { |f| File.read(f, encoding: "utf-8").scan(ELEMENT_PATTERN).flatten }
+          .uniq.sort
+        presentation = File.read("#{grammars_dir}/standoc-presentation.rnc",
+                                 encoding: "utf-8")
+          .scan(ELEMENT_PATTERN).flatten.uniq.sort
+        { semantic: semantic, presentation: presentation }
+      end
+
+      SEMANTIC_ELEMENTS = %w[
+        abbreviation-type abstract acknowledgements add admitted admonition
+        altsource amend annex annotation annotation-container appendix
+        area asciimath attribute audio author autonumber
+        bibdata bibitem bibliography body boilerplate bookmark
+        br callout callout-annotation classification clause col
+        colgroup colophon columnbreak concept coords copyright-statement
+        date dd definition definitions del deprecates
+        description display-text dl document domain dt
+        em eref erefstack errormsg example executivesummary
+        expression feedback-statement field-of-application figure floating-title fn
+        foreword form formula gender grammar grammar-value
+        graphical-symbol hr image imagemap index index-xref
+        indexsect input introduction isAdjective isAdverb isNoun
+        isParticiple isPreposition isVerb key keyword label
+        latexmath legal-statement letter-symbol li license-statement link
+        location metanorma metanorma-extension modification name newcontent
+        non-verbal-representation note number ol option origin
+        p pagebreak passthrough pre preface preferred
+        primary pronunciation quote radius references refterm
+        related renderterm review ruby ruby-annotation ruby-pronunciation
+        secondary section sections select smallcap source
+        sourcecode span stem strike strong sub
+        subject sup svg svgmap table tag
+        target tbody td term termdocsource termexample
+        termnote termref terms tertiary textarea tfoot
+        th thead title toc tr tt
+        ul underline usage-info value variable-ref variant-title
+        verbal-definition video xref
+      ].freeze
+
+      PRESENTATION_ELEMENTS = %w[
+        annotation-container attribution author bibitem biblio-tag dd
+        emf eref fmt-admitted fmt-annotation-body fmt-annotation-end fmt-annotation-start
+        fmt-concept fmt-date-inline fmt-definition fmt-deprecates fmt-eref fmt-figure
+        fmt-fn-body fmt-fn-label fmt-footnote-container fmt-identifier fmt-link fmt-name
+        fmt-ol fmt-origin fmt-preferred fmt-provision fmt-related fmt-source
+        fmt-sourcecode fmt-stem fmt-termsource fmt-title fmt-ul fmt-xref-label
+        image li localized-string localized-strings metanorma-extension name
+        p passthrough rb rt ruby semx
+        source source-highlighter-css sourcelocalityStack svg tab term
+        termexample termnote title toc variant-title
+      ].freeze
+
+      # Elements owned by other model trees — mapped there, not here:
+      #   standoc  : the sections/terms/metadata tree (metanorma-standoc)
+      #   relaton  : the bibliographic models (relaton)
+      #   reqt     : the requirements vocabulary (requirements models)
+      #   basicdoc_only : defined by basicdoc, referenced by no standoc
+      #                   content model — dead vocabulary until adopted
+      DELEGATED = {
+        standoc: %w[
+          abbreviation-type acknowledgements amend annex annotation-container
+          appendix autonumber bibliography boilerplate clause colophon
+          copyright-statement executivesummary expression
+          feedback-statement field-of-application floating-title foreword form
+          gender graphical-symbol grammar grammar-value indexsect introduction
+          isAdjective isAdverb isNoun isParticiple isPreposition isVerb label
+          legal-statement letter-symbol license-statement metanorma-extension
+          newcontent non-verbal-representation option preface pronunciation
+          related sections select source-highlighter-css term
+          termdocsource termexample terms textarea toc usage-info
+          fmt-deprecates fmt-related
+        ],
+        relaton: %w[altsource],
+        reqt: %w[attribute],
+        # basicdoc's own parallel document structure (basicdoc root
+        # `document`, generic nested `section`) — standoc content
+        # models reference neither; the collection manifest root is the
+        # metanorma gem's own model.
+        basicdoc_only: %w[document section variable-ref],
+      }.freeze
+
+      # Presentation-overlay elements with no model yet. EMPTY since
+      # the fmt-twins wave (#63): every element this gem owns —
+      # semantic and rendered — parses. sourcelocalityStack parses its
+      # text; its child pattern is a dangling grammar reference
+      # (standoc-models issue), extended when the grammar defines it.
+      KNOWN_GAPS = [].freeze
+    end
+  end
+end

data/lib/metanorma/document/mirror_seed.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Document
+    # Registers this gem's model knowledge into the metanorma-mirror
+    # format gem. Components-based handlers run through the lazy
+    # register_default hook; the class tables (mark builders, simple
+    # mark types, rich renderers, text substitutions, semx classes),
+    # the inline content iterator, and the Components part of the
+    # positional id categories register eagerly — none of them touch
+    # constants outside this gem. metanorma-standoc seeds the section
+    # and structural handlers and its own id categories when it loads.
+    class MirrorSeed
+      MARK_BUILDERS = {
+        Components::Inline::EmRawElement => ->(_el) {
+          Mirror::Handlers.build_mark("emphasis")
+        },
+        Components::Inline::StrongRawElement => ->(_el) {
+          Mirror::Handlers.build_mark("strong")
+        },
+        Components::Inline::SubElement => ->(_el) {
+          Mirror::Handlers.build_mark("subscript")
+        },
+        Components::Inline::SupElement => ->(_el) {
+          Mirror::Handlers.build_mark("superscript")
+        },
+        Components::Inline::TtElement => ->(_el) {
+          Mirror::Handlers.build_mark("code")
+        },
+        Components::TextElements::UnderlineElement => ->(_el) {
+          Mirror::Handlers.build_mark("underline")
+        },
+        Components::TextElements::StrikeElement => ->(_el) {
+          Mirror::Handlers.build_mark("strike")
+        },
+        Components::Inline::SmallCapElement => ->(_el) {
+          Mirror::Handlers.build_mark("smallcap")
+        },
+        Components::Inline::Bcp14Element => ->(_el) {
+          Mirror::Handlers.build_mark("bcp14")
+        },
+        Components::Inline::LinkElement => ->(el) {
+          attrs = {
+            href: Mirror::SafeAttr.read(el, :target) ||
+              Mirror::SafeAttr.read(el, :href),
+          }.compact
+          Mirror::Handlers.build_mark("link", attrs: attrs)
+        },
+        Components::Inline::XrefElement => ->(el) {
+          attrs = { target: Mirror::SafeAttr.read(el, :target) }.compact
+          Mirror::Handlers.build_mark("xref", attrs: attrs)
+        },
+        Components::Inline::ErefElement => ->(el) {
+          attrs = {
+            bibitemid: Mirror::SafeAttr.read(el, :bibitemid),
+            citeas: Mirror::SafeAttr.read(el, :citeas),
+          }.compact
+          Mirror::Handlers.build_mark("eref", attrs: attrs)
+        },
+        Components::Inline::FnElement => ->(el) {
+          attrs = {
+            id: Mirror::SafeAttr.read(el, :id),
+            reference: Mirror::SafeAttr.read(el, :reference),
+          }.compact
+          Mirror::Handlers.build_mark("footnote", attrs: attrs)
+        },
+        Components::Inline::StemInlineElement => ->(el) {
+          attrs = { stem_type: Mirror::SafeAttr.read(el, :stem_type) ||
+            "MathML" }
+          math = Mirror::SafeAttr.read(el, :math)
+          attrs[:mathml] = Mirror::MathUtil.mathml_from_math(math) if math
+          Mirror::Handlers.build_mark("stem", attrs: attrs.compact)
+        },
+        Components::Inline::ConceptElement => ->(el) {
+          attrs = {
+            refterm: Mirror::SafeAttr.read(el, :refterm),
+            renderterm: Mirror::SafeAttr.read(el, :renderterm),
+          }.compact
+          Mirror::Handlers.build_mark("concept", attrs: attrs)
+        },
+        Components::Inline::SpanElement => ->(el) {
+          attrs = {
+            class_attr: Mirror::SafeAttr.read(el, :class_attr),
+            style: Mirror::SafeAttr.read(el, :style),
+          }.compact
+          Mirror::Handlers.build_mark("span", attrs: attrs)
+        },
+        Components::TextElements::RubyElement => ->(el) {
+          pron = Mirror::SafeAttr.read(el, :pronunciation)
+          ann = Mirror::SafeAttr.read(el, :annotation)
+          attrs = {
+            pronunciation: pron && Mirror::SafeAttr.read(pron, :value),
+            annotation: ann && Mirror::SafeAttr.read(ann, :value),
+            ruby_text: Array(Mirror::SafeAttr.read(el, :ruby_text)).join,
+          }.compact
+          Mirror::Handlers.build_mark("ruby", attrs: attrs)
+        },
+      }.freeze
+
+      SIMPLE_MARK_TYPES = {
+        Components::Inline::EmRawElement => "emphasis",
+        Components::Inline::StrongRawElement => "strong",
+        Components::Inline::SubElement => "subscript",
+        Components::Inline::SupElement => "superscript",
+        Components::Inline::TtElement => "code",
+        Components::TextElements::UnderlineElement => "underline",
+        Components::TextElements::StrikeElement => "strike",
+        Components::Inline::SmallCapElement => "smallcap",
+        Components::Inline::Bcp14Element => "bcp14",
+        Components::Inline::ConceptElement => "concept",
+      }.freeze
+
+      RICH_RENDERERS = {
+        Components::Inline::StemInlineElement => ->(el) {
+          Renderer.for_stem(el)
+        },
+        Components::TextElements::StemElement => ->(el) {
+          Renderer.for_stem(el)
+        },
+        Components::Inline::XrefElement => ->(el) {
+          Renderer.for_xref(el)
+        },
+        Components::Inline::LinkElement => ->(el) {
+          Renderer.for_link(el)
+        },
+        Components::Inline::ErefElement => ->(el) {
+          Renderer.for_eref(el)
+        },
+        Components::Inline::FnElement => ->(el) {
+          Renderer.for_fn(el)
+        },
+        Components::Inline::SpanElement => ->(el) {
+          Renderer.for_span(el)
+        },
+        Components::Inline::BrElement => ->(_el) {
+          Renderer.wrap(&:br)
+        },
+      }.freeze
+
+      Renderer = Mirror::Handlers::Inline::RichHtmlRenderer
+
+      POSITIONAL_CATEGORIES = {
+        Components::AncillaryBlocks::FigureBlock => :figure,
+        Components::Tables::TableBlock => :table,
+      }.freeze
+
+      class << self
+        def register
+          register_handler_hook
+          register_tables
+          register_iterator
+          register_positional_categories
+        end
+
+        private
+
+        def register_handler_hook
+          Mirror.register_default do |registry|
+            registry.register(
+              Components::Paragraphs::ParagraphBlock,
+              Mirror::Handlers::Paragraph,
+            )
+            registry.register(
+              Components::Blocks::NoteBlock, Mirror::Handlers::Note
+            )
+            registry.register(
+              Components::MultiParagraph::AdmonitionBlock,
+              Mirror::Handlers::Admonition,
+            )
+            registry.register(
+              Components::AncillaryBlocks::ExampleBlock,
+              Mirror::Handlers::Example,
+            )
+            registry.register(
+              Components::AncillaryBlocks::FigureBlock,
+              Mirror::Handlers::Figure,
+            )
+            registry.register(
+              Components::AncillaryBlocks::SvgmapElement,
+              Mirror::Handlers::Svgmap,
+            )
+            registry.register(
+              Components::AncillaryBlocks::ImagemapElement,
+              Mirror::Handlers::Imagemap,
+            )
+            registry.register(
+              Components::AncillaryBlocks::SourcecodeBlock,
+              Mirror::Handlers::Sourcecode,
+            )
+            registry.register(
+              Components::AncillaryBlocks::FormulaBlock,
+              Mirror::Handlers::Formula,
+            )
+            registry.register(
+              Components::MultiParagraph::QuoteBlock,
+              Mirror::Handlers::Quote,
+            )
+            registry.register(
+              Components::Tables::TableBlock, Mirror::Handlers::Table
+            )
+            registry.register(
+              Components::MultiParagraph::ReviewBlock,
+              Mirror::Handlers::Review,
+            )
+            registry.register(
+              Components::Lists::UnorderedList,
+              Mirror::Handlers::List, method_name: :bullet
+            )
+            registry.register(
+              Components::Lists::OrderedList,
+              Mirror::Handlers::List, method_name: :ordered
+            )
+            registry.register(
+              Components::Lists::DefinitionList,
+              Mirror::Handlers::List, method_name: :definition
+            )
+            registry.register(
+              Components::Lists::ListItem,
+              Mirror::Handlers::List, method_name: :list_item
+            )
+          end
+        end
+
+        def register_tables
+          MARK_BUILDERS.each do |klass, builder|
+            Mirror.mark_builders.register(klass, builder)
+          end
+          SIMPLE_MARK_TYPES.each do |klass, mark|
+            Mirror.simple_inline_elements.register(klass, mark)
+          end
+          RICH_RENDERERS.each do |klass, renderer|
+            Mirror.rich_html_renderers.register(klass, renderer)
+          end
+          Mirror.inline_text_substitutions
+            .register(Components::Inline::TabElement, " ")
+          Mirror.inline_text_substitutions
+            .register(Components::Inline::BrElement, "\n")
+          Mirror.semx_elements.register(Components::Inline::SemxElement, true)
+        end
+
+        def register_iterator
+          Mirror.inline_content_iterator = Components::Inline::SemanticContent
+        end
+
+        def register_positional_categories
+          POSITIONAL_CATEGORIES.each do |klass, category|
+            Mirror::IdStrategy::Positional.register_category(klass, category)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/document/mko/collection.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Metanorma
+  module Mko
+    # Collection export (MN 116 §collections): a metanorma
+    # collection.yml declares a document family (OIML R 60 parts,
+    # amendments, R 129/138/144). Every member document exports as its
+    # own bundle (the document bundle is the unit of truth); the
+    # collection adds one small bundle with the membership contract:
+    # collection.json and the cross-document part_of edges binding the
+    # members to the family. Member identity comes from each member's
+    # exported document.json — the collection yml's identifiers are
+    # reference labels only.
+    class Collection
+      Member = Struct.new(:fileref, :xml_path, :identifier,
+                          :bundle_path, :result, :skipped,
+                          keyword_init: true)
+
+      class Result
+        attr_reader :collection_bundle, :members
+
+        def initialize(collection_bundle:, members:)
+          @collection_bundle = collection_bundle
+          @members = members
+        end
+
+        def skipped
+          members.select(&:skipped)
+        end
+      end
+
+      class << self
+        def export(yml, to:, assets_from: nil, zip: false)
+          new(yml, to: to, assets_from: assets_from, zip: zip).export
+        end
+      end
+
+      def initialize(yml, to:, assets_from: nil, zip: false)
+        @yml = if yml.is_a?(Hash)
+                 yml
+               elsif File.exist?(yml.to_s)
+                 YAML.safe_load_file(yml)
+               else
+                 YAML.safe_load(yml)
+               end
+        @base = File.exist?(yml.to_s) ? File.dirname(File.expand_path(yml.to_s)) : Dir.pwd
+        @to = to
+        @assets_from = assets_from
+        @zip = zip
+      end
+
+      def export
+        members = docrefs.filter_map do |ref|
+          member = resolve_member(ref)
+          next member if member.skipped
+
+          export = Mko.export(
+            File.read(member.xml_path), to: @to,
+                                        presentation_xml: presentation_for(member.xml_path),
+                                        assets_from: @assets_from || File.dirname(member.xml_path)
+          )
+          member.bundle_path = export.path
+          member.result = export.result
+          member
+        end
+        Result.new(collection_bundle: write_collection_bundle(members),
+                   members: members)
+      end
+
+      private
+
+      def docrefs
+        manifest = @yml["manifest"]
+        manifest = manifest.first if manifest.is_a?(Array)
+        Array(manifest.is_a?(Hash) ? manifest["docref"] : nil)
+      rescue StandardError
+        []
+      end
+
+      # The yml points members at .adoc sources; the compiled semantic
+      # XML sits beside them. Direct .xml filerefs work too.
+      def resolve_member(ref)
+        fileref = ref["fileref"].to_s
+        xml = fileref.end_with?(".xml") ? fileref : fileref.sub(/\.adoc$/, ".xml")
+        path = File.expand_path(xml, @base)
+        if File.file?(path)
+          Member.new(fileref: fileref, xml_path: path,
+                     identifier: ref["identifier"])
+        else
+          Member.new(fileref: fileref, xml_path: path,
+                     identifier: ref["identifier"], skipped: true)
+        end
+      end
+
+      def presentation_for(xml_path)
+        pres = xml_path.sub(/\.xml$/, ".presentation.xml")
+        File.file?(pres) ? File.read(pres) : nil
+      end
+
+      def canonical
+        docid = @yml.dig("bibdata", "docid")
+        docid = docid.first if docid.is_a?(Array)
+        docid.is_a?(Hash) ? docid["id"].to_s : docid.to_s
+      end
+
+      def title
+        t = @yml.dig("bibdata", "title")
+        t = t.first if t.is_a?(Array)
+        t.is_a?(Hash) ? t["content"].to_s : t.to_s
+      end
+
+      def collection_short
+        Mko.slug(canonical.empty? ? "collection" : canonical)
+      end
+
+      # Member identity from the export's own Result — the same truth
+      # the bundle was written from, never a read-back of our output.
+      def member_data(member)
+        document = member.result.document
+        { "bundle" => "#{document.ids.short}.mko",
+          "docidentifier" => document.ids.canonical,
+          "identifier" => member.identifier,
+          "title" => document.titles.first&.text,
+          "edition" => document.edition }
+      rescue StandardError
+        nil
+      end
+
+      def collection_member(member)
+        data = member_data(member) or return nil
+
+        Schema::CollectionMember.new(
+          bundle: data["bundle"], docidentifier: data["docidentifier"],
+          identifier: data["identifier"], title: data["title"],
+          edition: data["edition"]
+        )
+      end
+
+      def write_collection_bundle(members)
+        collection = Schema::Collection.new(
+          canonical: canonical, short: collection_short, title: title,
+          edition: @yml.dig("bibdata", "edition"),
+          members: members.filter_map { |m| collection_member(m) }
+        )
+        # A member whose identity collides with the family (e.g. a
+        # stale amendment bibdata — mn-samples-oiml#66) would emit a
+        # self-edge; skip those, they say nothing.
+        edges = collection.members.reject do |m|
+          m.docidentifier.to_s.empty? || m.docidentifier == collection.canonical
+        end.map do |m|
+          { "from" => "doc:#{m.docidentifier}",
+            "to" => "doc:#{collection.canonical}",
+            "kind" => "part_of" }
+        end
+        bundle = Bundle.open(collection_short, @to)
+        bundle.flavor = nil
+        bundle.add_json("collection", "collection.json", collection)
+        bundle.add_lines("edges", "edges.jsonl", edges) do |edge|
+          JSON.generate(edge)
+        end
+        bundle.write_manifest
+        @zip ? bundle.zip! : bundle.dir
+      end
+    end
+  end
+end

data/lib/metanorma/document/mko/exporter.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Metanorma
+  module Mko
+    # Registry-facing exporter: quacks like a renderer (generate_full_document)
+    # so the Core::Flavors format table and the compile pipeline can select
+    # mko like any output format.
+    class Exporter
+      def generate_full_document(document, **options)
+        output = options[:output] || options[:output_filename] || "."
+        to = File.directory?(output) ? output : File.dirname(output)
+        Mko.export(document, to: to,
+                             presentation_xml: options[:presentation_xml])
+      end
+    end
+  end
+end