metanorma-standoc 1.11.2 → 2.0.1

data/lib/metanorma/standoc/base.rb

@@ -0,0 +1,163 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "isodoc"
+require "relaton"
+require "fileutils"
+require "metanorma-utils"
+require "isodoc/xslfo_convert"
+require_relative "render"
+
+module Metanorma
+  module Standoc
+    module Base
+      XML_ROOT_TAG = "standard-document".freeze
+      XML_NAMESPACE = "https://www.metanorma.org/ns/standoc".freeze
+      FONTS_MANIFEST = "fonts-manifest".freeze
+
+      def xml_root_tag
+        self.class::XML_ROOT_TAG
+      end
+
+      def xml_namespace
+        self.class::XML_NAMESPACE
+      end
+
+      def init(node)
+        @fn_number ||= 0
+        @draft = false
+        @refids = Set.new
+        @anchors = {}
+        @internal_eref_namespaces = []
+        @draft = node.attributes.has_key?("draft")
+        @novalid = node.attr("novalid")
+        @smartquotes = node.attr("smartquotes") != "false"
+        @keepasciimath = node.attr("mn-keep-asciimath") &&
+          node.attr("mn-keep-asciimath") != "false"
+        @fontheader = default_fonts(node)
+        @files_to_delete = []
+        @filename = if node.attr("docfile")
+                      File.basename(node.attr("docfile"))&.gsub(/\.adoc$/, "")
+                    else ""
+                    end
+        @localdir = Metanorma::Utils::localdir(node)
+        @output_dir = outputdir node
+        @no_isobib_cache = node.attr("no-isobib-cache")
+        @no_isobib = node.attr("no-isobib")
+        @index_terms = node.attr("index-terms")
+        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
+        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
+        @bibdb = nil
+        @seen_headers = []
+        @datauriimage = node.attr("data-uri-image") != "false"
+        @boilerplateauthority = node.attr("boilerplate-authority")
+        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
+        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
+        @log = Metanorma::Utils::Log.new
+        init_bib_caches(node)
+        init_iev_caches(node)
+        @lang = (node.attr("language") || "en")
+        @script = (node.attr("script") ||
+                   Metanorma::Utils.default_script(node.attr("language")))
+        @isodoc = isodoc(@lang, @script, node.attr("i18nyaml"))
+        @i18n = @isodoc.i18n
+        @htmltoclevels = node.attr("htmltoclevels")
+        @doctoclevels = node.attr("doctoclevels")
+        @toclevels = node.attr("toclevels")
+        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def document(node)
+        init(node)
+        ret = makexml(node).to_xml(encoding: "US-ASCII", indent: 2)
+        outputs(node, ret) unless node.attr("nodoc") || !node.attr("docfile")
+        clean_exit
+        ret
+      rescue StandardError => e
+        @log.add("Fatal Error", nil, e.message)
+        clean_exit
+        raise e
+      end
+
+      def version
+        flavour = self.class.name.sub(/::Converter$/, "").sub(/^.+::/, "")
+        Metanorma.versioned(Metanorma, flavour)[-1]::VERSION
+      end
+
+      def clean_exit
+        @log.write("#{@output_dir}#{@filename}.err") unless @novalid
+
+        @files_to_delete.each { |f| FileUtils.rm f }
+      end
+
+      def clean_abort(msg, file = nil)
+        file and
+          File.open("#{@filename}.xml.abort", "w:UTF-8") { |f| f.write(file) }
+        clean_exit
+        abort(msg)
+      end
+
+      def makexml1(node)
+        result = ["<?xml version='1.0' encoding='UTF-8'?>",
+                  "<#{xml_root_tag} type='semantic' version='#{version}'>"]
+        result << noko { |ixml| front node, ixml }
+        result << noko { |ixml| middle node, ixml }
+        result << "</#{xml_root_tag}>"
+        textcleanup(result)
+      end
+
+      def makexml(node)
+        result = makexml1(node)
+        ret1 = cleanup(Nokogiri::XML(result))
+        ret1.root.add_namespace(nil, xml_namespace)
+        validate(ret1) unless @novalid
+        ret1
+      end
+
+      def draft?
+        @draft
+      end
+
+      def doctype(node)
+        node.attr("doctype")&.gsub(/\s+/, "-")&.downcase
+      end
+
+      def front(node, xml)
+        xml.bibdata **attr_code(type: "standard") do |b|
+          metadata node, b
+        end
+      end
+
+      def middle(node, xml)
+        xml.sections do |s|
+          s << node.content if node.blocks?
+        end
+      end
+
+      def metadata_attrs(node)
+        node.attributes.each_with_object([]) do |(k, v), ret|
+          %w(presentation semantic).each do |t|
+            next unless /^#{t}-metadata-/.match?(k)
+
+            k = k.sub(/^#{t}-metadata-/, "")
+            csv_split(v, ",")&.each do |c|
+              ret << "<#{t}-metadata><#{k}>#{c}</#{k}></#{t}-metadata>"
+            end
+          end
+        end.join
+      end
+
+      private
+
+      def outputdir(node)
+        if node.attr("output_dir").nil_or_empty?
+          Metanorma::Utils::localdir(node)
+        else
+          File.join(node.attr("output_dir"), "")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/blocks.rb

@@ -0,0 +1,239 @@
+require "htmlentities"
+require "uri" if /^2\./.match?(RUBY_VERSION)
+require_relative "./blocks_notes"
+
+module Metanorma
+  module Standoc
+    module Blocks
+      def id_attr(node = nil)
+        { id: Metanorma::Utils::anchor_or_uuid(node),
+          tag: node&.attr("tag"),
+          "multilingual-rendering": node&.attr("multilingual-rendering") }
+      end
+
+      def id_unnum_attrs(node)
+        attr_code(id_attr(node).merge(
+                    unnumbered: node.option?("unnumbered") ? "true" : nil,
+                    number: node.attr("number"),
+                    subsequence: node.attr("subsequence"),
+                  ))
+      end
+
+      def formula_attrs(node)
+        attr_code(id_unnum_attrs(node)
+          .merge(keep_attrs(node).merge(
+                   inequality: node.option?("inequality") ? "true" : nil,
+                 )))
+      end
+
+      def keep_attrs(node)
+        { "keep-with-next": node.attr("keep-with-next"),
+          "keep-lines-together": node.attr("keep-lines-together") }
+      end
+
+      # We append each contained block to its parent
+      def open(node)
+        role = node.role || node.attr("style")
+        reqt_subpart(role) and return requirement_subpart(node)
+        role == "form" and return form(node)
+        role == "definition" and return termdefinition(node)
+        result = []
+        node.blocks.each do |b|
+          result << send(b.context, b)
+        end
+        result
+      end
+
+      def form(node)
+        noko do |xml|
+          xml.form **attr_code(id_attr(node)
+            .merge(class: node.attr("class"),
+                   name: node.attr("name"), action: node.attr("action"))) do |f|
+            f << node.content
+          end
+        end
+      end
+
+      def literal_attrs(node)
+        attr_code(id_attr(node).merge(keep_attrs(node)))
+      end
+
+      def literal(node)
+        noko do |xml|
+          xml.figure **literal_attrs(node) do |f|
+            figure_title(node, f)
+            f.pre node.lines.join("\n"),
+                  **attr_code(id: Metanorma::Utils::anchor_or_uuid,
+                              alt: node.attr("alt"))
+          end
+        end
+      end
+
+      # NOTE: html escaping is performed by Nokogiri
+      def stem(node)
+        noko do |xml|
+          xml.formula **formula_attrs(node) do |s|
+            stem_parse(node.lines.join("\n"), s, node.style.to_sym)
+          end
+        end
+      end
+
+      def term_example(node)
+        noko do |xml|
+          xml.termexample **attr_code(id_attr(node)) do |ex|
+            wrap_in_para(node, ex)
+          end
+        end.join("\n")
+      end
+
+      def example(node)
+        return term_example(node) if in_terms?
+
+        role = node.role || node.attr("style")
+        %w(recommendation requirement permission).include?(role) and
+          return requirement(node, role)
+        return pseudocode_example(node) if role == "pseudocode"
+        return svgmap_example(node) if role == "svgmap"
+        return form(node) if role == "form"
+        return termdefinition(node) if role == "definition"
+
+        reqt_subpart(role) and return requirement_subpart(node)
+        example_proper(node)
+      end
+
+      def svgmap_attrs(node)
+        attr_code(id_attr(node)
+          .merge(id: node.id,
+                 unnumbered: node.option?("unnumbered") ? "true" : nil,
+                 number: node.attr("number"),
+                 subsequence: node.attr("subsequence"))
+        .merge(keep_attrs(node)))
+      end
+
+      def svgmap_example(node)
+        noko do |xml|
+          xml.svgmap **attr_code(svgmap_attrs(node).merge(
+                                   src: node.attr("src"), alt: node.attr("alt"),
+                                 )) do |ex|
+            figure_title(node, ex)
+            ex << node.content
+          end
+        end.join("\n")
+      end
+
+      # prevent A's and other subs inappropriate for pseudocode
+      def pseudocode_example(node)
+        node.blocks.each { |b| b.remove_sub(:replacements) }
+        noko do |xml|
+          xml.figure **example_attrs(node).merge(class: "pseudocode") do |ex|
+            figure_title(node, ex)
+            wrap_in_para(node, ex)
+          end
+        end.join("\n")
+      end
+
+      def example_attrs(node)
+        attr_code(id_unnum_attrs(node).merge(keep_attrs(node)))
+      end
+
+      def example_proper(node)
+        noko do |xml|
+          xml.example **example_attrs(node) do |ex|
+            node.title.nil? or ex.name { |name| name << node.title }
+            wrap_in_para(node, ex)
+          end
+        end.join("\n")
+      end
+
+      def figure_title(node, out)
+        return if node.title.nil?
+
+        out.name { |name| name << node.title }
+      end
+
+      def figure_attrs(node)
+        attr_code(id_unnum_attrs(node).merge(keep_attrs(node)))
+      end
+
+      def image(node)
+        noko do |xml|
+          xml.figure **figure_attrs(node) do |f|
+            figure_title(node, f)
+            f.image **image_attributes(node)
+          end
+        end
+      end
+
+      def para_attrs(node)
+        attr_code(id_attr(node).merge(keep_attrs(node)
+          .merge(align: node.attr("align"),
+                 variant_title: node.role == "variant-title" ? true : nil,
+                 type: node.attr("type"))))
+      end
+
+      def paragraph(node)
+        return termsource(node) if node.role == "source"
+
+        noko do |xml|
+          xml.p **para_attrs(node) do |xml_t|
+            xml_t << node.content
+          end
+        end.join("\n")
+      end
+
+      def quote_attrs(node)
+        attr_code(id_attr(node).merge(keep_attrs(node))
+          .merge(align: node.attr("align")))
+      end
+
+      def quote_attribution(node, out)
+        if node.attr("citetitle")
+          m = /^(?<cite>[^,]+)(,(?<text>.*$))?$/m.match node.attr("citetitle")
+          out.source **attr_code(target: m[:cite], type: "inline") do |s|
+            s <<  m[:text]
+          end
+        end
+        node.attr("attribution") and
+          out.author { |a| a << node.attr("attribution") }
+      end
+
+      def quote(node)
+        noko do |xml|
+          xml.quote **quote_attrs(node) do |q|
+            quote_attribution(node, q)
+            wrap_in_para(node, q)
+          end
+        end.join("\n")
+      end
+
+      def listing_attrs(node)
+        attr_code(id_attr(node).merge(keep_attrs(node)
+                  .merge(lang: node.attr("language"),
+                         unnumbered: node.option?("unnumbered") ? "true" : nil,
+                         number: node.attr("number"),
+                         filename: node.attr("filename"))))
+      end
+
+      # NOTE: html escaping is performed by Nokogiri
+      def listing(node)
+        fragment = ::Nokogiri::XML::Builder.new do |xml|
+          xml.sourcecode **listing_attrs(node) do |s|
+            figure_title(node, s)
+            s << node.content
+          end
+        end
+        fragment.to_xml(encoding: "US-ASCII", save_with:
+                        Nokogiri::XML::Node::SaveOptions::NO_DECLARATION)
+      end
+
+      def pass(node)
+        noko do |xml|
+          xml.passthrough **attr_code(formats:
+                                      node.attr("format") || "metanorma") do |p|
+            p << HTMLEntities.new.encode(node.content, :basic, :hexadecimal)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/blocks_notes.rb

@@ -0,0 +1,101 @@
+module Metanorma
+  module Standoc
+    module Blocks
+      def termnote_attrs(node)
+        attr_code(id_attr(node).merge(keep_attrs(node)
+          .merge(
+            unnumbered: node.attr("unnumbered"),
+            number: node.attr("number"),
+            subsequence: node.attr("subsequence"),
+            "keep-separate": node.attr("keep-separate"),
+          )))
+      end
+
+      def note_attrs(node)
+        attr_code(
+          termnote_attrs(node).merge(
+            type: node.attr("type"),
+            beforeclauses: node.attr("beforeclauses") == "true" ? "true" : nil,
+          ),
+        )
+      end
+
+      def sidebar_attrs(node)
+        todo_attrs(node).merge(
+          attr_code(
+            from: node.attr("from"),
+            to: node.attr("to") || node.attr("from"),
+          ),
+        )
+      end
+
+      def sidebar(node)
+        return unless draft?
+
+        noko do |xml|
+          xml.review **sidebar_attrs(node) do |r|
+            wrap_in_para(node, r)
+          end
+        end
+      end
+
+      def todo_attrs(node)
+        date = node.attr("date") || Date.today.iso8601.gsub(/\+.*$/, "")
+        date += "T00:00:00Z" unless /T/.match? date
+        attr_code(id_attr(node)
+          .merge(reviewer: node.attr("reviewer") || node.attr("source") ||
+                 "(Unknown)",
+                 date: date))
+      end
+
+      def todo(node)
+        noko do |xml|
+          xml.review **todo_attrs(node) do |r|
+            wrap_in_para(node, r)
+          end
+        end
+      end
+
+      def termnote(node)
+        noko do |xml|
+          xml.termnote **termnote_attrs(node) do |ex|
+            wrap_in_para(node, ex)
+          end
+        end.join("\n")
+      end
+
+      def note(node)
+        noko do |xml|
+          xml.note **note_attrs(node) do |c|
+            wrap_in_para(node, c)
+          end
+        end.join("\n")
+      end
+
+      def admonition_attrs(node)
+        name = node.attr("name")
+        a = node.attr("type") and ["danger", "safety precautions"].each do |t|
+          name = t if a.casecmp(t).zero?
+        end
+        attr_code(keep_attrs(node).merge(id_attr(node)
+          .merge(
+            type: name,
+            beforeclauses: node.attr("beforeclauses") == "true" ? "true" : nil,
+          )))
+      end
+
+      def admonition(node)
+        return termnote(node) if in_terms?
+        return note(node) if node.attr("name") == "note"
+        return todo(node) if node.attr("name") == "todo"
+
+        noko do |xml|
+          xml.admonition **admonition_attrs(node) do |a|
+            node.title.nil? or a.name { |name| name << node.title }
+            wrap_in_para(node, a)
+          end
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup.rb

@@ -0,0 +1,157 @@
+require "nokogiri"
+require "pathname"
+require "html2doc"
+require_relative "./cleanup_block"
+require_relative "./cleanup_table"
+require_relative "./cleanup_footnotes"
+require_relative "./cleanup_ref"
+require_relative "./cleanup_ref_dl"
+require_relative "./cleanup_boilerplate"
+require_relative "./cleanup_section"
+require_relative "./cleanup_terms"
+require_relative "./cleanup_symbols"
+require_relative "./cleanup_xref"
+require_relative "./cleanup_inline"
+require_relative "./cleanup_amend"
+require_relative "./cleanup_maths"
+require_relative "./cleanup_image"
+require_relative "./cleanup_reqt"
+require_relative "./cleanup_text"
+require_relative "./cleanup_toc"
+require "relaton_iev"
+
+module Metanorma
+  module Standoc
+    module Cleanup
+      def cleanup(xmldoc)
+        element_name_cleanup(xmldoc)
+        sections_cleanup(xmldoc)
+        obligations_cleanup(xmldoc)
+        table_cleanup(xmldoc)
+        formula_cleanup(xmldoc)
+        form_cleanup(xmldoc)
+        sourcecode_cleanup(xmldoc)
+        figure_cleanup(xmldoc)
+        element_name_cleanup(xmldoc)
+        ref_cleanup(xmldoc)
+        note_cleanup(xmldoc)
+        clausebefore_cleanup(xmldoc)
+        floatingtitle_cleanup(xmldoc)
+        bibitem_cleanup(xmldoc)
+        normref_cleanup(xmldoc)
+        biblio_cleanup(xmldoc)
+        reference_names(xmldoc)
+        symbols_cleanup(xmldoc)
+        xref_cleanup(xmldoc)
+        concept_cleanup(xmldoc)
+        related_cleanup(xmldoc)
+        origin_cleanup(xmldoc)
+        bookmark_cleanup(xmldoc)
+        termdef_cleanup(xmldoc)
+        RelatonIev::iev_cleanup(xmldoc, @bibdb)
+        element_name_cleanup(xmldoc)
+        index_cleanup(xmldoc)
+        bpart_cleanup(xmldoc)
+        quotesource_cleanup(xmldoc)
+        callout_cleanup(xmldoc)
+        footnote_cleanup(xmldoc)
+        mathml_cleanup(xmldoc)
+        script_cleanup(xmldoc)
+        docidentifier_cleanup(xmldoc)
+        requirement_cleanup(xmldoc)
+        bibdata_cleanup(xmldoc)
+        svgmap_cleanup(xmldoc)
+        boilerplate_cleanup(xmldoc)
+        toc_cleanup(xmldoc)
+        metadata_cleanup(xmldoc)
+        smartquotes_cleanup(xmldoc)
+        variant_cleanup(xmldoc)
+        para_cleanup(xmldoc)
+        empty_element_cleanup(xmldoc)
+        img_cleanup(xmldoc)
+        anchor_cleanup(xmldoc)
+        xmldoc
+      end
+
+      def docidentifier_cleanup(xmldoc); end
+
+      TEXT_ELEMS =
+        %w{status language script version author name callout phone email
+           street city state country postcode identifier referenceFrom surname
+           referenceTo docidentifier docnumber prefix initial addition forename
+           title draft secretariat title-main title-intro title-part
+           verbal-definition non-verbal-representation}.freeze
+
+      # it seems Nokogiri::XML is treating the content of <script> as cdata,
+      # because of its use in HTML. Bad nokogiri. Undoing that, since we use
+      # script as a normal tag
+      def script_cleanup(xmldoc)
+        xmldoc.xpath("//script").each { |x| x.content = x.to_str }
+      end
+
+      def empty_element_cleanup(xmldoc)
+        xmldoc.xpath("//#{TEXT_ELEMS.join(' | //')}").each do |x|
+          next if x.name == "name" && x.parent.name == "expression"
+
+          x.remove if x.children.empty?
+        end
+      end
+
+      def element_name_cleanup(xmldoc)
+        xmldoc.traverse { |n| n.name = n.name.gsub(/_/, "-") }
+      end
+
+      # allows us to deal with doc relation localities,
+      # temporarily stashed to "bpart"
+      def bpart_cleanup(xmldoc)
+        xmldoc.xpath("//relation/bpart").each do |x|
+          extract_localities(x)
+          x.replace(x.children)
+        end
+      end
+
+      def variant_cleanup(xmldoc)
+        variant_space_cleanup(xmldoc)
+        xmldoc.xpath("//*[variant]").each do |c|
+          next unless c.children.any? do |n|
+            n.name != "variant" && (!n.text? || !n.text.gsub(/\s/, "").empty?)
+          end
+
+          variant_cleanup1(c)
+        end
+        xmldoc.xpath("//variantwrap").each { |n| n.name = "variant" }
+      end
+
+      def variant_cleanup1(elem)
+        elem.xpath("./variant").each do |n|
+          if n.at_xpath("preceding-sibling::node()"\
+                        "[not(self::text()[not(normalize-space())])][1]"\
+                        "[self::variantwrap]")
+            n.previous_element << n
+          else
+            n.replace("<variantwrap/>").first << n
+          end
+        end
+      end
+
+      def variant_space_cleanup(xmldoc)
+        xmldoc.xpath("//*[variant]").each do |c|
+          next if c.next.nil? || c.next.next.nil?
+
+          if c.next.text? && c.next.next.name == "variant"
+            c.next.text.gsub(/\s/, "").empty? and
+              c.next.remove
+          end
+        end
+      end
+
+      def metadata_cleanup(xmldoc)
+        return if @metadata_attrs.nil? || @metadata_attrs.empty?
+
+        ins = xmldoc.at("//misc-container") ||
+          xmldoc.at("//bibdata").after("<misc-container/>").next_element
+        ins << @metadata_attrs
+      end
+    end
+  end
+end