metanorma-standoc 2.0.3 → 2.0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cc4b55c7be36492191bd86bc19646a520c76e94bd2e2af9f96cf0d80d25fb05e
-  data.tar.gz: 533fc5380552ab239528a559ee41a47c246203e17cb33df64cb5cdef5df21f34
+  metadata.gz: cfc9dd3ae0dd167d4c3b92c4b051619cde74ae602a66c9b34e363af1498682a0
+  data.tar.gz: 1cde7c4c003cdf59c2ef85e0ec1fc3247095716b6fc4f99b121c638f0744aaf9
 SHA512:
-  metadata.gz: adc2d0b10dd30e5f59060b20574701dc6049b25a9d25eff06557ac51ca66645f62b553517690380009460f01fe135d26e83ab653b81c7746209e8f55221c3703
-  data.tar.gz: 8bc78cf1ab42e80d93c8a22f0028e44e7ba735914fa43c2dcdd5eb53cb57b3a3ee52ea1677cae5033dbc738f8b444bc0dc2e5021ff08a443d991535c84c373b2
+  metadata.gz: af0f552acb3f93c8c2e1f1bff1ca60fadff61a5242e08bf0de29df086906a14025a1f6b2d2fb23a6e97afdd086fb60c9b812a91359056e6195a42a6d1d305630
+  data.tar.gz: cfad71e3de0a31950aee00c477a50449b97eb436ea6261584b73c616d7d24d30ab380ff7504b0357bd5e234752639a7705a43dca46f9f15983114ea50eaf2210

data/lib/metanorma/standoc/base.rb

@@ -26,17 +26,54 @@ module Metanorma
       end
 
       def init(node)
+        init_vars
+        init_misc(node)
+        init_processing(node)
+        init_toc(node)
+        init_output(node)
+        init_i18n(node)
+        init_biblio(node)
+        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def init_vars
         @fn_number ||= 0
-        @draft = false
         @refids = Set.new
         @anchors = {}
         @internal_eref_namespaces = []
+        @seen_headers = []
+        @embed_hdr = []
+      end
+
+      def init_misc(node)
         @draft = node.attributes.has_key?("draft")
+        @index_terms = node.attr("index-terms")
+        @boilerplateauthority = node.attr("boilerplate-authority")
+        @embed_hdr = node.attr("embed_hdr")
+      end
+
+      def init_processing(node)
         @novalid = node.attr("novalid")
         @smartquotes = node.attr("smartquotes") != "false"
         @keepasciimath = node.attr("mn-keep-asciimath") &&
           node.attr("mn-keep-asciimath") != "false"
+        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
+        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
+        @datauriimage = node.attr("data-uri-image") != "false"
+      end
+
+      def init_toc(node)
+        @htmltoclevels = node.attr("htmltoclevels")
+        @doctoclevels = node.attr("doctoclevels")
+        @toclevels = node.attr("toclevels")
+        @tocfigures = node.attr("toc-figures")
+        @toctables = node.attr("toc-tables")
+        @tocrecommendations = node.attr("toc-recommendations")
+      end
+
+      def init_output(node)
         @fontheader = default_fonts(node)
+        @log = Metanorma::Utils::Log.new
         @files_to_delete = []
         @filename = if node.attr("docfile")
                       File.basename(node.attr("docfile"))&.gsub(/\.adoc$/, "")
@@ -44,35 +81,26 @@ module Metanorma
                     end
         @localdir = Metanorma::Utils::localdir(node)
         @output_dir = outputdir node
-        @no_isobib_cache = node.attr("no-isobib-cache")
-        @no_isobib = node.attr("no-isobib")
-        @index_terms = node.attr("index-terms")
-        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
-        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
-        @bibdb = nil
-        @seen_headers = []
-        @datauriimage = node.attr("data-uri-image") != "false"
-        @boilerplateauthority = node.attr("boilerplate-authority")
-        @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
-        @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
-        @log = Metanorma::Utils::Log.new
-        init_bib_caches(node)
-        init_iev_caches(node)
+      end
+
+      def init_i18n(node)
         @lang = (node.attr("language") || "en")
         @script = (node.attr("script") ||
                    Metanorma::Utils.default_script(node.attr("language")))
         @isodoc = isodoc(@lang, @script, node.attr("i18nyaml"))
         @i18n = @isodoc.i18n
-        @htmltoclevels = node.attr("htmltoclevels")
-        @doctoclevels = node.attr("doctoclevels")
-        @toclevels = node.attr("toclevels")
-        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def init_biblio(node)
+        @no_isobib_cache = node.attr("no-isobib-cache")
+        @no_isobib = node.attr("no-isobib")
+        @bibdb = nil
+        init_bib_caches(node)
+        init_iev_caches(node)
       end
 
       def document(node)
-        init(node)
-        ret = makexml(node).to_xml(encoding: "UTF-8", indent: 2)
-        outputs(node, ret) unless node.attr("nodoc") || !node.attr("docfile")
+        ret = document1(node)
         clean_exit
         ret
       rescue StandardError => e
@@ -81,6 +109,28 @@ module Metanorma
         raise e
       end
 
+      def document1(node)
+        init(node)
+        ret = makexml(node)
+          .to_xml(encoding: "UTF-8", indent: 2,
+                  save_with: Nokogiri::XML::Node::SaveOptions::AS_XML)
+        outputs(node, ret) unless node.attr("nodoc") || !node.attr("docfile")
+        ret
+      end
+
+      def insert_xml_cr(doc)
+        doc
+          .gsub(%r{(</(clause|table|figure|p|bibitem|ul|ol|dl|dt|dd|li|example|
+                       sourcecode|formula|quote|references|annex|appendix|title|
+                       name|note|thead|tbody|tfoot|th|td|form|requirement|
+                       recommendation|permission|imagemap|svgmap|preferred|
+                       admitted|related|deprecates|letter-symbol|domain|
+                       graphical-symbol|expression|abbreviation-type|subject|
+                       pronunciation|grammar|term|terms|termnote|termexample|
+                       termsource|origin|termref|modification)>)}x, "\\1\n")
+          .gsub(%r{(<(title|name))}x, "\n\\1")
+      end
+
       def version
         flavour = self.class.name.sub(/::Converter$/, "").sub(/^.+::/, "")
         Metanorma.versioned(Metanorma, flavour)[-1]::VERSION
@@ -110,7 +160,7 @@ module Metanorma
 
       def makexml(node)
         result = makexml1(node)
-        ret1 = cleanup(Nokogiri::XML(result))
+        ret1 = cleanup(Nokogiri::XML(insert_xml_cr(result)))
         ret1.root.add_namespace(nil, xml_namespace)
         validate(ret1) unless @novalid
         ret1

data/lib/metanorma/standoc/blocks.rb

@@ -188,7 +188,7 @@ module Metanorma
 
       def quote_attribution(node, out)
         if node.attr("citetitle")
-          m = /^(?<cite>[^,]+)(,(?<text>.*$))?$/m.match node.attr("citetitle")
+          m = /^(?<cite>[^,]+)(?:,(?<text>.*$))?$/m.match node.attr("citetitle")
           out.source **attr_code(target: m[:cite], type: "inline") do |s|
             s <<  m[:text]
           end

data/lib/metanorma/standoc/blocks_notes.rb

@@ -36,7 +36,7 @@ module Metanorma
           xml.review **sidebar_attrs(node) do |r|
             wrap_in_para(node, r)
           end
-        end
+        end.join("\n")
       end
 
       def todo_attrs(node)
@@ -53,7 +53,7 @@ module Metanorma
           xml.review **todo_attrs(node) do |r|
             wrap_in_para(node, r)
           end
-        end
+        end.join("\n")
       end
 
       def termnote(node)

data/lib/metanorma/standoc/cleanup_amend.rb

@@ -1,53 +1,56 @@
 module Metanorma
   module Standoc
     module Cleanup
-      def change_clauses(x)
-        x.xpath("//clause[@change]").each do |c|
-          a = create_amend(c)
+      def change_clauses(docxml)
+        docxml.xpath("//clause[@change]").each do |c|
+          create_amend(c)
         end
       end
 
-      def create_amend(c)
-        a = c.add_child("<amend id='_#{UUIDTools::UUID.random_create}'/>").first
-        c.elements.each do |e|
+      def create_amend(clause)
+        a = clause.add_child("<amend id='_#{UUIDTools::UUID.random_create}'/>")
+          .first
+        clause.elements.each do |e|
           e.parent = a unless %w(amend title).include? e.name
         end
-        create_amend1(c, a)
+        create_amend1(clause, a)
       end
 
-      def create_amend1(c, a)
-        create_amend2(c, a)
-        d = a.at("./description")
+      def create_amend1(clause, amend)
+        create_amend2(clause, amend)
+        d = amend.at("./description")
         d.xpath(".//autonumber").each { |e| d.previous = e }
-        d.xpath(".//p[normalize-space(.)='']").each { |e| e.remove }
-        move_attrs_to_amend(c, a)
-        a
+        d.xpath(".//p[normalize-space(.)='']").each(&:remove)
+        move_attrs_to_amend(clause, amend)
+        amend
       end
 
-      def create_amend2(c, a)
-        q = a.at("./quote") and q.name = "newcontent"
+      def create_amend2(_clause, amend)
+        q = amend.at("./quote") and q.name = "newcontent"
         if q.nil?
-          a.children = "<description>#{a.children.to_xml}</description>"
-        else
-          pre = q&.xpath("./preceding-sibling::*")&.remove
-          post = q&.xpath("./following-sibling::*")&.remove
-          pre.empty? or a << "<description>#{pre.to_xml}</description>"
-          a << q.remove
-          post.empty? or a << "<description>#{post.to_xml}</description>"
+          amend.children = "<description>#{amend.children.to_xml}</description>"
+          return
         end
+        pre = q&.xpath("./preceding-sibling::*")&.remove
+        post = q&.xpath("./following-sibling::*")&.remove
+        pre.empty? or amend << "<description>#{pre.to_xml}</description>"
+        amend << q.remove
+        post.empty? or amend << "<description>#{post.to_xml}</description>"
       end
 
-      def move_attrs_to_amend(c, a)
+      def move_attrs_to_amend(clause, amend)
         %w(change path path_end title).each do |e|
-          next unless c[e]
-          a[e] = c[e]
-          c.delete(e)
+          next unless clause[e]
+
+          amend[e] = clause[e]
+          clause.delete(e)
         end
-        return unless a["locality"]
-        loc = a.children.add_previous_sibling("<location/>")
-        extract_localities1(loc, a["locality"])
+        return unless amend["locality"]
+
+        loc = amend.children.add_previous_sibling("<location/>")
+        extract_localities1(loc, amend["locality"])
         loc1 = loc.at("./localityStack") and loc.replace(loc1.elements)
-        a.delete("locality")
+        amend.delete("locality")
       end
     end
   end

data/lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -56,9 +56,9 @@ module Metanorma
       TERM_CLAUSE = "//sections/terms | "\
                     "//sections/clause[descendant::terms]".freeze
 
-      NORM_REF = "//bibliography/references[@normative = 'true'] | "\
-                 "//bibliography/clause[.//references[@normative = 'true']]"
-        .freeze
+      NORM_REF =
+        "//bibliography/references[@normative = 'true'][not(@hidden)] | "\
+        "//bibliography/clause[.//references[@normative = 'true']]".freeze
 
       def boilerplate_isodoc(xmldoc)
         x = xmldoc.dup
@@ -133,6 +133,7 @@ module Metanorma
       def bibdata_cleanup(xmldoc)
         bibdata_anchor_cleanup(xmldoc)
         bibdata_docidentifier_cleanup(xmldoc)
+        bibdata_embed_hdr_cleanup(xmldoc)
         biblio_indirect_erefs(xmldoc, @internal_eref_namespaces&.uniq)
       end
 
@@ -190,8 +191,8 @@ module Metanorma
       def resolve_local_indirect_erefs(xmldoc, refs, prefix)
         refs.each_with_object([]) do |r, m|
           id = r.sub(/^#{prefix}_/, "")
-          if n = xmldoc.at("//*[@id = '#{id}']") and
-              n.at("./ancestor-or-self::*[@type = '#{prefix}']")
+          n = xmldoc.at("//*[@id = '#{id}']")
+          if n&.at("./ancestor-or-self::*[@type = '#{prefix}']")
             xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
               indirect_eref_to_xref(e, id)
             end
@@ -208,6 +209,29 @@ module Metanorma
           insert_indirect_biblio(xmldoc, refs, prefix)
         end
       end
+
+      def bibdata_embed_hdr_cleanup(xmldoc)
+        return if @embed_hdr.nil? || @embed_hdr.empty?
+
+        embed_recurse(xmldoc.at("//bibdata"), @embed_hdr.first)
+      end
+
+      def hdr2bibitem(hdr)
+        xml = Asciidoctor
+          .convert(hdr[:text], backend: Processor.new.asciidoctor_backend,
+                               header_footer: true)
+        b = Nokogiri::XML(xml).at("//xmlns:bibdata")
+        b.name = "bibitem"
+        b.delete("type")
+        embed_recurse(b, hdr)
+        b.to_xml
+      end
+
+      def embed_recurse(bibitem, node)
+        node[:child].map { |x| hdr2bibitem(x) }.each do |x|
+          bibitem << "<relation type='derivedFrom'>#{x}</relation>"
+        end
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_ref.rb

@@ -16,7 +16,7 @@ module Metanorma
         insert = refs&.at("./bibitem")&.previous_element
         refs.xpath("./bibitem").each(&:remove)
         bib.reverse.each do |b|
-          insert and insert.next = b.to_xml or
+          (insert and insert.next = b.to_xml) or
             refs.children.first.add_previous_sibling b.to_xml
         end
         extract_notes_from_biblio(refs)
@@ -84,9 +84,39 @@ module Metanorma
         biblio_reorder(xmldoc)
         biblio_nested(xmldoc)
         biblio_renumber(xmldoc)
+        biblio_linkonly(xmldoc)
+        biblio_hidden_inherit(xmldoc)
         biblio_no_ext(xmldoc)
       end
 
+      def biblio_hidden_inherit(xmldoc)
+        xmldoc.xpath("//references[@hidden = 'true']").each do |r|
+          r.xpath("./bibitem").each do |b|
+            b["hidden"] = true
+          end
+        end
+      end
+
+      def biblio_linkonly(xmldoc)
+        return unless xmldoc.at("//xref[@hidden]")
+
+        ins = xmldoc.at("//bibliography")
+          .add_child("<references hidden='true' normative='true'/>").first
+        refs = xmldoc.xpath("//xref[@hidden]").each_with_object([]) do |x, m|
+          @refids << x["target"]
+          m << { id: x["target"], ref: x["hidden"] }
+          x.delete("hidden")
+        end
+        ins << insert_hidden_bibitems(refs)
+      end
+
+      def insert_hidden_bibitems(bib)
+        refs = bib.each_with_object([]) do |b, m|
+          m << reference1code(%(<ref id="#{b[:id]}">[#{b[:ref]}]</ref>), nil)
+        end
+        reference_populate(refs)
+      end
+
       def biblio_no_ext(xmldoc)
         xmldoc.xpath("//bibitem/ext").each(&:remove)
       end
@@ -114,7 +144,6 @@ module Metanorma
 
       def reference_names(xmldoc)
         xmldoc.xpath("//bibitem[not(ancestor::bibitem)]").each do |ref|
-          # isopub = ref.at(ISO_PUBLISHER_XPATH)
           docid = ref.at("./docidentifier[@type = 'metanorma']") ||
             ref.at("./docidentifier[not(@type = 'DOI')]") or next
           reference = format_ref(docid.children.to_xml, docid["type"])

data/lib/metanorma/standoc/cleanup_ref_dl.rb

@@ -15,19 +15,25 @@ module Metanorma
         end
       end
 
-      def validate_ref_dl(bib, c)
+      # do not accept implicit id
+      def validate_ref_dl(bib, clause)
         id = bib["id"]
-        id ||= c["id"] unless /^_/.match?(c["id"]) # do not accept implicit id
+        id ||= clause["id"] unless /^_/.match?(clause["id"])
         unless id
-          @log.add("Anchors", c,
-                   "The following reference is missing an anchor:\n" + c.to_xml)
+          @log.add("Anchors", clause,
+                   "The following reference is missing an anchor:\n"\
+                   "#{clause.to_xml}")
           return
         end
         @refids << id
+        validate_ref_dl1(bib, id, clause)
+      end
+
+      def validate_ref_dl1(bib, id, clause)
         bib["title"] or
-          @log.add("Bibliography", c, "Reference #{id} is missing a title")
+          @log.add("Bibliography", clause, "Reference #{id} is missing a title")
         bib["docid"] or
-          @log.add("Bibliography", c,
+          @log.add("Bibliography", clause,
                    "Reference #{id} is missing a document identifier (docid)")
       end
 
@@ -76,27 +82,30 @@ module Metanorma
       end
 
       # definition list, with at most one level of unordered lists
-      def dl_bib_extract(c, nested = false)
-        dl = c.at("./dl") or return
+      def dl_bib_extract(clause, nested = false)
+        dl = clause.at("./dl") or return
         bib = {}
         key = ""
         dl.xpath("./dt | ./dd").each do |dtd|
           (dtd.name == "dt" and key = dtd.text.sub(/:+$/, "")) or
             add_to_hash(bib, key, dd_bib_extract(dtd))
         end
-        c.xpath("./clause").each do |c1|
+        clause.xpath("./clause").each do |c1|
           key = c1&.at("./title")&.text&.downcase&.strip
           next unless %w(contributor relation series).include? key
 
           add_to_hash(bib, key, dl_bib_extract(c1, true))
         end
-        if !nested && c.at("./title")
-          title = c.at("./title").remove.children.to_xml
-          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
-          bib["title"] = [bib["title"]] if bib["title"].is_a? String
-          bib["title"] = [] unless bib["title"]
-          bib["title"] << title if !title.empty?
-        end
+        dl_bib_extract_title(bib, clause, nested)
+      end
+
+      def dl_bib_extract_title(bib, clause, nested)
+        (!nested && clause.at("./title")) or return bib
+        title = clause.at("./title").remove.children.to_xml
+        bib["title"] = [bib["title"]] if bib["title"].is_a?(Hash) ||
+          bib["title"].is_a?(String)
+        bib["title"] = [] unless bib["title"]
+        bib["title"] << title if !title.empty?
         bib
       end
     end

data/lib/metanorma/standoc/cleanup_section.rb

@@ -54,7 +54,7 @@ module Metanorma
       end
 
       def make_bibliography(xml, sect)
-        if xml.at("//sections/references")
+        if xml.at("//sections/references | //xref[@hidden]")
           biblio = sect.add_next_sibling("<bibliography/>").first
           xml.xpath("//sections/references").each do |r|
             biblio.add_child r.remove

data/lib/metanorma/standoc/cleanup_terms_designations.rb

@@ -182,10 +182,12 @@ module Metanorma
 
       def term_designation_reorder(xmldoc)
         xmldoc.xpath("//term").each do |t|
-          %w(preferred admitted deprecates related)
+          des = %w(preferred admitted deprecates related)
             .each_with_object([]) do |tag, m|
             t.xpath("./#{tag}").each { |x| m << x.remove }
-          end.reverse.each do |x|
+          end.reverse
+          t << " "
+          des.each do |x|
             t.children.first.previous = x
           end
         end

data/lib/metanorma/standoc/cleanup_text.rb

@@ -9,12 +9,6 @@ module Metanorma
                   </passthrough>}mx) { HTMLEntities.new.decode($1) }
       end
 
-      IGNORE_DUMBQUOTES =
-        "//pre | //pre//* | //tt | //tt//* | "\
-        "//sourcecode | //sourcecode//* | //bibdata//* | //stem | "\
-        "//stem//* | //figure[@class = 'pseudocode'] | "\
-        "//figure[@class = 'pseudocode']//*".freeze
-
       def smartquotes_cleanup(xmldoc)
         xmldoc.xpath("//date").each { |d| Metanorma::Utils::endash_date(d) }
         if @smartquotes then smartquotes_cleanup1(xmldoc)
@@ -31,14 +25,21 @@ module Metanorma
       def uninterrupt_quotes_around_xml(xmldoc)
         xmldoc.traverse do |n|
           next unless n.text? && n&.previous&.element?
-          next unless /^['"]/.match?(n.text)
-          next unless n.previous.ancestors("pre, tt, sourcecode, stem, figure")
-            .empty?
+          next if uninterrupt_quotes_around_xml_skip(n)
 
           uninterrupt_quotes_around_xml1(n.previous)
         end
       end
 
+      def uninterrupt_quotes_around_xml_skip(elem)
+        !(/^['"]/.match?(elem.text) &&
+          elem.previous.ancestors("pre, tt, sourcecode, stem, figure, bibdata")
+          .empty? &&
+          ((elem.previous.text.strip.empty? &&
+            !empty_tag_with_text_content?(elem.previous)) ||
+           elem.previous.name == "index"))
+      end
+
       def uninterrupt_quotes_around_xml1(elem)
         prev = elem.at(".//preceding::text()[1]") or return
         /\S$/.match?(prev.text) or return
@@ -49,19 +50,40 @@ module Metanorma
         prev.content = "#{prev.text}#{m[1]}"
       end
 
-      def dumb2smart_quotes(xmldoc)
-        (xmldoc.xpath("//*[child::text()]") - xmldoc.xpath(IGNORE_DUMBQUOTES))
-          .each do |x|
-          x.children.each do |n|
-            next unless n.text?
+      def block?(elem)
+        %w(title name variant-title clause figure annex example introduction
+           foreword acknowledgements note li th td dt dd p quote label
+           abstract preferred admitted related deprecates field-of-application
+           usage-info expression pronunciation grammar-value domain
+           definition termnote termexample modification description
+           newcontent floating-title).include? elem.name
+      end
 
-            /[-'"(<>]|\.\.|\dx/.match(n) or next
+      def empty_tag_with_text_content?(elem)
+        %w(eref xref termref link).include? elem.name
+      end
 
-            n.replace(Metanorma::Utils::smartformat(n.text))
-          end
+      def dumb2smart_quotes(xmldoc)
+        prev = ""
+        xmldoc.traverse do |x|
+          block?(x) and prev = ""
+          empty_tag_with_text_content?(x) and prev = "dummy"
+          next unless x.text?
+
+          x.ancestors("pre, tt, sourcecode, stem, figure, bibdata").empty? and
+            dumb2smart_quotes1(x, prev)
+          prev = x.text if x.ancestors("index").empty?
         end
       end
 
+      def dumb2smart_quotes1(curr, prev)
+        /[-'"(<>]|\.\.|\dx/.match?(curr.text) or return
+
+        /^["']/.match?(curr.text) && prev.match?(/\S$/) and
+          curr.content = curr.text.sub(/^"/, "”").sub(/"’"/, "‘")
+        curr.replace(Metanorma::Utils::smartformat(curr.text))
+      end
+
       def dumbquote_cleanup(xmldoc)
         xmldoc.traverse do |n|
           next unless n.text?