metanorma-standoc 2.0.1 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5ab9cbdae658ec4088c0670d2ce1a117799a9325dbb8b8aa24e59b44e94e24c
-  data.tar.gz: da30d4d7b276b67e2363643e8ae1296a352bc4acee0ef9f2727da00ad276b3cc
+  metadata.gz: 48ca241528c217b9b563a2d3f6f2e13b52f2fa24e679ca2de431ebde8d10fb98
+  data.tar.gz: f6b70cd60a959376474a35ef4d364f69f9e15fd6582a53571a71cec995d90a01
 SHA512:
-  metadata.gz: e8600c60d56d27820da72dbc7259db4e3a91a3e362a4cefa0a5497b545076dab93277e87c5a0368f3037249ed79b5421e92300a8847bcf7c5bdcaca2392a3eab
-  data.tar.gz: 216bdaa7621b5b722f65a7d86eabbeaf65788df7f135927d04044838c6db31d50999155de829df6e57f5563a4f201121f36cde9c9c023a9af546019ab65d848e
+  metadata.gz: f175a4b49b2aa24d8c8b566a6929401f1a61fb217cf2ee03b506d93482a086afb7b09c85e7c6d88c93d1ef57d41e5a79dc6122bf41624842118dfa75759823b4
+  data.tar.gz: c395e93a7f1d3c03c63f1063f1689beb3e1725b81d26b2b38769038d491e8fade8462ab1b2f3b6324eede90c7e03b1473ce40873b903fd3859852436d69e7a9a

data/lib/metanorma/standoc/base.rb

@@ -31,48 +31,65 @@ module Metanorma
         @refids = Set.new
         @anchors = {}
         @internal_eref_namespaces = []
+        @seen_headers = []
         @draft = node.attributes.has_key?("draft")
         @novalid = node.attr("novalid")
         @smartquotes = node.attr("smartquotes") != "false"
         @keepasciimath = node.attr("mn-keep-asciimath") &&
           node.attr("mn-keep-asciimath") != "false"
-        @fontheader = default_fonts(node)
-        @files_to_delete = []
-        @filename = if node.attr("docfile")
-                      File.basename(node.attr("docfile"))&.gsub(/\.adoc$/, "")
-                    else ""
-                    end
-        @localdir = Metanorma::Utils::localdir(node)
-        @output_dir = outputdir node
-        @no_isobib_cache = node.attr("no-isobib-cache")
-        @no_isobib = node.attr("no-isobib")
         @index_terms = node.attr("index-terms")
         @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
         @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
-        @bibdb = nil
-        @seen_headers = []
         @datauriimage = node.attr("data-uri-image") != "false"
         @boilerplateauthority = node.attr("boilerplate-authority")
         @sourcecode_markup_start = node.attr("sourcecode-markup-start") || "{{{"
         @sourcecode_markup_end = node.attr("sourcecode-markup-end") || "}}}"
+        init_toc(node)
+        init_output(node)
+        init_i18n(node)
+        init_biblio(node)
+        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def init_toc(node)
+        @htmltoclevels = node.attr("htmltoclevels")
+        @doctoclevels = node.attr("doctoclevels")
+        @toclevels = node.attr("toclevels")
+        @tocfigures = node.attr("toc-figures")
+        @toctables = node.attr("toc-tables")
+        @tocrecommendations = node.attr("toc-recommendations")
+      end
+
+      def init_output(node)
+        @fontheader = default_fonts(node)
         @log = Metanorma::Utils::Log.new
-        init_bib_caches(node)
-        init_iev_caches(node)
+        @files_to_delete = []
+        @filename = if node.attr("docfile")
+                      File.basename(node.attr("docfile"))&.gsub(/\.adoc$/, "")
+                    else ""
+                    end
+        @localdir = Metanorma::Utils::localdir(node)
+        @output_dir = outputdir node
+      end
+
+      def init_i18n(node)
         @lang = (node.attr("language") || "en")
         @script = (node.attr("script") ||
                    Metanorma::Utils.default_script(node.attr("language")))
         @isodoc = isodoc(@lang, @script, node.attr("i18nyaml"))
         @i18n = @isodoc.i18n
-        @htmltoclevels = node.attr("htmltoclevels")
-        @doctoclevels = node.attr("doctoclevels")
-        @toclevels = node.attr("toclevels")
-        @metadata_attrs = metadata_attrs(node)
+      end
+
+      def init_biblio(node)
+        @no_isobib_cache = node.attr("no-isobib-cache")
+        @no_isobib = node.attr("no-isobib")
+        @bibdb = nil
+        init_bib_caches(node)
+        init_iev_caches(node)
       end
 
       def document(node)
-        init(node)
-        ret = makexml(node).to_xml(encoding: "US-ASCII", indent: 2)
-        outputs(node, ret) unless node.attr("nodoc") || !node.attr("docfile")
+        ret = document1(node)
         clean_exit
         ret
       rescue StandardError => e
@@ -81,6 +98,28 @@ module Metanorma
         raise e
       end
 
+      def document1(node)
+        init(node)
+        ret = makexml(node)
+          .to_xml(encoding: "UTF-8", indent: 2,
+                  save_with: Nokogiri::XML::Node::SaveOptions::AS_XML)
+        outputs(node, ret) unless node.attr("nodoc") || !node.attr("docfile")
+        ret
+      end
+
+      def insert_xml_cr(doc)
+        doc
+          .gsub(%r{(</(clause|table|figure|p|bibitem|ul|ol|dl|dt|dd|li|example|
+                       sourcecode|formula|quote|references|annex|appendix|title|
+                       name|note|thead|tbody|tfoot|th|td|form|requirement|
+                       recommendation|permission|imagemap|svgmap|preferred|
+                       admitted|related|deprecates|letter-symbol|domain|
+                       graphical-symbol|expression|abbreviation-type|subject|
+                       pronunciation|grammar|term|terms|termnote|termexample|
+                       termsource|origin|termref|modification)>)}x, "\\1\n")
+          .gsub(%r{(<(title|name))}x, "\n\\1")
+      end
+
       def version
         flavour = self.class.name.sub(/::Converter$/, "").sub(/^.+::/, "")
         Metanorma.versioned(Metanorma, flavour)[-1]::VERSION
@@ -110,7 +149,7 @@ module Metanorma
 
       def makexml(node)
         result = makexml1(node)
-        ret1 = cleanup(Nokogiri::XML(result))
+        ret1 = cleanup(Nokogiri::XML(insert_xml_cr(result)))
         ret1.root.add_namespace(nil, xml_namespace)
         validate(ret1) unless @novalid
         ret1

data/lib/metanorma/standoc/basicdoc.rng

@@ -173,9 +173,11 @@
           <data type="dateTime"/>
         </attribute>
       </optional>
-      <attribute name="from">
-        <data type="IDREF"/>
-      </attribute>
+      <optional>
+        <attribute name="from">
+          <data type="IDREF"/>
+        </attribute>
+      </optional>
       <optional>
         <attribute name="to">
           <data type="IDREF"/>

data/lib/metanorma/standoc/biblio.rng

@@ -209,9 +209,6 @@
       <zeroOrMore>
         <ref name="contact"/>
       </zeroOrMore>
-      <zeroOrMore>
-        <ref name="uri"/>
-      </zeroOrMore>
     </element>
   </define>
   <define name="fullname">
@@ -401,9 +398,9 @@
       <choice>
         <!-- iso191606 TODO -->
         <group>
-          <oneOrMore>
+          <zeroOrMore>
             <ref name="street"/>
-          </oneOrMore>
+          </zeroOrMore>
           <ref name="city"/>
           <optional>
             <ref name="state"/>
@@ -828,6 +825,11 @@
       <optional>
         <attribute name="scope"/>
       </optional>
+      <optional>
+        <attribute name="primary">
+          <data type="boolean"/>
+        </attribute>
+      </optional>
       <text/>
     </element>
   </define>

data/lib/metanorma/standoc/blocks.rb

@@ -174,11 +174,11 @@ module Metanorma
       def paragraph(node)
         return termsource(node) if node.role == "source"
 
-        noko do |xml|
+        ret = noko do |xml|
           xml.p **para_attrs(node) do |xml_t|
             xml_t << node.content
           end
-        end.join("\n")
+        end.join
       end
 
       def quote_attrs(node)
@@ -203,7 +203,7 @@ module Metanorma
             quote_attribution(node, q)
             wrap_in_para(node, q)
           end
-        end.join("\n")
+        end.join
       end
 
       def listing_attrs(node)

data/lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -56,9 +56,9 @@ module Metanorma
       TERM_CLAUSE = "//sections/terms | "\
                     "//sections/clause[descendant::terms]".freeze
 
-      NORM_REF = "//bibliography/references[@normative = 'true'] | "\
-                 "//bibliography/clause[.//references[@normative = 'true']]"
-        .freeze
+      NORM_REF =
+        "//bibliography/references[@normative = 'true'][not(@hidden)] | "\
+        "//bibliography/clause[.//references[@normative = 'true']]".freeze
 
       def boilerplate_isodoc(xmldoc)
         x = xmldoc.dup

data/lib/metanorma/standoc/cleanup_image.rb

@@ -2,6 +2,7 @@ module Metanorma
   module Standoc
     module Cleanup
       def svgmap_cleanup(xmldoc)
+        svg_uniqueids(xmldoc)
         svgmap_moveattrs(xmldoc)
         svgmap_populate(xmldoc)
         Metanorma::Utils::svgmap_rewrite(xmldoc, @localdir)
@@ -59,12 +60,125 @@ module Metanorma
       end
 
       def img_cleanup(xmldoc)
-        return xmldoc unless @datauriimage
+        if @datauriimage
+          xmldoc.xpath("//image").each do |i|
+            # do not datauri encode SVG, we need to deduplicate its IDs
+            unless read_in_if_svg(i, @localdir)
+              i["src"] = Metanorma::Utils::datauri(i["src"], @localdir)
+            end
+          end
+        end
+        svg_uniqueids(xmldoc)
+        xmldoc
+      end
+
+      def read_in_if_svg(img, localdir)
+        return false unless img["src"]
+
+        path = Metanorma::Utils::svgmap_rewrite0_path(img["src"], localdir)
+        File.file?(path) or return false
+        types = MIME::Types.type_for(path) or return false
+        types.first == "image/svg+xml" or return false
+        svg = File.read(path, encoding: "utf-8") or return false
+        img.replace(Nokogiri::XML(svg).root.to_xml)
+        true
+      end
+
+      IRI_TAG_PROPERTIES_MAP = {
+        clipPath: ["clip-path"],
+        "color-profile": nil,
+        cursor: nil,
+        filter: nil,
+        linearGradient: ["fill", "stroke"],
+        marker: ["marker", "marker-end", "marker-mid", "marker-start"],
+        mask: nil,
+        pattern: ["fill", "stroke"],
+        radialGradient: ["fill", "stroke"],
+      }.freeze
+
+      SVG_NS = "http://www.w3.org/2000/svg".freeze
+
+      def svg_uniqueids(xmldoc)
+        # only keep non-unique identifiers
+        ids = xmldoc.xpath("//m:svg//*/@id | //svg/@id", "m" => SVG_NS)
+          .map(&:text).group_by(&:itself).transform_values(&:count)
+          .delete_if { |_, v| v < 2 }
+        xmldoc.xpath("//m:svg", "m" => SVG_NS).each_with_index do |s, i|
+          ids = svg_uniqueids1(s, i, ids)
+        end
+      end
+
+      def svg_iri_properties(id_elems)
+        iri_tag_names = id_elems.each_with_object([]) do |e, m|
+          IRI_TAG_PROPERTIES_MAP.key?(e.name.to_sym) and m = m << e.name
+        end.uniq
+        iri_properties = iri_tag_names.each_with_object([]) do |t, m|
+          (IRI_TAG_PROPERTIES_MAP[t.to_sym] || [t]).each { |t1| m = m << t1 }
+        end.uniq
+        return [] if iri_properties.empty?
+
+        iri_properties << "style"
+      end
 
-        xmldoc.xpath("//image").each do |i|
-          i["src"] = Metanorma::Utils::datauri(i["src"], @localdir)
+      def svg_uniqueids1(svg, idx, ids)
+        id_elems = svg.xpath(".//*[@id] | ./@id/..")
+        iri_properties = svg_iri_properties(id_elems)
+        svg_uniqueids2(svg, iri_properties, idx, ids)
+        new_ids = id_elems.map { |x| x["id"] }
+          .map { |x| x + (ids[x] ? "_inject_#{idx}" : "") }
+        ids.merge(new_ids.each.map { |value| [value, true] }.to_h)
+      end
+
+      def svg_uniqueids2(svg, iri_properties, idx, ids)
+        svg.traverse do |e|
+          next unless e.element?
+
+          if e.name == "style"
+            svg_styleupdate(e, idx, ids)
+          elsif !e.attributes.empty?
+            svg_attrupdate(e, iri_properties, idx, ids)
+          end
+          svg_linkupdate(e, idx, ids)
+          svg_idupdate(e, idx, ids)
         end
       end
+
+      def svg_update_url(text, idx, ids)
+        text.gsub(/url\("?#([a-zA-Z][\w:.-]*)"?\)/) do |x|
+          if ids[$1] then "url(##{$1}_inject_#{idx})"
+          else x
+          end
+        end
+      end
+
+      def svg_styleupdate(elem, idx, ids)
+        elem.children = svg_update_url(elem.text, idx, ids)
+      end
+
+      def svg_attrupdate(elem, iri_properties, idx, ids)
+        iri_properties.each do |p|
+          next unless elem[p]
+
+          elem[p] = svg_update_url(elem[p], idx, ids)
+        end
+      end
+
+      def svg_linkupdate(elem, idx, ids)
+        %w(xlink:href href).each do |ref|
+          iri = elem[ref]&.strip
+          next unless /^#/.match?(iri)
+          next unless ids[iri.sub(/^#/, "")]
+
+          elem[ref] += "_inject_#{idx}"
+        end
+      end
+
+      def svg_idupdate(elem, idx, ids)
+        return unless elem["id"]
+        return unless ids[elem["id"]]
+
+        elem["id"] += "_inject_#{idx}"
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_ref.rb

@@ -16,7 +16,7 @@ module Metanorma
         insert = refs&.at("./bibitem")&.previous_element
         refs.xpath("./bibitem").each(&:remove)
         bib.reverse.each do |b|
-          insert and insert.next = b.to_xml or
+          (insert and insert.next = b.to_xml) or
             refs.children.first.add_previous_sibling b.to_xml
         end
         extract_notes_from_biblio(refs)
@@ -84,9 +84,39 @@ module Metanorma
         biblio_reorder(xmldoc)
         biblio_nested(xmldoc)
         biblio_renumber(xmldoc)
+        biblio_linkonly(xmldoc)
+        biblio_hidden_inherit(xmldoc)
         biblio_no_ext(xmldoc)
       end
 
+      def biblio_hidden_inherit(xmldoc)
+        xmldoc.xpath("//references[@hidden = 'true']").each do |r|
+          r.xpath("./bibitem").each do |b|
+            b["hidden"] = true
+          end
+        end
+      end
+
+      def biblio_linkonly(xmldoc)
+        return unless xmldoc.at("//xref[@hidden]")
+
+        ins = xmldoc.at("//bibliography")
+          .add_child("<references hidden='true' normative='true'/>").first
+        refs = xmldoc.xpath("//xref[@hidden]").each_with_object([]) do |x, m|
+          @refids << x["target"]
+          m << { id: x["target"], ref: x["hidden"] }
+          x.delete("hidden")
+        end
+        ins << insert_hidden_bibitems(refs)
+      end
+
+      def insert_hidden_bibitems(bib)
+        refs = bib.each_with_object([]) do |b, m|
+          m << reference1code(%(<ref id="#{b[:id]}">[#{b[:ref]}]</ref>), nil)
+        end
+        reference_populate(refs)
+      end
+
       def biblio_no_ext(xmldoc)
         xmldoc.xpath("//bibitem/ext").each(&:remove)
       end
@@ -114,10 +144,9 @@ module Metanorma
 
       def reference_names(xmldoc)
         xmldoc.xpath("//bibitem[not(ancestor::bibitem)]").each do |ref|
-          # isopub = ref.at(ISO_PUBLISHER_XPATH)
           docid = ref.at("./docidentifier[@type = 'metanorma']") ||
             ref.at("./docidentifier[not(@type = 'DOI')]") or next
-          reference = format_ref(docid.text, docid["type"])
+          reference = format_ref(docid.children.to_xml, docid["type"])
           @anchors[ref["id"]] = { xref: reference }
         end
       end

data/lib/metanorma/standoc/cleanup_ref_dl.rb

@@ -15,19 +15,20 @@ module Metanorma
         end
       end
 
-      def validate_ref_dl(bib, c)
+      def validate_ref_dl(bib, clause)
         id = bib["id"]
-        id ||= c["id"] unless /^_/.match?(c["id"]) # do not accept implicit id
+        # do not accept implicit id
+        id ||= clause["id"] unless /^_/.match?(clause["id"])
         unless id
-          @log.add("Anchors", c,
-                   "The following reference is missing an anchor:\n" + c.to_xml)
+          @log.add("Anchors", clause,
+                   "The following reference is missing an anchor:\n#{clause.to_xml}")
           return
         end
         @refids << id
         bib["title"] or
-          @log.add("Bibliography", c, "Reference #{id} is missing a title")
+          @log.add("Bibliography", clause, "Reference #{id} is missing a title")
         bib["docid"] or
-          @log.add("Bibliography", c,
+          @log.add("Bibliography", clause,
                    "Reference #{id} is missing a document identifier (docid)")
       end
 
@@ -76,27 +77,30 @@ module Metanorma
       end
 
       # definition list, with at most one level of unordered lists
-      def dl_bib_extract(c, nested = false)
-        dl = c.at("./dl") or return
+      def dl_bib_extract(clause, nested = false)
+        dl = clause.at("./dl") or return
         bib = {}
         key = ""
         dl.xpath("./dt | ./dd").each do |dtd|
           (dtd.name == "dt" and key = dtd.text.sub(/:+$/, "")) or
             add_to_hash(bib, key, dd_bib_extract(dtd))
         end
-        c.xpath("./clause").each do |c1|
+        clause.xpath("./clause").each do |c1|
           key = c1&.at("./title")&.text&.downcase&.strip
           next unless %w(contributor relation series).include? key
 
           add_to_hash(bib, key, dl_bib_extract(c1, true))
         end
-        if !nested && c.at("./title")
-          title = c.at("./title").remove.children.to_xml
-          bib["title"] = [bib["title"]] if bib["title"].is_a? Hash
-          bib["title"] = [bib["title"]] if bib["title"].is_a? String
-          bib["title"] = [] unless bib["title"]
-          bib["title"] << title if !title.empty?
-        end
+        dl_bib_extract_title(bib, clause, nested)
+      end
+
+      def dl_bib_extract_title(bib, clause, nested)
+        (!nested && clause.at("./title")) or return bib
+        title = clause.at("./title").remove.children.to_xml
+        bib["title"] = [bib["title"]] if bib["title"].is_a?(Hash) ||
+          bib["title"].is_a?(String)
+        bib["title"] = [] unless bib["title"]
+        bib["title"] << title if !title.empty?
         bib
       end
     end

data/lib/metanorma/standoc/cleanup_section.rb

@@ -54,7 +54,7 @@ module Metanorma
       end
 
       def make_bibliography(xml, sect)
-        if xml.at("//sections/references")
+        if xml.at("//sections/references | //xref[@hidden]")
           biblio = sect.add_next_sibling("<bibliography/>").first
           xml.xpath("//sections/references").each do |r|
             biblio.add_child r.remove
@@ -193,7 +193,7 @@ module Metanorma
             next unless t.next_element.nil?
             next if %w(sections annex preface).include? t.parent.name
 
-            t.parent.parent << t
+            t.parent.next = t
             found = true
           end
           break unless found

data/lib/metanorma/standoc/cleanup_terms_designations.rb

@@ -182,10 +182,12 @@ module Metanorma
 
       def term_designation_reorder(xmldoc)
         xmldoc.xpath("//term").each do |t|
-          %w(preferred admitted deprecates related)
+          des = %w(preferred admitted deprecates related)
             .each_with_object([]) do |tag, m|
             t.xpath("./#{tag}").each { |x| m << x.remove }
-          end.reverse.each do |x|
+          end.reverse
+          t << " "
+          des.each do |x|
             t.children.first.previous = x
           end
         end

data/lib/metanorma/standoc/cleanup_text.rb

@@ -28,28 +28,6 @@ module Metanorma
       end
 
       # "abc<tag/>", def => "abc",<tag/> def
-=begin
-      def uninterrupt_quotes_around_xml(xmldoc)
-        xmldoc.xpath("//*[following::text()[1]"\
-                     "[starts-with(., '\"') or starts-with(., \"'\")]]")
-          .each do |x|
-          next if !x.ancestors("pre, tt, sourcecode, stem, figure").empty?
-
-          uninterrupt_quotes_around_xml1(x)
-        end
-      end
-=end
-=begin
-      def uninterrupt_quotes_around_xml(xmldoc)
-        xmldoc.traverse do |n|
-          next unless n.element? && n&.next&.text? &&
-            n.ancestors("pre, tt, sourcecode, stem, figure").empty?
-          next unless /^['"]/.match?(n.next.text)
-
-          uninterrupt_quotes_around_xml1(n)
-        end
-      end
-=end
       def uninterrupt_quotes_around_xml(xmldoc)
         xmldoc.traverse do |n|
           next unless n.text? && n&.previous&.element?