metanorma 2.0.0 → 2.1.2

data/lib/metanorma/collection/renderer/utils.rb

@@ -3,7 +3,7 @@ module Metanorma
     class Renderer
       def dir_name_cleanse(name)
         path = Pathname.new(name)
-        clean_regex = /[<>:"|?*]/
+        clean_regex = /[<>:"|?*\p{Zs}]/
         fallback_sym = "_"
         return name.gsub(clean_regex, fallback_sym) unless path.absolute?
 
@@ -40,6 +40,13 @@ module Metanorma
         end
       end
 
+      def new_hidden_ref(xmldoc)
+        ins = xmldoc.at(ns("bibliography")) or
+          xmldoc.root << "<bibliography/>" and ins = xmldoc.at(ns("bibliography"))
+        ins.at(ns("./referenced[@hidden = 'true']")) or
+          ins.add_child("<references hidden='true' normative='false'/>").first
+      end
+
       def strip_eref(eref)
         eref.xpath(ns("./locality | ./localityStack")).each(&:remove)
         eref.replace(eref.children)
@@ -48,11 +55,12 @@ module Metanorma
       def docid_to_citeas(bib)
         docid = bib.at(ns("./docidentifier[@primary = 'true']")) ||
           bib.at(ns("./docidentifier")) or return
-        docid_prefix(docid)
+        ::Metanorma::Collection::Util::key(docid_prefix(docid))
       end
 
-      def collect_erefs(docxml)
-        docxml.xpath(ns("//eref"))
+      def collect_erefs(docxml, presxml)
+        tag = presxml ? "fmt-eref" : "eref"
+        docxml.xpath(ns("//#{tag}"))
           .each_with_object({ citeas: {}, bibitemid: {} }) do |i, m|
           m[:citeas][i["citeas"]] = true
           m[:bibitemid][i["bibitemid"]] = true
@@ -104,9 +112,9 @@ module Metanorma
       end
 
       def pdfconv
-        doctype = @doctype.to_sym
-        x = Asciidoctor.load nil, backend: doctype
-        x.converter.pdf_converter(PdfOptionsNode.new(doctype,
+        flavor = @flavor.to_sym
+        x = Asciidoctor.load nil, backend: flavor
+        x.converter.pdf_converter(PdfOptionsNode.new(flavor,
                                                      @compile_options))
       end
 
@@ -119,14 +127,35 @@ module Metanorma
 
       def datauri_encode(docxml, directory)
         docxml.xpath(ns("//image")).each do |i|
+          read_in_if_svg(i, directory.sub(%r{(?<!=/)$}, "/")) and i["src"] = nil
+        end
+        docxml.xpath(ns("//image")).each do |i| # rubocop:disable Style/CombinableLoops
+          i["src"] && !i["src"].empty? or next
           i["src"] = Vectory::Utils::datauri(i["src"], directory)
         end
         docxml
       end
 
+      def read_in_if_svg(img, localdir)
+        img["src"] or return false
+        img.elements.map(&:name).include?("svg") and return true
+        path = Vectory::Utils.svgmap_rewrite0_path(img["src"], localdir)
+        svg = svg_in_path?(path) or return false
+        img.children = (Nokogiri::XML(svg).root)
+        true
+      end
+
+      def svg_in_path?(path)
+        File.file?(path) or return false
+        types = MIME::Types.type_for(path) or return false
+        types.first == "image/svg+xml" or return false
+        svg = File.read(path, encoding: "utf-8") or return false
+        svg
+      end
+
       class PdfOptionsNode
-        def initialize(doctype, options)
-          p = Metanorma::Registry.instance.find_processor(doctype)
+        def initialize(flavor, options)
+          p = Metanorma::Registry.instance.find_processor(flavor)
           if ::Metanorma::Util::FontistHelper.has_custom_fonts?(p, options, {})
             @fonts_manifest =
               ::Metanorma::Util::FontistHelper.location_manifest(p, options)
@@ -140,21 +169,13 @@ module Metanorma
         end
       end
 
-      def isodoc_create
-        isodoc = Util::load_isodoc(@doctype)
-        isodoc.i18n_init(@lang, @script, @locale) # read in internationalisation
-        isodoc.metadata_init(@lang, @script, @locale, isodoc.i18n)
-        isodoc.info(@xml, nil)
-        isodoc
-      end
-
       # create the @meta class of isodoc, for populating Liquid,
       # with "navigation" set to the index bar.
       # extracted from the manifest
       def isodoc_populate
         @isodoc.info(@xml, nil)
         { navigation: indexfile(@manifest), nav_object: index_object(@manifest),
-          docrefs: liquid_docrefs(@manifest),
+          bibdata: @bibdata.to_hash, docrefs: liquid_docrefs(@manifest),
           "prefatory-content": isodoc_builder(@xml.at("//prefatory-content")),
           "final-content": isodoc_builder(@xml.at("//final-content")),
           doctitle: @bibdata.title.first.title.content,
@@ -175,6 +196,22 @@ module Metanorma
         end.doc.root.to_html
       end
 
+      def eref2link(docxml, presxml)
+        isodoc = IsoDoc::PresentationXMLConvert.new({})
+        isodoc.bibitem_lookup(docxml)
+        isodoc.eref2link(docxml)
+      end
+
+      def error_anchor(erefs, docid)
+        erefs.each do |e|
+          msg = "<strong>** Unresolved reference to document #{docid} " \
+            "from eref</strong>"
+          e << msg
+          strip_eref(e)
+          @log&.add("Cross-References", e, msg)
+        end
+      end
+
       def ns(xpath)
         @isodoc.ns(xpath)
       end

data/lib/metanorma/collection/sectionsplit/collection.rb

@@ -0,0 +1,100 @@
+module Metanorma
+  class Collection
+    class Sectionsplit
+      def build_collection
+        collection_setup(@base, @dir)
+        files = sectionsplit
+        input_xml = Nokogiri::XML(File.read(@input_filename,
+                                            encoding: "UTF-8"), &:huge)
+        collection_manifest(@base, files, input_xml, @xml, @dir).render(
+          { format: %i(html), output_folder: "#{@output_filename}_collection",
+            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
+        )
+        section_split_attachments(out: "#{@output_filename}_collection")
+      end
+
+      def collection_setup(filename, dir)
+        FileUtils.mkdir_p "#{filename}_collection" if filename
+        FileUtils.mkdir_p dir
+        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
+          f.write(coll_cover)
+        end
+      end
+
+      def coll_cover
+        <<~COVER
+          <html><head><meta charset="UTF-8"/></head><body>
+                <h1>{{ doctitle }}</h1>
+                <h2>{{ docnumber }}</h2>
+                <nav>{{ navigation }}</nav>
+              </body></html>
+        COVER
+      end
+
+      def collection_manifest(filename, files, origxml, _presxml, dir)
+        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
+          f.write(collectionyaml(files, origxml))
+        end
+        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
+      end
+
+      def collectionyaml(files, xml)
+        ret = {
+          directives: ["presentation-xml", "bare-after-first"],
+          bibdata: {
+            title: {
+              type: "title-main", language: @lang,
+              content: xml.at(ns("//bibdata/title")).text
+            },
+            type: "collection",
+            docid: {
+              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
+              id: xml.at(ns("//bibdata/docidentifier")).text,
+            },
+          },
+          manifest: {
+            level: "collection", title: "Collection",
+            docref: files.sort_by { |f| f[:order] }.each.map do |f|
+              { fileref: f[:url], identifier: f[:title] }
+            end
+          },
+        }
+        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
+      end
+
+      def att_dir(file)
+        "_#{File.basename(file, '.*')}_attachments"
+      end
+
+      def section_split_attachments(out: nil)
+        attachments = att_dir(@tmp_filename)
+        File.directory?(attachments) or return
+        dir = out || File.dirname(@input_filename)
+        ret = File.join(dir, att_dir(@output_filename))
+        FileUtils.rm_rf ret
+        FileUtils.mv attachments, ret
+        File.basename(ret)
+      end
+
+      def section_split_cover(col, ident, one_doc_coll)
+        dir = File.dirname(col.file)
+        collection_setup(nil, dir)
+        r = ::Metanorma::Collection::Renderer
+          .new(col, dir, output_folder: "#{ident}_collection",
+                         format: %i(html),
+                         coverpage: File.join(dir, "cover.html"))
+        r.coverpage
+        section_split_cover1(ident, r, dir, one_doc_coll)
+      end
+
+      def section_split_cover1(ident, renderer, dir, _one_doc_coll)
+        filename = File.basename("#{ident}_index.html")
+        # ident can be a directory with YAML indirection
+        dest = File.join(dir, filename)
+        FileUtils.mv File.join(renderer.outdir, "index.html"), dest
+        FileUtils.rm_rf renderer.outdir
+        filename
+      end
+    end
+  end
+end

data/lib/metanorma/collection/sectionsplit/sectionsplit.rb

@@ -1,6 +1,8 @@
 require "yaml"
 require_relative "../../util/util"
 require_relative "../xrefprocess/xrefprocess"
+require_relative "collection"
+require "concurrent-ruby"
 
 module Metanorma
   class Collection
@@ -17,48 +19,13 @@ module Metanorma
         @fileslookup = opts[:fileslookup]
         @ident = opts[:ident]
         @isodoc = opts[:isodoc]
+        @document_suffix = opts[:document_suffix]
       end
 
       def ns(xpath)
         @isodoc.ns(xpath)
       end
 
-      def build_collection
-        collection_setup(@base, @dir)
-        files = sectionsplit # (@input_filename, @base, @dir, @compile_opts)
-        input_xml = Nokogiri::XML(File.read(@input_filename,
-                                            encoding: "UTF-8"), &:huge)
-        collection_manifest(@base, files, input_xml, @xml, @dir).render(
-          { format: %i(html), output_folder: "#{@output_filename}_collection",
-            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
-        )
-      end
-
-      def collection_manifest(filename, files, origxml, _presxml, dir)
-        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
-          f.write(collectionyaml(files, origxml))
-        end
-        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
-      end
-
-      def collection_setup(filename, dir)
-        FileUtils.mkdir_p "#{filename}_collection" if filename
-        FileUtils.mkdir_p dir
-        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
-          f.write(coll_cover)
-        end
-      end
-
-      def coll_cover
-        <<~COVER
-          <html><head><meta charset="UTF-8"/></head><body>
-                <h1>{{ doctitle }}</h1>
-                <h2>{{ docnumber }}</h2>
-                <nav>{{ navigation }}</nav>
-              </body></html>
-        COVER
-      end
-
       SPLITSECTIONS =
         [["//preface/*", "preface"], ["//sections/*", "sections"],
          ["//annex", nil],
@@ -66,21 +33,41 @@ module Metanorma
          ["//indexsect", nil], ["//colophon", nil]].freeze
 
       # Input XML is Semantic
-      # def sectionsplit(filename, basename, dir, compile_options, fileslookup = nil, ident = nil)
       def sectionsplit
         xml = sectionsplit_prep(File.read(@input_filename), @base, @dir)
         @key = Metanorma::Collection::XrefProcess::xref_preprocess(xml, @isodoc)
-        SPLITSECTIONS.each_with_object([]) do |n, ret|
+        empty = empty_doc(xml)
+        empty1 = empty_attachments(empty)
+        @mutex = Mutex.new
+        #@pool = Concurrent::FixedThreadPool.new(4)
+        @pool = Concurrent::FixedThreadPool.new(1)
+        sectionsplit1(xml, empty, empty1, 0)
+      end
+
+      def sectionsplit1(xml, empty, empty1, idx)
+        ret = SPLITSECTIONS.each_with_object([]) do |n, m|
           conflate_floatingtitles(xml.xpath(ns(n[0]))).each do |s|
-            ret << sectionfile(xml, emptydoc(xml), "#{@base}.#{ret.size}", s,
-                               n[1])
+            sectionsplit2(xml, idx.zero? ? empty : empty1, s, n[1],
+                          { acc: m, idx: idx })
+            idx += 1
           end
         end
+        @pool.shutdown
+        @pool.wait_for_termination
+        ret
+      end
+
+      def sectionsplit2(xml, empty, chunks, parentnode, opt)
+        @pool.post do
+          a = sectionfile(xml, empty, "#{@base}.#{opt[:idx]}", chunks,
+                          parentnode)
+          @mutex.synchronize { opt[:acc] << a }
+        end
       end
 
       def block?(node)
         %w(p table formula admonition ol ul dl figure quote sourcecode example
-           pre note pagebrreak hr bookmark requirement recommendation permission
+           pre note pagebreak hr bookmark requirement recommendation permission
            svgmap inputform toc passthrough review imagemap).include?(node.name)
       end
 
@@ -96,53 +83,79 @@ module Metanorma
 
       def sectionsplit_prep(file, filename, dir)
         @splitdir = dir
-        xml1filename, type = sectionsplit_preprocess_semxml(file, filename)
-        Compile.new.compile(
-          xml1filename,
-          { format: :asciidoc, extension_keys: [:presentation], type: type }
-         .merge(@compile_opts),
-        )
-        Nokogiri::XML(File.read(xml1filename.sub(/\.xml$/, ".presentation.xml"),
-                                encoding: "utf-8"), &:huge)
+        xml, type = sectionsplit_preprocess_semxml(file, filename)
+        flags = { format: :asciidoc, extension_keys: [:presentation],
+                  type: type }.merge(@compile_opts)
+        #require "debug"; binding.b
+        Compile.new.compile(xml, flags)
+        f = File.open(xml.sub(/\.xml$/, ".presentation.xml"), encoding: "utf-8")
+        r = Nokogiri::XML(f, &:huge)
+        f.close
+        r.xpath("//xmlns:svgmap1").each { |x| x.name = "svgmap" }
+        r
       end
 
       def sectionsplit_preprocess_semxml(file, filename)
         xml = Nokogiri::XML(file, &:huge)
-        type = xml.root.name.sub("-standard", "").to_sym
+        type = xml.root["flavor"]
+        type ||= xml.root.name.sub("-standard", "").to_sym
         sectionsplit_update_xrefs(xml)
         xml1 = sectionsplit_write_semxml(filename, xml)
-        @filecache ||= []
-        @filecache << xml1
-        [xml1.path, type]
+        @tmp_filename = xml1
+        [xml1, type]
       end
 
       def sectionsplit_update_xrefs(xml)
         if c = @fileslookup&.parent
           n = c.nested
           c.nested = true # so unresolved erefs are not deleted
+        #require "debug"; binding.b
           c.update_xrefs(xml, @ident, {})
           c.nested = n
+          xml.xpath("//xmlns:svgmap").each { |x| x.name = "svgmap1" }
+          # do not process svgmap until after files are split
         end
       end
 
       def sectionsplit_write_semxml(filename, xml)
-        Tempfile.open([filename, ".xml"], encoding: "utf-8") do |f|
-          # f.write(@isodoc.to_xml(svg_preprocess(xml)))
+        outname = Pathname.new("tmp_#{filename}").sub_ext(".xml").to_s
+        File.open(outname, "w:UTF-8") do |f|
           f.write(@isodoc.to_xml(xml))
-          f
         end
+        outname
+      end
+
+      def emptydoc(xml, ordinal)
+        out = xml.dup
+        out.xpath(
+          ns("//preface | //sections | //annex | //bibliography/clause | " \
+             "//bibliography/references[not(@hidden = 'true')] | " \
+             "//indexsect | //colophon"),
+        ).each(&:remove)
+        ordinal.zero? or out.xpath(ns("//metanorma-ext//attachment | " \
+                                      "//semantic__metanorma-ext//semantic__attachment"))
+          .each(&:remove) # keep only one copy of attachments
+        out
       end
 
-      def emptydoc(xml)
+      def empty_doc(xml)
         out = xml.dup
         out.xpath(
           ns("//preface | //sections | //annex | //bibliography/clause | " \
-             "//bibliography/references[not(@hidden = 'true')] | //indexsect | " \
-             "//colophon"),
+             "//bibliography/references[not(@hidden = 'true')] | " \
+             "//indexsect | //colophon"),
         ).each(&:remove)
         out
       end
 
+      def empty_attachments(xml)
+        out = xml.dup
+        out.xpath(ns("//metanorma-ext//attachment | " \
+                     "//semantic__metanorma-ext//semantic__attachment"))
+          .each(&:remove) # keep only one copy of attachments
+        out
+      end
+
       def sectionfile(fulldoc, xml, file, chunks, parentnode)
         fname = create_sectionfile(fulldoc, xml.dup, file, chunks, parentnode)
         { order: chunks.last["displayorder"].to_i, url: fname,
@@ -151,8 +164,11 @@ module Metanorma
 
       def create_sectionfile(xml, out, file, chunks, parentnode)
         ins = out.at(ns("//metanorma-extension")) || out.at(ns("//bibdata"))
+        #require "debug"; binding.b
         sectionfile_insert(ins, chunks, parentnode)
-        Metanorma::Collection::XrefProcess::xref_process(out, xml, @key, @ident, @isodoc)
+        Metanorma::Collection::XrefProcess::xref_process(out, xml, @key,
+                                                         @ident, @isodoc, true)
+        #truncate_semxml(out, chunks)
         outname = "#{file}.xml"
         File.open(File.join(@splitdir, outname), "w:UTF-8") do |f|
           f.write(out)
@@ -160,6 +176,31 @@ module Metanorma
         outname
       end
 
+      # KILL
+      def semantic_xml_ids_gather(out)
+        out.at(ns("//semantic__bibdata")) or return
+        SPLITSECTIONS.each_with_object({}) do |s, m|
+          out.xpath(ns(s[0].sub("//", "//semantic__"))).each do |x|
+            x["id"] or next
+            m[x["id"].sub(/^semantic__/, "")] = x
+          end
+        end
+      end
+
+      # KILL
+      def semxml_presxml_nodes_match(nodes, chunks)
+        chunks.each do |x|
+          nodes[x["id"]] and nodes.delete(x["id"])
+        end
+      end
+
+      # KILL
+      def truncate_semxml(out, chunks)
+        nodes = semantic_xml_ids_gather(out) or return
+        semxml_presxml_nodes_match(nodes, chunks)
+        nodes.each_value(&:remove)
+      end
+
       def sectionfile_insert(ins, chunks, parentnode)
         if parentnode
           ins.next = "<#{parentnode}/>"
@@ -169,49 +210,11 @@ module Metanorma
       end
 
       def titlerender(section)
-        title = section.at(ns("./title")) or return "[Untitled]"
+        title = section.at(ns("./fmt-title")) or return "[Untitled]"
         t = title.dup
         t.xpath(ns(".//tab | .//br")).each { |x| x.replace(" ") }
-        t.xpath(ns(".//strong")).each { |x| x.replace(x.children) }
-        t.children.to_xml
-      end
-
-      def collectionyaml(files, xml)
-        ret = {
-          directives: ["presentation-xml", "bare-after-first"],
-          bibdata: {
-            title: {
-              type: "title-main", language: @lang,
-              content: xml.at(ns("//bibdata/title")).text
-            },
-            type: "collection",
-            docid: {
-              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
-              id: xml.at(ns("//bibdata/docidentifier")).text,
-            },
-          },
-          manifest: {
-            level: "collection", title: "Collection",
-            docref: files.sort_by { |f| f[:order] }.each.map do |f|
-              { fileref: f[:url], identifier: f[:title] }
-            end
-          },
-        }
-        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
-      end
-
-      def section_split_cover(col, ident, _one_doc_coll)
-        dir = File.dirname(col.file)
-        collection_setup(nil, dir)
-        r = ::Metanorma::Collection::Renderer
-          .new(col, dir, output_folder: "#{ident}_collection",
-                         format: %i(html), coverpage: File.join(dir, "cover.html"))
-        r.coverpage
-        # filename = one_doc_coll ? "#{ident}_index.html" : "index.html"
-        filename = File.basename("#{ident}_index.html") # ident can be a directory with YAML indirection
-        FileUtils.mv File.join(r.outdir, "index.html"), File.join(dir, filename)
-        FileUtils.rm_rf r.outdir
-        filename
+        t.xpath(ns(".//bookmark")).each(&:remove)
+        t.xpath('.//text()').map(&:text).join
       end
     end
   end

data/lib/metanorma/collection/util/util.rb

@@ -14,24 +14,26 @@ module Metanorma
           end
         end
 
-        def gather_bibitemids(xml)
+        def gather_bibitemids(xml, presxml)
           xml.xpath("//*[@bibitemid]").each_with_object({}) do |e, m|
-            /^semantic__/.match?(e.name) and next
+            #/^semantic__/.match?(e.name) and next
+            presxml && %w(xref eref link).include?(e.name) and next
             m[e["bibitemid"]] ||= []
             m[e["bibitemid"]] << e
           end
         end
 
-        def gather_citeases(xml)
+        def gather_citeases(xml, presxml)
           xml.xpath("//*[@citeas]").each_with_object({}) do |e, m|
-            /^semantic__/.match?(e.name) and next
-            m[e["citeas"]] ||= []
-            m[e["citeas"]] << e
+            #/^semantic__/.match?(e.name) and next
+            presxml && %w(xref eref link).include?(e.name) and next
+            k = key(e["citeas"])
+            m[k] ||= []
+            m[k] << e
           end
         end
 
-        def add_suffix_to_attributes(doc, suffix, tag_name, attr_name,
-isodoc)
+        def add_suffix_to_attrs(doc, suffix, tag_name, attr_name, isodoc)
           (suffix.nil? || suffix.empty?) and return
           doc.xpath(isodoc.ns("//#{tag_name}[@#{attr_name}]")).each do |elem|
             a = elem.attributes[attr_name].value
@@ -62,10 +64,23 @@ isodoc)
           def attr(_key); end
         end
 
-        def load_isodoc(doctype)
-          x = Asciidoctor.load nil, backend: doctype.to_sym
+        def load_isodoc(flavor, presxml: false)
+          x = Asciidoctor.load nil, backend: flavor.to_sym
+          if presxml
+          x.converter.presentation_xml_converter(Dummy.new)
+          else
           x.converter.html_converter(Dummy.new) # to obtain Isodoc class
+          end
         end
+
+        def isodoc_create(flavor, lang, script, xml, presxml: false)
+        isodoc = Util::load_isodoc(flavor, presxml: presxml)
+        isodoc.i18n_init(lang, script, nil) # read in internationalisation
+        # TODO locale?
+        isodoc.metadata_init(lang, script, nil, isodoc.i18n)
+        isodoc.info(xml, nil)
+        isodoc
+      end
       end
     end
   end