metanorma 2.0.5 → 2.0.7

data/lib/metanorma/collection/renderer/fileparse.rb

@@ -19,6 +19,7 @@ module Metanorma
         @nested || sso or
           Metanorma::Collection::XrefProcess::xref_process(xml, xml, nil, docid,
                                                            @isodoc)
+        @ncnames = {}
         @nested or update_indirect_refs_to_docs(xml, docid, internal_refs)
         @files.add_document_suffix(docid, xml)
         @nested or update_sectionsplit_refs_to_docs(xml, internal_refs)
@@ -49,9 +50,11 @@ module Metanorma
         end
       end
 
+      ANCHOR_XPATH = "xmlns:locality[@type = 'anchor']/xmlns:referenceFrom"
+        .freeze
+
       def update_sectionsplit_eref_to_doc(eref, internal_refs, doclist, opts)
-        a = eref.at(ns("./localityStack/locality[@type = 'anchor']/" \
-                       "referenceFrom")) or return
+        a = eref.at("./xmlns:localityStack/#{ANCHOR_XPATH}") or return
         doc = internal_refs[opts[:key]]["#{a.text}_#{opts[:target_suffix]}"]
         bibitemid = Metanorma::Utils::to_ncname("#{doc}_#{opts[:source_suffix]}")
         eref["bibitemid"] = bibitemid
@@ -59,12 +62,6 @@ module Metanorma
         doclist
       end
 
-      def eref2link(docxml)
-        isodoc = IsoDoc::PresentationXMLConvert.new({})
-        isodoc.bibitem_lookup(docxml)
-        isodoc.eref2link(docxml)
-      end
-
       BIBITEM_NOT_REPO_XPATH = "//bibitem[not(ancestor::bibitem)]" \
         "[not(./docidentifier[@type = 'repository'])]".freeze
 
@@ -79,64 +76,37 @@ module Metanorma
         end
       end
 
-      def svg_datauri(docxml, docid)
-        rel = @files.get(docid, :rel_path)
-        parent = @files.get(docid, :parentid) and
-          rel = @files.get(parent, :rel_path)
-        # if sectionsplit, use orig file dir
-        dir = File.join(@dirname, File.dirname(rel))
-        datauri_encode(docxml, dir)
-      end
-
-      def svgmap_resolve(docxml, docid)
-        ids = @files.get(docid, :ids)
-        docxml = svg_unnest(svg_datauri(docxml, docid))
-        isodoc = IsoDoc::PresentationXMLConvert.new({})
-        isodoc.bibitem_lookup(docxml)
-        docxml.xpath(ns("//svgmap//eref")).each do |e|
-          svgmap_resolve_eref(e, isodoc, docxml, ids)
-        end
-        Vectory::SvgMapping.new(docxml, "").call
-        docxml.xpath(ns("//svgmap")).each { |s| isodoc.svgmap_extract(s) }
-      end
-
-      def svg_unnest(docxml)
-        docxml.xpath(ns("//svgmap//image[.//*[name() = 'image']]")).each do |i|
-          s = i.elements.detect { |e| e.name == "svg" } and
-            i.replace(s)
-        end
-        docxml
-      end
-
-      def svgmap_resolve_eref(eref, isodoc, _docxml, ids)
-        href = isodoc.eref_target(eref) or return
-        href = href[:link]
-        href == "##{eref['bibitemid']}" ||
-          (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
-        eref["target"] = href.strip
-        eref.name = "link"
-        eref.elements&.remove
-      end
-
       # repo(current-metanorma-collection/ISO 17301-1:2016)
       # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
       # Any erefs to that bibitem id are replaced with relative URL
       # Preferably with anchor, and is a job to realise dynamic lookup
       # of localities.
       def update_direct_refs_to_docs(docxml, identifier)
-        erefs, erefs1 = update_direct_refs_to_docs_prep(docxml)
+        erefs, erefs_no_anchor, anchors, erefs1 =
+          update_direct_refs_to_docs_prep(docxml)
         docxml.xpath(ns("//bibitem")).each do |b|
           docid = b.at(ns("./docidentifier[@type = 'repository']")) or next
           strip_unresolved_repo_erefs(identifier, docid, erefs1, b) or next
           update_bibitem(b, identifier)
           docid = docid_to_citeas(b) or next
-          erefs[docid] and update_anchors(b, docid, erefs[docid])
+          erefs[docid] and
+            update_anchors(b, docid, erefs[docid], erefs_no_anchor[docid],
+                           anchors[docid])
         end
       end
 
+      # Hash(docid) of arrays
       def update_direct_refs_to_docs_prep(docxml)
-        @ncnames = {}
-        [Util::gather_citeases(docxml), Util::gather_bibitemids(docxml)]
+        erefs = Util::gather_citeases(docxml)
+        no_anchor = erefs.keys.each_with_object({}) { |k, m| m[k] = [] }
+        anchors = erefs.keys.each_with_object({}) { |k, m| m[k] = [] }
+        erefs.each do |k, v|
+          v.each do |e|
+            if loc = e.at(".//#{ANCHOR_XPATH}") then anchors[k] << loc
+            else no_anchor[k] << e end
+          end
+        end
+        [erefs, no_anchor, anchors, Util::gather_bibitemids(docxml)]
       end
 
       # strip erefs if they are repository erefs, but do not point to a document
@@ -155,46 +125,78 @@ module Metanorma
       # Resolve erefs to a container of ids in another doc,
       # to an anchor eref (direct link)
       def update_indirect_refs_to_docs(docxml, _docidentifier, internal_refs)
-        bibitems, erefs = update_indirect_refs_to_docs_prep(docxml)
+        bib, erefs, doc_suffix, doc_type, f = update_indirect_refs_prep(docxml)
         internal_refs.each do |schema, ids|
+          add_suffix = doc_suffix && doc_type && doc_type != schema
           ids.each do |id, file|
-            k = indirect_ref_key(schema, id, docxml)
-            update_indirect_refs_to_docs1(docxml, k, file, bibitems, erefs)
+            f[file] ||= { url: @files.url?(file),
+                          parentid: @files.get(file) && @files.get(file,
+                                                                   :parentid) }
+            k = indirect_ref_key(schema, id, doc_suffix, add_suffix)
+            update_indirect_refs_to_docs1(f[file], k, file, bib, erefs)
           end
         end
       end
 
-      def update_indirect_refs_to_docs_prep(docxml)
+      def update_indirect_refs_prep(docxml)
         @updated_anchors = {}
-        [Util::gather_bibitems(docxml), Util::gather_bibitemids(docxml)]
+        @indirect_keys = {}
+        [Util::gather_bibitems(docxml), Util::gather_bibitemids(docxml),
+         docxml.root["document_suffix"], docxml.root["type"], {}]
       end
 
-      def indirect_ref_key(schema, id, docxml)
+      def indirect_ref_key(schema, id, doc_suffix, doc_type)
         /^#{schema}_/.match?(id) and return id
+        key = [schema, id, doc_suffix, doc_type].join("::")
+        x = @indirect_keys[key] and return x
         ret = "#{schema}_#{id}"
-        suffix = docxml.root["document_suffix"]
-        (k = docxml.root["type"]) && k != schema && suffix and
-          ret = "#{ret}_#{suffix}"
+        doc_suffix && doc_type && doc_type != schema and
+          ret = "#{ret}_#{doc_suffix}"
+        @indirect_keys[key] = ret
         ret
       end
 
-      def update_indirect_refs_to_docs1(_docxml, key, file, bibitems, erefs)
+      def indirect_ref_key(schema, id, doc_suffix, add_suffix)
+        /^#{schema}_/.match?(id) and return id
+        #key = "#{schema}_#{id}"
+        x = @indirect_keys.dig(schema, id) and return x
+        @indirect_keys[schema] ||= {}
+        @indirect_keys[schema][id] = if add_suffix
+                                  "#{schema}_#{id}_#{doc_suffix}"
+                                else
+                                   "#{schema}_#{id}"
+                                end
+      end
+
+      def indirect_ref_keyx(schema, id, doc_suffix, doc_type)
+        /^#{schema}_/.match?(id) and return id
+        ret = "#{schema}_#{id}"
+        doc_suffix && doc_type && doc_type != schema and
+          ret = "#{ret}_#{doc_suffix}"
+        ret
+      end
+
+      def update_indirect_refs_to_docs1(filec, key, file, bibitems, erefs)
         erefs[key]&.each do |e|
           e["citeas"] = file
-          update_indirect_refs_to_docs_anchor(e, file)
+          update_indirect_refs_to_docs_anchor(e, file, filec[:url],
+                                              filec[:parentid])
         end
         update_indirect_refs_to_docs_docid(bibitems[key], file)
       end
 
-      def update_indirect_refs_to_docs_anchor(eref, file)
-        a = eref.at(ns(".//locality[@type = 'anchor']/referenceFrom")) or return
-        suffix = file
-        @files.get(file) && p = @files.get(file, :parentid) and
-          suffix = "#{p}_#{suffix}"
+      def update_indirect_refs_to_docs_anchor(eref, file, url, parentid)
+        a = eref.at(".//#{ANCHOR_XPATH}") or return
+        parentid and file = "#{parentid}_#{file}"
         existing = a.text
-        anchor = existing
-        @files.url?(file) or
-          anchor = Metanorma::Utils::to_ncname("#{anchor}_#{suffix}")
+        anchor = if url then existing
+                   else 
+                     #suffix_anchor_indirect(existing, suffix)
+                     #k = "#{existing}_#{file}"
+        #@ncnames[k] ||= Metanorma::Utils::to_ncname(k)
+        @indirect_keys[existing] ||= {}
+        @indirect_keys[existing][file] ||= Metanorma::Utils::to_ncname("#{existing}_#{file}")
+                   end
         @updated_anchors[existing] or a.children = anchor
         @updated_anchors[anchor] = true
       end
@@ -206,37 +208,36 @@ module Metanorma
           "<docidentifier type='metanorma-collection'>#{file}</docidentifier>"
       end
 
-      # update crossrefences to other documents, to include
-      # disambiguating document suffix on id
-      def update_anchors(bib, docid, erefs)
-        erefs.each do |e|
-          if @files.get(docid) then update_anchor_loc(bib, e, docid)
-          else
-            msg = "<strong>** Unresolved reference to document #{docid} " \
-                  "from eref</strong>"
-            e << msg
-            strip_eref(e)
-            @log&.add("Cross-References", e, msg)
-          end
+      # bottleneck
+      def update_anchors(bib, docid, erefs, erefs_no_anchor, erefs_anchors)
+        @files.get(docid) or error_anchor(erefs, docid)
+        has_anchors, url, ncn_docid = update_anchors_prep(docid)
+        erefs_no_anchor.each do |e|
+          update_anchor_create_loc(bib, e, docid)
+        end
+        !url && has_anchors or return
+        erefs_anchors.each do |e|
+          update_anchors1(docid, ncn_docid, e)
+        end
+      end
+
+      def update_anchors1(docid, ncn_docid, anchor)
+        @concat_anchors[anchor.text] ||= "#{ncn_docid}_#{anchor.text}"
+        if @files.get(docid).dig(:anchors_lookup, @concat_anchors[anchor.text])
+          anchor.content = @concat_anchors[anchor.text]
         end
       end
 
-      def update_anchor_loc(bib, eref, docid)
-        loc = eref.at(".//xmlns:locality[@type = 'anchor']") or
-          return update_anchor_create_loc(bib, eref, docid)
-        ref = loc.at("./xmlns:referenceFrom") or return
-        anchor = suffix_anchor(ref, docid)
-        a = @files.get(docid, :anchors) or return
-        a.inject([]) { |m, (_, x)| m + x.values }
-          .include?(anchor) or return
-        ref.content = anchor
+      def update_anchors_prep(docid)
+        @concat_anchors = {}
+        [@files.get(docid)&.key?(:anchors_lookup), @files.url?(docid),
+         Metanorma::Utils::to_ncname(docid)]
       end
 
-      def suffix_anchor(ref, docid)
-        @ncnames[docid] ||= Metanorma::Utils::to_ncname(docid)
-        anchor = ref.text
-        @files.url?(docid) or anchor = "#{@ncnames[docid]}_#{anchor}"
-        anchor
+      # encode both prefix and suffix to NCName
+      def suffix_anchor_indirect(prefix, suffix)
+        k = "#{prefix}_#{suffix}"
+        @ncnames[k] ||= Metanorma::Utils::to_ncname(k)
       end
 
       # if there is a crossref to another document, with no anchor, retrieve the
@@ -248,7 +249,7 @@ module Metanorma
         ref = ins.at(ns("./locality/referenceFrom"))&.text
         a = @files.get(docid, :anchors).dig(type, ref) or return
         ins << "<locality type='anchor'><referenceFrom>#{a.sub(/^_/, '')}" \
-               "</referenceFrom></locality>"
+          "</referenceFrom></locality>"
       end
     end
   end

data/lib/metanorma/collection/renderer/fileprocess.rb

@@ -85,6 +85,7 @@ module Metanorma
       def gather_internal_refs
         @files.keys.each_with_object({}) do |i, refs|
           @files.get(i, :attachment) and next
+          @files.get(i, :sectionsplit) and next
           file, = @files.targetfile_id(i, read: true)
           gather_internal_refs1(file, i, refs)
         end

data/lib/metanorma/collection/renderer/render_word.rb

@@ -3,8 +3,8 @@ module Metanorma
     class Renderer
       def docconv
         @tempfile_cache ||= []
-        doctype = @doctype.to_sym
-        x = Asciidoctor.load nil, backend: doctype
+        flavor = @flavor.to_sym
+        x = Asciidoctor.load nil, backend: flavor
         x.converter.doc_converter(DocOptionsNode.new(@directives, @dirname))
       end
 
@@ -51,8 +51,8 @@ module Metanorma
         doc
       end
 
-      SECTION_BREAK = '<p class="MsoNormal"><br clear="all" class="section"/></p>'
-        .freeze
+      SECTION_BREAK =
+        '<p class="MsoNormal"><br clear="all" class="section"/></p>'.freeze
       DIV1 = '<div class="WordSection1">&#xa0;</div>'.freeze
       DIV2 = '<div class="WordSection2">&#xa0;</div>'.freeze
 
@@ -70,10 +70,10 @@ module Metanorma
       end
 
       def collection_coverpages(conv, docs)
-        conv.wordintropage and [DIV2, SECTION_BREAK].reverse.each do |s|
+        conv.wordintropage and [DIV2, SECTION_BREAK].reverse_each do |s|
           docs.unshift(Nokogiri::XML(s).root)
         end
-        conv.wordcoverpage and [DIV1, SECTION_BREAK].reverse.each do |s|
+        conv.wordcoverpage and [DIV1, SECTION_BREAK].reverse_each do |s|
           docs.unshift(Nokogiri::XML(s).root)
         end
         docs

data/lib/metanorma/collection/renderer/renderer.rb

@@ -8,6 +8,7 @@ require_relative "../multilingual/multilingual"
 require_relative "utils"
 require_relative "render_word"
 require_relative "navigation"
+require_relative "svg"
 
 module Metanorma
   class Collection
@@ -38,9 +39,9 @@ module Metanorma
         @script = collection.bibdata.script.first || "Latn"
         @locale = @xml.at("//xmlns:bibdata/xmlns:locale")&.text
         @registry = Metanorma::Registry.instance
-        @doctype = doctype
+        @flavor = options[:flavor] || flavor
         @compile = Compile.new
-        @compile.load_flavor(@doctype)
+        @compile.load_flavor(@flavor)
 
         @isodoc = isodoc_create # output processor for flavour
         @outdir = dir_name_cleanse(options[:output_folder])
@@ -74,6 +75,7 @@ module Metanorma
 
       def flush_files
         warn "\n\n\n\n\nDone: #{DateTime.now.strftime('%H:%M:%S')}"
+        warn "\nFiles to delete:\n"
         warn @files.files_to_delete
         @files.files_to_delete.each { |f| FileUtils.rm_f(f) }
         @files_to_delete.each { |f| FileUtils.rm_f(f) }
@@ -136,8 +138,8 @@ module Metanorma
         @directives.detect { |d| d.key == "bilingual" } &&
           options[:format].include?(:html) and
           Metanorma::Collection::Multilingual.new(
-            { doctype: doctype.to_sym,
-              converter_options: PdfOptionsNode.new(doctype, @compile_options),
+            { flavor: flavor.to_sym,
+              converter_options: PdfOptionsNode.new(flavor, @compile_options),
               outdir: @outdir },
           ).to_html(pres)
       end
@@ -155,14 +157,9 @@ module Metanorma
         out
       end
 
-      # infer the flavour from the first document identifier; relaton does that
-      def doctype
-        if (docid = @xml.at("//bibdata/docidentifier/@type")&.text)
-          dt = docid.downcase
-        elsif (docid = @xml.at("//bibdata/docidentifier")&.text)
-          dt = docid.sub(/\s.*$/, "").lowercase
-        else return "standoc"
-        end
+      # TODO: infer flavor from publisher when available
+      def flavor
+        dt = @xml.at("//bibdata/ext/flavor")&.text or return "standoc"
         @registry.alias(dt.to_sym)&.to_s || dt
       end
 

data/lib/metanorma/collection/renderer/svg.rb

@@ -0,0 +1,44 @@
+module Metanorma
+  class Collection
+    class Renderer
+      def svg_datauri(docxml, docid)
+        rel = @files.get(docid, :rel_path)
+        parent = @files.get(docid, :parentid) and
+          rel = @files.get(parent, :rel_path)
+        # if sectionsplit, use orig file dir
+        dir = File.join(@dirname, File.dirname(rel))
+        datauri_encode(docxml, dir)
+      end
+
+      def svgmap_resolve(docxml, docid)
+        ids = @files.get(docid, :ids)
+        docxml = svg_unnest(svg_datauri(docxml, docid))
+        isodoc = IsoDoc::PresentationXMLConvert.new({})
+        isodoc.bibitem_lookup(docxml)
+        docxml.xpath(ns("//svgmap//eref")).each do |e|
+          svgmap_resolve_eref(e, isodoc, docxml, ids)
+        end
+        Vectory::SvgMapping.new(docxml, "").call
+        docxml.xpath(ns("//svgmap")).each { |s| isodoc.svgmap_extract(s) }
+      end
+
+      def svg_unnest(docxml)
+        docxml.xpath(ns("//svgmap//image[.//*[name() = 'image']]")).each do |i|
+          s = i.elements.detect { |e| e.name == "svg" } and
+            i.replace(s)
+        end
+        docxml
+      end
+
+      def svgmap_resolve_eref(eref, isodoc, _docxml, ids)
+        href = isodoc.eref_target(eref) or return
+        href = href[:link]
+        href == "##{eref['bibitemid']}" ||
+          (href =~ /^#/ && !ids[href.sub(/^#/, "")]) and return
+        eref["target"] = href.strip
+        eref.name = "link"
+        eref.elements&.remove
+      end
+    end
+  end
+end

data/lib/metanorma/collection/renderer/utils.rb

@@ -111,9 +111,9 @@ module Metanorma
       end
 
       def pdfconv
-        doctype = @doctype.to_sym
-        x = Asciidoctor.load nil, backend: doctype
-        x.converter.pdf_converter(PdfOptionsNode.new(doctype,
+        flavor = @flavor.to_sym
+        x = Asciidoctor.load nil, backend: flavor
+        x.converter.pdf_converter(PdfOptionsNode.new(flavor,
                                                      @compile_options))
       end
 
@@ -153,8 +153,8 @@ module Metanorma
       end
 
       class PdfOptionsNode
-        def initialize(doctype, options)
-          p = Metanorma::Registry.instance.find_processor(doctype)
+        def initialize(flavor, options)
+          p = Metanorma::Registry.instance.find_processor(flavor)
           if ::Metanorma::Util::FontistHelper.has_custom_fonts?(p, options, {})
             @fonts_manifest =
               ::Metanorma::Util::FontistHelper.location_manifest(p, options)
@@ -169,7 +169,7 @@ module Metanorma
       end
 
       def isodoc_create
-        isodoc = Util::load_isodoc(@doctype)
+        isodoc = Util::load_isodoc(@flavor)
         isodoc.i18n_init(@lang, @script, @locale) # read in internationalisation
         isodoc.metadata_init(@lang, @script, @locale, isodoc.i18n)
         isodoc.info(@xml, nil)
@@ -203,6 +203,22 @@ module Metanorma
         end.doc.root.to_html
       end
 
+      def eref2link(docxml)
+        isodoc = IsoDoc::PresentationXMLConvert.new({})
+        isodoc.bibitem_lookup(docxml)
+        isodoc.eref2link(docxml)
+      end
+
+      def error_anchor(erefs, docid)
+        erefs.each do |e|
+          msg = "<strong>** Unresolved reference to document #{docid} " \
+            "from eref</strong>"
+          e << msg
+          strip_eref(e)
+          @log&.add("Cross-References", e, msg)
+        end
+      end
+
       def ns(xpath)
         @isodoc.ns(xpath)
       end

data/lib/metanorma/collection/sectionsplit/collection.rb

@@ -0,0 +1,100 @@
+module Metanorma
+  class Collection
+    class Sectionsplit
+      def build_collection
+        collection_setup(@base, @dir)
+        files = sectionsplit
+        input_xml = Nokogiri::XML(File.read(@input_filename,
+                                            encoding: "UTF-8"), &:huge)
+        collection_manifest(@base, files, input_xml, @xml, @dir).render(
+          { format: %i(html), output_folder: "#{@output_filename}_collection",
+            coverpage: File.join(@dir, "cover.html") }.merge(@compile_opts),
+        )
+        section_split_attachments(out: "#{@output_filename}_collection")
+      end
+
+      def collection_setup(filename, dir)
+        FileUtils.mkdir_p "#{filename}_collection" if filename
+        FileUtils.mkdir_p dir
+        File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
+          f.write(coll_cover)
+        end
+      end
+
+      def coll_cover
+        <<~COVER
+          <html><head><meta charset="UTF-8"/></head><body>
+                <h1>{{ doctitle }}</h1>
+                <h2>{{ docnumber }}</h2>
+                <nav>{{ navigation }}</nav>
+              </body></html>
+        COVER
+      end
+
+      def collection_manifest(filename, files, origxml, _presxml, dir)
+        File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
+          f.write(collectionyaml(files, origxml))
+        end
+        Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
+      end
+
+      def collectionyaml(files, xml)
+        ret = {
+          directives: ["presentation-xml", "bare-after-first"],
+          bibdata: {
+            title: {
+              type: "title-main", language: @lang,
+              content: xml.at(ns("//bibdata/title")).text
+            },
+            type: "collection",
+            docid: {
+              type: xml.at(ns("//bibdata/docidentifier/@type")).text,
+              id: xml.at(ns("//bibdata/docidentifier")).text,
+            },
+          },
+          manifest: {
+            level: "collection", title: "Collection",
+            docref: files.sort_by { |f| f[:order] }.each.map do |f|
+              { fileref: f[:url], identifier: f[:title] }
+            end
+          },
+        }
+        ::Metanorma::Util::recursive_string_keys(ret).to_yaml
+      end
+
+      def att_dir(file)
+        "_#{File.basename(file, '.*')}_attachments"
+      end
+
+      def section_split_attachments(out: nil)
+        attachments = att_dir(@tmp_filename)
+        File.directory?(attachments) or return
+        dir = out || File.dirname(@input_filename)
+        ret = File.join(dir, att_dir(@output_filename))
+        FileUtils.rm_rf ret
+        FileUtils.mv attachments, ret
+        File.basename(ret)
+      end
+
+      def section_split_cover(col, ident, one_doc_coll)
+        dir = File.dirname(col.file)
+        collection_setup(nil, dir)
+        r = ::Metanorma::Collection::Renderer
+          .new(col, dir, output_folder: "#{ident}_collection",
+                         format: %i(html),
+                         coverpage: File.join(dir, "cover.html"))
+        r.coverpage
+        section_split_cover1(ident, r, dir, one_doc_coll)
+      end
+
+      def section_split_cover1(ident, renderer, dir, _one_doc_coll)
+        filename = File.basename("#{ident}_index.html")
+        # ident can be a directory with YAML indirection
+        dest = File.join(dir, filename)
+        FileUtils.mv File.join(renderer.outdir, "index.html"), dest
+        FileUtils.rm_rf renderer.outdir
+        filename
+      end
+    end
+  end
+end