metanorma 1.3.4 → 1.3.8

data/lib/metanorma/document.rb

@@ -1,13 +1,15 @@
 module Metanorma
   class Document
     # @return [Strin]
-    attr_reader :file, :attachment, :bibitem
+    attr_reader :file, :attachment, :bibitem, :index
 
     # @param bibitem [RelatonBib::BibliographicItem]
     def initialize(bibitem, file, options = {})
       @bibitem = bibitem
       @file = file
       @attachment = options[:attachment]
+      @index = options[:index]
+      @index = true if @index.nil?
       @raw = options[:raw]
     end
 
@@ -16,10 +18,11 @@ module Metanorma
       # @param attachment [Bool] is an attachment
       # @param identifier [String] is the identifier assigned the file
       # in the collection file
+      # @param index [Bool] is indication on whether to index this file in coverpage
       # @return [Metanorma::Document]
-      def parse_file(file, attachment, identifier = nil)
+      def parse_file(file, attachment, identifier = nil, index = true)
         new(bibitem(file, attachment, identifier), file,
-            { attachment: attachment })
+            { attachment: attachment, index: index })
       end
 
       # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
@@ -30,7 +33,9 @@ module Metanorma
 
       # raw XML file, can be used to put in entire file instead of just bibitem
       def raw_file(filename)
-        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8"))
+        doc = Nokogiri::XML(File.read(filename, encoding: "UTF-8")) do |config|
+          config.huge
+        end
         new(doc, filename, raw: true)
       end
 
@@ -100,7 +105,9 @@ module Metanorma
       if @raw
         builder << @bibitem.root.to_xml
       else
-        builder.send(type + "-standard") { |b| b << @bibitem.to_xml(bibdata: true) }
+        builder.send("#{type}-standard") do |b|
+          b << @bibitem.to_xml(bibdata: true)
+        end
       end
     end
   end

data/lib/metanorma/input/asciidoc.rb

@@ -57,9 +57,9 @@ module Metanorma
       end
 
       ADOC_OPTIONS = %w(htmlstylesheet htmlcoverpage htmlintropage scripts
-                        scripts-override scripts-pdf wordstylesheet bare i18nyaml
+                        scripts-override scripts-pdf wordstylesheet i18nyaml
                         standardstylesheet header wordcoverpage wordintropage
-                        ulstyle olstyle htmlstylesheet-override
+                        ulstyle olstyle htmlstylesheet-override bare
                         htmltoclevels doctoclevels sectionsplit
                         body-font header-font monospace-font title-font
                         wordstylesheet-override).freeze

data/lib/metanorma/processor.rb

@@ -3,10 +3,7 @@
 
 module Metanorma
   class Processor
-
-    attr_reader :short
-    attr_reader :input_format
-    attr_reader :asciidoctor_backend
+    attr_reader :short, :input_format, :asciidoctor_backend
 
     def initialize
       raise "This is an abstract class!"
@@ -16,7 +13,7 @@ module Metanorma
       {
         xml: "xml",
         presentation: "presentation.xml",
-        rxl: "rxl"
+        rxl: "rxl",
       }
     end
 
@@ -36,7 +33,7 @@ module Metanorma
       end
     end
 
-    def output(isodoc_node, inname, outname, format, options={})
+    def output(isodoc_node, _inname, outname, _format, _options = {})
       File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
     end
 

data/lib/metanorma/sectionsplit.rb

@@ -0,0 +1,270 @@
+require "yaml"
+
+module Metanorma
+  class Compile
+    # assume we pass in Presentation XML, but we want to recover Semantic XML
+    def sectionsplit_convert(input_filename, file, output_filename = nil,
+                             opts = {})
+      @isodoc = IsoDoc::Convert.new({})
+      input_filename += ".xml" unless input_filename.match?(/\.xml$/)
+      File.exist?(input_filename) or
+        File.open(input_filename, "w:UTF-8") { |f| f.write(file) }
+      presxml = File.read(input_filename, encoding: "utf-8")
+      @openmathdelim, @closemathdelim = @isodoc.extract_delims(presxml)
+      xml, filename, dir = @isodoc.convert_init(presxml, input_filename, false)
+      build_collection(xml, presxml, output_filename || filename, dir, opts)
+    end
+
+    def ns(xpath)
+      @isodoc.ns(xpath)
+    end
+
+    def build_collection(xml, presxml, filename, dir, opts = {})
+      base = File.basename(filename)
+      collection_setup(base, dir)
+      files = sectionsplit(xml, base, dir)
+      collection_manifest(base, files, xml, presxml, dir).render(
+        { format: %i(html), output_folder: "#{filename}_collection",
+          coverpage: File.join(dir, "cover.html") }.merge(opts),
+      )
+    end
+
+    def collection_manifest(filename, files, origxml, _presxml, dir)
+      File.open(File.join(dir, "#{filename}.html.yaml"), "w:UTF-8") do |f|
+        f.write(collectionyaml(files, origxml))
+      end
+      Metanorma::Collection.parse File.join(dir, "#{filename}.html.yaml")
+    end
+
+    def collection_setup(filename, dir)
+      FileUtils.mkdir_p "#{filename}_collection" if filename
+      FileUtils.mkdir_p dir
+      File.open(File.join(dir, "cover.html"), "w:UTF-8") do |f|
+        f.write(coll_cover)
+      end
+    end
+
+    def coll_cover
+      <<~COVER
+        <html>
+          <head/>
+            <body>
+              <h1>{{ doctitle }}</h1>
+              <h2>{{ docnumber }}</h2>
+              <nav>{{ labels["navigation"] }}</nav>
+            </body>
+        </html>
+      COVER
+    end
+
+    SPLITSECTIONS =
+      [["//preface/*", "preface"], ["//sections/*", "sections"],
+       ["//annex", nil],
+       ["//bibliography/*[not(@hidden = 'true')]", "bibliography"],
+       ["//indexsect", nil]].freeze
+
+    def sectionsplit(xml, filename, dir)
+      @key = xref_preprocess(xml)
+      @splitdir = dir
+      out = emptydoc(xml)
+      SPLITSECTIONS.each_with_object([]) do |n, ret|
+        xml.xpath(ns(n[0])).each do |s|
+          ret << sectionfile(xml, out, "#{filename}.#{ret.size}", s, n[1])
+        end
+      end
+    end
+
+    def emptydoc(xml)
+      out = xml.dup
+      out.xpath(
+        ns("//preface | //sections | //annex | //bibliography/clause | "\
+           "//bibliography/references[not(@hidden = 'true')] | //indexsect"),
+      ).each(&:remove)
+      out
+    end
+
+    def sectionfile(fulldoc, xml, file, chunk, parentnode)
+      fname = create_sectionfile(fulldoc, xml.dup, file, chunk, parentnode)
+      { order: chunk["displayorder"].to_i, url: fname,
+        title: titlerender(chunk) }
+    end
+
+    def create_sectionfile(xml, out, file, chunk, parentnode)
+      ins = out.at(ns("//misccontainer")) || out.at(ns("//bibdata"))
+      if parentnode
+        ins.next = "<#{parentnode}/>"
+        ins.next.add_child(chunk.dup)
+      else ins.next = chunk.dup
+      end
+      xref_process(out, xml, @key)
+      outname = "#{file}.xml"
+      File.open(File.join(@splitdir, outname), "w:UTF-8") { |f| f.write(out) }
+      outname
+    end
+
+    #     def xref_preprocess(xml)
+    #       svg_preprocess(xml)
+    #       key = (0...8).map { rand(65..90).chr }.join # random string
+    #       xml.root["type"] = key # to force recognition of internal refs
+    #       refs = eref_to_internal_eref(xml, key)
+    #       refs += xref_to_internal_eref(xml, key)
+    #       ins = new_hidden_ref(xml)
+    #       copy_repo_items_biblio(ins, xml)
+    #       insert_indirect_biblio(ins, refs, key)
+    #     end
+
+    def xref_preprocess(xml)
+      svg_preprocess(xml)
+      key = (0...8).map { rand(65..90).chr }.join # random string
+      xml.root["type"] = key # to force recognition of internal refs
+      key
+    end
+
+    def xref_process(section, xml, key)
+      refs = eref_to_internal_eref(section, xml, key)
+      refs += xref_to_internal_eref(section, key)
+      ins = new_hidden_ref(section)
+      copied_refs = copy_repo_items_biblio(ins, section, xml)
+      insert_indirect_biblio(ins, refs - copied_refs, key)
+    end
+
+    def svg_preprocess(xml)
+      xml.xpath("//m:svg", "m" => "http://www.w3.org/2000/svg").each do |s|
+        m = svgmap_wrap(s)
+        s.xpath(".//m:a", "m" => "http://www.w3.org/2000/svg").each do |a|
+          next unless /^#/.match? a["href"]
+
+          a["href"] = a["href"].sub(/^#/, "")
+          m << "<target href='#{a['href']}'>"\
+            "<xref target='#{a['href']}'/></target>"
+        end
+      end
+    end
+
+    def svgmap_wrap(svg)
+      ret = svg.at("./ancestor::xmlns:svgmap") and return ret
+      ret = svg.at("./ancestor::xmlns:figure")
+      ret.wrap("<svgmap/>")
+      svg.at("./ancestor::xmlns:svgmap")
+    end
+
+    def make_anchor(anchor)
+      "<localityStack><locality type='anchor'><referenceFrom>"\
+        "#{anchor}</referenceFrom></locality></localityStack>"
+    end
+
+    def xref_to_internal_eref(xml, key)
+      xml.xpath(ns("//xref")).each_with_object({}) do |x, m|
+        x["bibitemid"] = "#{key}_#{x['target']}"
+        x << make_anchor(x["target"])
+        m[x["bibitemid"]] = true
+        x.delete("target")
+        x["type"] = key
+        x.name = "eref"
+      end.keys
+    end
+
+    def eref_to_internal_eref(section, xml, key)
+      eref_to_internal_eref_select(section, xml).each_with_object([]) do |x, m|
+        url = xml.at(ns("//bibitem[@id = '#{x}']/url[@type = 'citation']"))
+        section.xpath(("//*[@bibitemid = '#{x}']")).each do |e|
+          id = eref_to_internal_eref1(e, key, url)
+          id and m << id
+        end
+      end
+    end
+
+    def eref_to_internal_eref1(elem, key, url)
+      if url
+        elem.name = "link"
+        elem["target"] = url
+        nil
+      else
+        elem["bibitemid"] = "#{key}_#{elem['bibitemid']}"
+        elem << make_anchor(elem["bibitemid"])
+        elem["type"] = key
+        elem["bibitemid"]
+      end
+    end
+
+    def eref_to_internal_eref_select(section, xml)
+      refs = section.xpath(("//*/@bibitemid")).map { |x| x.text } # rubocop:disable Style/SymbolProc
+      refs.uniq.reject do |x|
+        xml.at(ns("//bibitem[@id = '#{x}'][@type = 'internal']")) ||
+          xml.at(ns("//bibitem[@id = '#{x}']"\
+                    "[docidentifier/@type = 'repository']"))
+      end
+    end
+
+    # from standoc
+    def new_hidden_ref(xmldoc)
+      ins = xmldoc.at("bibliography") or
+        xmldoc.root << "<bibliography/>" and ins = xmldoc.at("bibliography")
+      ins.add_child("<references hidden='true' normative='false'/>").first
+    end
+
+    def copy_repo_items_biblio(ins, section, xml)
+      xml.xpath(ns("//references/bibitem[docidentifier/@type = 'repository']"))
+        .each_with_object([]) do |b, m|
+        section.at("//*[@bibitemid = '#{b['id']}']") or next
+        ins << b.dup
+        m << b["id"]
+      end
+    end
+
+    def insert_indirect_biblio(ins, refs, prefix)
+      refs.each do |x|
+        ins << <<~BIBENTRY
+          <bibitem id="#{x}" type="internal">
+          <docidentifier type="repository">#{x.sub(/^#{prefix}_/, "#{prefix}/")}</docidentifier>
+          </bibitem>
+        BIBENTRY
+      end
+    end
+
+    def recursive_string_keys(hash)
+      case hash
+      when Hash then Hash[
+        hash.map { |k, v| [k.to_s, recursive_string_keys(v)] }
+      ]
+      when Enumerable then hash.map { |v| recursive_string_keys(v) }
+      else
+        hash
+      end
+    end
+
+    def titlerender(section)
+      title = section.at(ns("./title")) or return "[Untitled]"
+      t = title.dup
+      t.xpath(ns(".//tab | .//br")).each { |x| x.replace(" ") }
+      t.xpath(ns(".//strong")).each { |x| x.replace(x.children) }
+      t.children.to_xml
+    end
+
+    def collectionyaml(files, xml)
+      ret = {
+        directives: ["presentation-xml", "bare-after-first"],
+        bibdata: {
+          title: {
+            type: "title-main",
+            language: @lang,
+            content: xml.at(ns("//bibdata/title")).text,
+          },
+          type: "collection",
+          docid: {
+            type: xml.at(ns("//bibdata/docidentifier/@type")).text,
+            id: xml.at(ns("//bibdata/docidentifier")).text,
+          },
+        },
+        manifest: {
+          level: "collection",
+          title: "Collection",
+          docref: files.sort_by { |f| f[:order] }.each.map do |f|
+            { fileref: f[:url], identifier: f[:title] }
+          end,
+        },
+      }
+      recursive_string_keys(ret).to_yaml
+    end
+  end
+end

data/lib/metanorma/util.rb

@@ -12,8 +12,49 @@ module Metanorma
       end
     end
 
-    def self.source2dest_filename(name)
-      name.sub(%r{^(\./)?(\.\./)+}, "")
+    # dependency ordering
+    def self.sort_extensions_execution_ord(ext)
+      case ext
+      when :xml then 0
+      when :rxl then 1
+      when :presentation then 2
+      else
+        99
+      end
+    end
+
+    def self.sort_extensions_execution(ext)
+      ext.sort do |a, b|
+        sort_extensions_execution_ord(a) <=> sort_extensions_execution_ord(b)
+      end
+    end
+
+    class DisambigFiles
+      def initialize
+        @seen_filenames = []
+      end
+
+      def source2dest_filename(name, disambig = true)
+        n = name.sub(%r{^(\./)?(\.\./)+}, "")
+        dir = File.dirname(n)
+        base = File.basename(n)
+        if disambig && @seen_filenames.include?(base)
+          base = disambiguate_filename(base)
+        end
+        @seen_filenames << base
+        dir == "." ? base : File.join(dir, base)
+      end
+
+      def disambiguate_filename(base)
+        m = /^(?<start>.+\.)(?!0)(?<num>\d+)\.(?<suff>[^.]*)$/.match(base) ||
+          /^(?<start>.+\.)(?<suff>[^.]*)/.match(base) ||
+          /^(?<start>.+)$/.match(base)
+        i = m.names.include?("num") ? m["num"].to_i + 1 : 1
+        while @seen_filenames.include? base = "#{m['start']}#{i}.#{m['suff']}"
+          i += 1
+        end
+        base
+      end
     end
   end
 end

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.3.4".freeze
+  VERSION = "1.3.8".freeze
 end

data/metanorma.gemspec

@@ -20,10 +20,10 @@ Gem::Specification.new do |spec|
   spec.bindir        = "bin"
   # spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.4.0"
+  spec.required_ruby_version = ">= 2.5.0"
 
   spec.add_runtime_dependency "asciidoctor"
-  spec.add_runtime_dependency "fontist", "~> 1.8"
+  spec.add_runtime_dependency "fontist", "~> 1.9.0"
   spec.add_runtime_dependency "htmlentities"
   spec.add_runtime_dependency "metanorma-utils", "~> 1.2.0"
   spec.add_runtime_dependency "mn2pdf", "~> 1"
@@ -36,10 +36,11 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "byebug", "~> 10.0"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.8.0"
+  spec.add_development_dependency "metanorma-iso", "~> 1.9.0"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "rubocop", "~> 1.5.2"
   spec.add_development_dependency "sassc", "~> 2.4.0"
+  spec.add_development_dependency "mnconvert"
 end