metanorma 1.2.4 → 1.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf7d6f99cb5eec27df20786b79a6932eb9e339c49072f4183b5cf9cb2ff747a3
-  data.tar.gz: '049577c6c56eb8092765b8e3679ac7f0c8362f9129e787e4e7b923f5b3b61734'
+  metadata.gz: 72a5ff12b5b2793f4d8b8ced7b5cdc747a7b59cb938300f1e288150787d82951
+  data.tar.gz: 2a8f1a0fda7ceed680324833c18ab595d9de381aab36f3a5848040743029dfc8
 SHA512:
-  metadata.gz: 7de4db35db282e2ec0665d79533885c31fd8a1b46493177d47a48920ab685e20d2f5a74e473b11d0dfa54b357b8d4e0be15d2908885a770b4e3ec518dd427975
-  data.tar.gz: 8ef45f441c0129920575751b10bf3d86b132395d020794d39e69429e099ba6bcde65f1d5930dca240ad07209e12eddada6926bf56ec54322674ef8f47e07cedc
+  metadata.gz: d8ad8063e83a0d51494cb9d69f552676b27ac2022c9d477af6e12902bc2a9c7ceddc07d51965c594ab3ddf92839da6071d8a407836774e26c676b2fd3c9d584d
+  data.tar.gz: 18b394b6fd082a3072d543b120656ee181a68ce4c1a4cf3f0551f53719e07a96ec1b2140d61fac86e328370464e024df38872a0d85bc0b95aee588bb3e844982

data/.github/workflows/notify.yml

@@ -3,16 +3,16 @@
 name: notify
 
 on:
-  push:
-    branches: [ master ]
-    tags:
-      - '*'
+  repository_dispatch:
+    types: [ notify ]
 
 jobs:
   notify:
     name: Notify dependent repos
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v2
+
       - name: Trigger repositories
         env:
           GH_USERNAME: metanorma-ci
@@ -21,7 +21,7 @@ jobs:
           curl -LO --retry 3 https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/trigger-gh-actions.sh
           [[ -f ".github/workflows/dependent_repos.env" ]] && source .github/workflows/dependent_repos.env
           CLIENT_PAYLOAD=$(cat <<EOF
-          "{ "ref": "${GITHUB_REF}", "repo": "${GITHUB_REPOSITORY}" }"
+          "{ "ref": "${{ github.event.client_payload.ref }}", "repo": "${GITHUB_REPOSITORY}" }"
           EOF
           )
           for repo in $TEMPLATE_REPOS" $SAMPLES_REPOS"
@@ -30,7 +30,7 @@ jobs:
           done
 
       - name: Trigger release repositories
-        if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')
+        if: github.event.client_payload.ref == 'refs/heads/master' || startsWith(github.event.client_payload.ref, 'refs/tags/v')
         env:
           GH_USERNAME: metanorma-ci
           GH_ACCESS_TOKEN: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
@@ -38,7 +38,7 @@ jobs:
           curl -LO --retry 3 https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/trigger-gh-actions.sh
           [[ -f ".github/workflows/dependent_repos.env" ]] && source .github/workflows/dependent_repos.env
           CLIENT_PAYLOAD=$(cat <<EOF
-          "{ "ref": "${GITHUB_REF}", "repo": "${GITHUB_REPOSITORY}" }"
+          "{ "ref": "${{ github.event.client_payload.ref }}", "repo": "${GITHUB_REPOSITORY}" }"
           EOF
           )
           for repo in $DEPENDENT_REPOS

data/.github/workflows/rake.yml

@@ -4,7 +4,7 @@ name: rake
 
 on:
   push:
-    branches: [ master, main  ]
+    branches: [ master, main ]
     tags: [ v* ]
   pull_request:
 
@@ -16,38 +16,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.6', '2.5', '2.4' ]
+        ruby: [ '2.7', '2.6', '2.5', '2.4' ]
         os: [ ubuntu-latest, windows-latest, macos-latest ]
         experimental: [ false ]
         include:
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'ubuntu-latest'
             experimental: true
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'windows-latest'
             experimental: true
-          - ruby: '2.7'
+          - ruby: '3.0'
             os: 'macos-latest'
             experimental: true
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
 
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-
-      - if: matrix.os == 'macos-latest'
-        run: brew install autoconf automake libtool
-
-      - uses: actions/cache@v2
-        with:
-          path: vendor/bundle
-          key: bundle-${{ matrix.os }}-${{ matrix.ruby }}-${{ hashFiles('**/*.gemspec') }}
-          restore-keys: bundle-${{ matrix.os }}-${{ matrix.ruby }}
-
-      - run: bundle config set path 'vendor/bundle'
-
-      - run: bundle install --jobs 4 --retry 3
+          bundler-cache: true
 
       - run: bundle exec rake
 
@@ -55,11 +45,9 @@ jobs:
     needs: rake
     runs-on: ubuntu-latest
     steps:
-      - name: Trigger tests passed event
-        uses: Sibz/github-status-action@v1
+      - uses: peter-evans/repository-dispatch@v1
         with:
-          authToken: ${{ secrets.GITHUB_TOKEN }}
-          context: 'tests-passed-successfully'
-          description: 'Tests passed successfully'
-          state: 'success'
-          sha: ${{ github.event.pull_request.head.sha || github.sha }}
+          token: ${{ secrets.METANORMA_CI_PAT_TOKEN || secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          event-type: notify
+          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

data/.gitignore

@@ -20,3 +20,5 @@
 Gemfile.lock
 relaton/
 .vscode/
+
+.rubocop-https--*

data/.rubocop.yml

@@ -1,6 +1,14 @@
 # This project follows the Ribose OSS style guide.
 # https://github.com/riboseinc/oss-guides
 # All project-specific additions and overrides should be specified in this file.
-
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
+
+# local repo-specific modifications
+
+AllCops:
+  DisplayCopNames: false
+  StyleGuideCopsOnly: false
+  TargetRubyVersion: 2.4
+Rails:
+  Enabled: true

data/lib/metanorma/collection.rb

@@ -3,6 +3,7 @@
 require "relaton"
 require "relaton/cli"
 require "metanorma/collection_manifest"
+require "metanorma-utils"
 
 module Metanorma
   # Metanorma collection of documents
@@ -39,8 +40,13 @@ module Metanorma
       @documents.merge! @manifest.documents(File.dirname(@file))
       @prefatory = args[:prefatory]
       @final = args[:final]
+      @log = Metanorma::Utils::Log.new
     end
+
     # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
+     def clean_exit
+       @log.write(File.join(File.dirname(@file), File.basename(@file, ".*") + ".err"))
+     end
 
     # @return [String] XML
     def to_xml
@@ -57,7 +63,8 @@ module Metanorma
     end
 
     def render(opts)
-      CollectionRenderer.render self, opts
+      CollectionRenderer.render self, opts.merge(log: @log)
+      clean_exit
     end
 
     class << self

data/lib/metanorma/collection_fileprocess.rb

@@ -0,0 +1,292 @@
+# frozen_string_literal: true
+
+require "isodoc"
+require "metanorma-utils"
+
+module Metanorma
+  # XML collection renderer
+  class CollectionRenderer
+    # hash for each document in collection of document identifier to:
+    # document reference (fileref or id), type of document reference,
+    # and bibdata entry for that file
+    # @param path [String] path to collection
+    # @return [Hash{String=>Hash}]
+    def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      files = {}
+      @xml.xpath(ns("//docref")).each do |d|
+        identifier = d.at(ns("./identifier")).text
+        files[identifier] = if d["fileref"]
+                              { type: "fileref",
+                                ref: File.join(path, d["fileref"]) }
+                            else { type: "id", ref: d["id"] }
+                            end
+        file, _filename = targetfile(files[identifier], true)
+        xml = Nokogiri::XML(file)
+        add_document_suffix(identifier, xml)
+        files[identifier][:anchors] = read_anchors(xml)
+        files[identifier][:bibdata] = xml.at(ns("//bibdata"))
+      end
+      files
+    end
+
+    def add_suffix_to_attributes(doc, suffix, tag_name, attribute_name)
+      doc.xpath(ns("//#{tag_name}[@#{attribute_name}]")).each do |elem|
+        elem.attributes[attribute_name].value =
+          "#{elem.attributes[attribute_name].value}_#{suffix}"
+      end
+    end
+
+    def add_document_suffix(identifier, doc)
+      document_suffix = Metanorma::Utils::to_ncname(identifier)
+      [%w[* id],
+      %w[* bibitemid],
+      %w[review from],
+      %w[review to],
+      %w[index to],
+      %w[xref target],
+      %w[callout target]]
+      .each do |(tag_name, attribute_name)|
+        add_suffix_to_attributes(doc, document_suffix, tag_name, attribute_name)
+      end
+    end
+
+    # map locality type and label (e.g. "clause" "1") to id = anchor for
+    # a document
+    def read_anchors(xml)
+      ret = {}
+      xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
+      xrefs.parse xml
+      xrefs.get.each do |k, v|
+        ret[v[:type]] ||= {}
+        index = v[:container] || v[:label].nil? || v[:label].empty? ? 
+          UUIDTools::UUID.random_create.to_s : v[:label]
+        # Note: will only key clauses, which have unambiguous reference label in locality.
+        # Notes, examples etc with containers are just plunked agaisnt UUIDs, so that their
+        # IDs can at least be registered to be tracked as existing.
+        ret[v[:type]][index] = k
+      end
+      ret
+    end
+
+    # return file contents + output filename for each file in the collection,
+    # given a docref entry
+    # @param data [Hash]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def targetfile(data, read = false)
+      if data[:type] == "fileref" then ref_file data[:ref], read
+      else xml_file data[:id], read
+      end
+    end
+
+    # @param ref [String]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def ref_file(ref, read)
+      file = File.read(ref, encoding: "utf-8") if read
+      filename = ref.sub(/\.xml$/, ".html")
+      [file, filename]
+    end
+
+    # @param id [String]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def xml_file(id, read)
+      file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
+      filename = id + ".html"
+      [file, filename]
+    end
+
+    # @param bib [Nokogiri::XML::Element]
+    # @param identifier [String]
+    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      docid = bib&.at(ns("./docidentifier"))&.text
+      unless @files[docid]
+        error = "[metanorma] Cannot find crossreference to document #{docid} in document #{identifier}."
+        @log.add("Cross-References", nil, error)
+        Util.log(error, :warning)
+        return
+      end
+      id = bib["id"]
+      newbib = bib.replace(@files[docid][:bibdata])
+      newbib.name = "bibitem"
+      newbib["id"] = id
+      newbib["hidden"] = "true"
+      newbib&.at(ns("./ext"))&.remove
+      _file, url = targetfile(@files[docid], false)
+      uri_node = Nokogiri::XML::Node.new "uri", newbib
+      uri_node[:type] = "citation"
+      uri_node.content = url
+      newbib.at(ns("./docidentifier")).previous = uri_node
+    end
+
+    # Resolves direct links to other files in collection (repo(current-metanorma-collection/x),
+    # and indirect links to other files in collection (bibitem[@type = 'internal'] pointing to a file anchor
+    # in another file in the collection)
+    # @param file [String] XML content
+    # @param identifier [String] docid
+    # @param internal_refs [Hash{String=>Hash{String=>String}] schema name to anchor to filename
+    # @return [String] XML content
+    def update_xrefs(file, identifier, internal_refs)
+      docxml = Nokogiri::XML(file)
+      update_indirect_refs_to_docs(docxml, internal_refs)
+      add_document_suffix(identifier, docxml)
+      update_direct_refs_to_docs(docxml, identifier)
+      svgmap_resolve(datauri_encode(docxml))
+      docxml.xpath(ns("//references[not(./bibitem[not(@hidden) or @hidden = 'false'])]")).each do |f|
+        f["hidden"] = "true"
+      end
+      docxml.to_xml
+    end
+
+    def datauri_encode(docxml)
+      docxml.xpath(ns("//image")).each { |i| i["src"] = Metanorma::Utils::datauri(i["src"]) }
+      docxml
+    end
+
+    def svgmap_resolve(docxml)
+      isodoc = IsoDoc::Convert.new({})
+      docxml.xpath(ns("//svgmap//eref")).each do |e|
+        href = isodoc.eref_target(e)
+        next if href == "#" + e["bibitemid"]
+        if href.match(/^#/)
+          next unless docxml.at("//*[@id = '#{href.sub(/^#/, '')}']")
+        end
+        e["target"] = href.strip
+        e.name = "link"
+        e&.elements&.remove
+      end
+      Metanorma::Utils::svgmap_rewrite(docxml, "")
+    end
+
+    # repo(current-metanorma-collection/ISO 17301-1:2016)
+    # replaced by bibdata of "ISO 17301-1:2016" in situ as bibitem.
+    # Any erefs to that bibitem id are replaced with relative URL
+    # Preferably with anchor, and is a job to realise dynamic lookup of localities.
+    def update_direct_refs_to_docs(docxml, identifier)
+      docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
+        docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
+        next unless docid && %r{^current-metanorma-collection/}.match(docid)
+        update_bibitem(b, identifier)
+        update_anchors(b, docxml, docid)
+      end
+    end
+
+    # Resolve erefs to a container of ids in another doc, to an anchor eref (direct link)
+    def update_indirect_refs_to_docs(docxml, internal_refs)
+      internal_refs.each do |schema, ids|
+        ids.each do |id, file|
+          update_indirect_refs_to_docs1(docxml, schema, id, file)
+        end
+      end
+    end
+
+    def update_indirect_refs_to_docs1(docxml, schema, id, file)
+      docxml.xpath(ns("//eref[@bibitemid = '#{schema}_#{id}']")).each do |e|
+        e["citeas"] = file
+      end
+      docid = docxml.at(ns("//bibitem[@id = '#{schema}_#{id}']/docidentifier[@type = 'repository']")) or return
+      docid.children = "current-metanorma-collection/#{file}"
+      docid.previous = "<docidentifier type='X'>#{file}</docidentifier>"
+    end
+
+    # update crossrefences to other documents, to include disambiguating document suffix on id
+    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      docid = bib&.at(ns("./docidentifier"))&.text
+      docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
+        if @files[docid]
+          update_anchor_loc(bib, e, docid)
+        else
+          e << "<strong>** Unresolved reference to document #{docid}, id #{e['bibitemid']}</strong>"
+        end
+      end
+    end
+
+    def update_anchor_loc(bib, e, docid)
+      loc = e.at(ns(".//locality[@type = 'anchor']")) or return update_anchor_create_loc(bib, e, docid)
+      document_suffix = Metanorma::Utils::to_ncname(docid)
+      ref = loc.at(ns("./referenceFrom")) || return
+      anchor = "#{ref.text}_#{document_suffix}"
+      return unless @files[docid][:anchors].inject([]) { |m, (_, x)| m+= x.values }.include?(anchor)
+      ref.content = anchor
+    end
+
+    # if there is a crossref to another document, with no anchor, retrieve the
+    # anchor given the locality, and insert it into the crossref
+    def update_anchor_create_loc(bib, e, docid)
+      ins = e.at(ns("./localityStack")) || return
+      type = ins&.at(ns("./locality/@type"))&.text
+      ref = ins&.at(ns("./locality/referenceFrom"))&.text
+      (anchor = @files[docid][:anchors][type][ref]) || return
+      ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
+      ref_from.content = anchor.sub(/^_/, "")
+      locality = Nokogiri::XML::Node.new "locality", bib
+      locality[:type] = "anchor"
+      locality.add_child ref_from
+      ins << locality
+    end
+
+    # compile and output individual file in collection
+    def file_compile(f, filename, identifier)
+      # warn "metanorma compile -x html #{f.path}"
+      c = Compile.new
+      c.compile f.path, { format: :asciidoc, extension_keys: @format }.merge(@compile_options)
+      @files[identifier][:outputs] = {}
+      @format.each do |e|
+        ext = c.processor.output_formats[e]
+        fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
+        FileUtils.mv f.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
+        @files[identifier][:outputs][e] = File.join(@outdir, fn)
+      end
+    end
+
+    # gather internal bibitem references
+    def gather_internal_refs
+      @files.each_with_object({}) do |(identifier, x), refs|
+        file, _ = targetfile(x, true)
+        Nokogiri::XML(file).xpath(ns("//bibitem[@type = 'internal']/docidentifier[@type = 'repository']")).each do |d|
+          a = d.text.split(%r{/}, 2)
+          a.size > 1 or next
+          refs[a[0]] ||= {}
+          refs[a[0]][a[1]] = true
+        end
+      end
+    end
+
+    # resolve file location for the target of each internal reference
+    def locate_internal_refs
+      refs = gather_internal_refs
+      @files.each do |identifier, x|
+        file, filename = targetfile(x, true)
+        docxml = Nokogiri::XML(file)
+        refs.each do |schema, ids|
+          ids.keys.each do |id|
+            n = docxml.at("//*[@id = '#{id}']") and n.at("./ancestor-or-self::*[@type = '#{schema}']") and
+              refs[schema][id] = identifier
+          end
+        end
+      end
+      refs.each do |schema, ids|
+        ids.each do |id, key|
+          key == true and refs[schema][id] = "Missing:#{schema}:#{id}"
+        end
+      end
+      refs
+    end
+
+    # process each file in the collection
+    # files are held in memory, and altered as postprocessing
+    def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      internal_refs = locate_internal_refs
+      @files.each do |identifier, x|
+        file, filename = targetfile(x, true)
+        file = update_xrefs(file, identifier, internal_refs)
+        Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
+          f.write(file)
+          f.close
+          file_compile(f, filename, identifier)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/collection_renderer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "isodoc"
+require_relative "./collection_fileprocess"
 
 module Metanorma
   # XML collection renderer
@@ -34,6 +35,7 @@ module Metanorma
       @coverpage = options[:coverpage]
       @format = options[:format]
       @compile_options = options[:compile] || {}
+      @log = options[:log]
 
       # list of files in the collection
       @files = read_files folder
@@ -114,68 +116,6 @@ module Metanorma
       IsoDoc::Convert.new({}).ns(xpath)
     end
 
-    # hash for each document in collection of document identifier to:
-    # document reference (fileref or id), type of document reference,
-    # and bibdata entry for that file
-    # @param path [String] path to collection
-    # @return [Hash{String=>Hash}]
-    def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      files = {}
-      @xml.xpath(ns("//docref")).each do |d|
-        identifier = d.at(ns("./identifier")).text
-        files[identifier] = if d["fileref"]
-                              { type: "fileref",
-                                ref: File.join(path, d["fileref"]) }
-                            else { type: "id", ref: d["id"] }
-                            end
-        file, _filename = targetfile(files[identifier], true)
-        xml = Nokogiri::XML(file)
-        add_document_suffix(identifier, xml)
-        files[identifier][:anchors] = read_anchors(xml)
-        files[identifier][:bibdata] = xml.at(ns("//bibdata"))
-      end
-      files
-    end
-
-    def add_suffix_to_attributes(doc, suffix, tag_name, attribute_name)
-      doc.xpath(ns("//#{tag_name}[@#{attribute_name}]")).each do |elem|
-        elem.attributes[attribute_name].value =
-          "#{elem.attributes[attribute_name].value}_#{suffix}"
-      end
-    end
-
-    def add_document_suffix(identifier, doc)
-      document_suffix = Asciidoctor::Standoc::Cleanup.to_ncname(identifier)
-      [%w[* id],
-      %w[* bibitemid],
-      %w[review from],
-      %w[review to],
-      %w[index to],
-      %w[xref target],
-      %w[callout target]]
-      .each do |(tag_name, attribute_name)|
-        add_suffix_to_attributes(doc, document_suffix, tag_name, attribute_name)
-      end
-    end
-
-    # map locality type and label (e.g. "clause" "1") to id = anchor for
-    # a document
-    def read_anchors(xml)
-      ret = {}
-      xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
-      xrefs.parse xml
-      xrefs.get.each do |k, v|
-        ret[v[:type]] ||= {}
-        index = v[:container] || v[:label].nil? || v[:label].empty? ? 
-          UUIDTools::UUID.random_create.to_s : v[:label]
-        # Note: will only key clauses, which have unambiguous reference label in locality.
-        # Notes, examples etc with containers are just plunked agaisnt UUIDs, so that their
-        # IDs can at least be registered to be tracked as existing.
-        ret[v[:type]][index] = k
-      end
-      ret
-    end
-
     # populate liquid template of ARGV[1] with metadata extracted from
     # collection manifest
     def coverpage
@@ -235,132 +175,6 @@ module Metanorma
       end.doc.root.to_html
     end
 
-    # return file contents + output filename for each file in the collection,
-    # given a docref entry
-    # @param data [Hash]
-    # @param read [Boolean]
-    # @return [Array<String, nil>]
-    def targetfile(data, read = false)
-      if data[:type] == "fileref" then ref_file data[:ref], read
-      else xml_file data[:id], read
-      end
-    end
-
-    # @param ref [String]
-    # @param read [Boolean]
-    # @return [Array<String, nil>]
-    def ref_file(ref, read)
-      file = File.read(ref, encoding: "utf-8") if read
-      filename = ref.sub(/\.xml$/, ".html")
-      [file, filename]
-    end
-
-    # @param id [String]
-    # @param read [Boolean]
-    # @return [Array<String, nil>]
-    def xml_file(id, read)
-      file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
-      filename = id + ".html"
-      [file, filename]
-    end
-
-    # @param bib [Nokogiri::XML::Element]
-    # @param identifier [String]
-    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      docid = bib&.at(ns("./docidentifier"))&.text
-      unless @files[docid]
-        warn "Cannot find crossreference to document #{docid} in document "\
-          "#{identifier}!"
-        abort
-      end
-      id = bib["id"]
-      newbib = bib.replace(@files[docid][:bibdata])
-      newbib.name = "bibitem"
-      newbib["id"] = id
-      newbib["hidden"] = "true"
-      newbib&.at(ns("./ext"))&.remove
-      _file, url = targetfile(@files[docid], false)
-      uri_node = Nokogiri::XML::Node.new "uri", newbib
-      uri_node[:type] = "citation"
-      uri_node.content = url
-      newbib.at(ns("./docidentifier")).previous = uri_node
-    end
-
-    # TODO: update crossreferences to other files in the selection
-    # repo(current-metanorma-collection/ISO 17301-1:2016)
-    # replaced by
-    # bibdata of "ISO 17301-1:2016" in situ as bibitem
-    # Any erefs to that bibitem id are replaced with relative URL
-    # Preferably with anchor, and is a job to realise dynamic lookup of
-    # localities
-    # @param file [String] XML content
-    # @param identifier [String] docid
-    # @return [String] XML content
-    def update_xrefs(file, identifier)
-      docxml = Nokogiri::XML(file)
-      add_document_suffix(identifier, docxml)
-      docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
-        docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
-        next unless docid && %r{^current-metanorma-collection/}.match(docid)
-        update_bibitem(b, identifier)
-        update_anchors(b, docxml, docid)
-      end
-      docxml.xpath(ns("//references[not(./bibitem[not(@hidden) or @hidden = 'false'])]")).each do |f|
-        f["hidden"] = "true"
-      end
-      docxml.to_xml
-    end
-
-    # if there is a crossref to another document, with no anchor, retrieve the
-    # anchor given the locality, and insert it into the crossref
-    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
-      docid = bib&.at(ns("./docidentifier"))&.text
-      document_suffix = Asciidoctor::Standoc::Cleanup.to_ncname(docid)
-      docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
-        if loc = e.at(ns(".//locality[@type = 'anchor']"))
-          ref = loc.at(ns("./referenceFrom")) || next
-          anchor = "#{ref.text}_#{document_suffix}"
-          next unless @files[docid][:anchors].inject([]) { |m, (_, x)| m+= x.values }.include?(anchor)
-          ref.content = anchor
-        else
-          ins = e.at(ns("./localityStack")) || next
-          type = ins&.at(ns("./locality/@type"))&.text
-          ref = ins&.at(ns("./locality/referenceFrom"))&.text
-          (anchor = @files[docid][:anchors][type][ref]) || next
-          ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
-          ref_from.content = anchor.sub(/^_/, "")
-          locality = Nokogiri::XML::Node.new "locality", bib
-          locality[:type] = "anchor"
-          locality.add_child ref_from
-          ins << locality
-        end
-      end
-    end
-
-    # process each file in the collection
-    # files are held in memory, and altered as postprocessing
-    def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      @files.each do |identifier, x|
-        file, filename = targetfile(x, true)
-        file = update_xrefs(file, identifier)
-        Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
-          f.write(file)
-          f.close
-          # warn "metanorma compile -x html #{f.path}"
-          c = Compile.new
-          options = { format: :asciidoc, extension_keys: @format }.merge @compile_options
-          c.compile f.path, options
-          @files[identifier][:outputs] = {}
-          @format.each do |e|
-            ext = c.processor.output_formats[e]
-            fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
-            FileUtils.mv f.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
-            @files[identifier][:outputs][e] = File.join(@outdir, fn)
-          end
-        end
-      end
-    end
-
     private
 
     # @param options [Hash]
@@ -370,7 +184,6 @@ module Metanorma
         raise ArgumentError, "Need to specify formats (xml,html,pdf,doc)"
       end
       return if !options[:format].include?(:html) || options[:coverpage]
-
       raise ArgumentError, "Need to specify a coverpage to render HTML"
     end
   end

data/lib/metanorma/compile.rb

@@ -1,6 +1,7 @@
 require "fileutils"
 require "nokogiri"
 require "htmlentities"
+require "yaml"
 
 require "fontist"
 require "fontist/manifest/install"
@@ -251,6 +252,10 @@ module Metanorma
         isodoc_options[:datauriimage] = true if options[:datauriimage]
         file_extension = @processor.output_formats[ext]
         outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
+        if ext == :pdf
+          font_locations = fontist_font_locations(options)
+          isodoc_options[:mn2pdf] = { font_manifest_file: font_locations.path } if font_locations
+        end
         if ext == :rxl
           options[:relaton] = outfilename
           relaton_export(isodoc, options)
@@ -264,62 +269,105 @@ module Metanorma
             puts e.backtrace.join("\n")
           end
         end
+        if ext == :pdf
+          # font_locations.unlink
+        end
         wrap_html(options, file_extension, outfilename)
       end
     end
 
     def install_fonts(options)
-      if options[:"no-install-fonts"]
+      if options[:no_install_fonts]
         Util.log("[fontist] Skip font installation because" \
           " --no-install-fonts argument passed", :debug)
         return
       end
 
-      if !@processor.respond_to?(:fonts_manifest) || @processor.fonts_manifest.nil?
+      if missing_fontist_manifest?
         Util.log("[fontist] Skip font installation because font_manifest is missing", :debug)
         return
       end
 
+      @updated_formulas_repo = false
+
       manifest = @processor.fonts_manifest
-      agree_to_terms = options[:"agree-to-terms"] || false
-      continue_without_fonts = options[:"continue-without-fonts"] || false
+      agree_to_terms = options[:agree_to_terms] || false
+      continue_without_fonts = options[:continue_without_fonts] || false
+      no_progress = options[:no_progress] || false
 
-      install_fonts_safe(manifest, agree_to_terms, continue_without_fonts)
+      install_fonts_safe(manifest, agree_to_terms, continue_without_fonts, no_progress)
     end
 
     private
 
-    def install_fonts_safe(manifest, agree, continue)
-      fontist_install(manifest, agree)
+    def install_fonts_safe(manifest, agree, continue, no_progress)
+      fontist_install(manifest, agree, no_progress)
     rescue Fontist::Errors::LicensingError
       if continue
-        Util.log("[fontist] Processing will continue without fonts installed", :debug)
+        Util.log(
+          "[fontist] Processing will continue without fonts installed",
+          :debug
+        )
       else
         Util.log("[fontist] Aborting without proper fonts installed," \
           " make sure that you have set option --agree-to-terms", :fatal)
       end
     rescue Fontist::Errors::FontError => e
+      log_level = continue ? :warning : :fatal
       Util.log("[fontist] '#{e.font}' font is not supported. " \
-        "Please report this issue at github.com/metanorma/metanorma-#{@processor.short}/issues" \
-        " to report this issue.", :info)
+        "Please report this issue at github.com/metanorma/metanorma-"\
+        "#{@processor.short}/issues to report this issue.", log_level)
     rescue Fontist::Errors::FormulaIndexNotFoundError
+      if @updated_formulas_repo
+        Util.log(
+          "[fontist] Bug: formula index not found after 'fontist update'",
+          :fatal
+        )
+      end
       Util.log("[fontist] Missing formula index. Fetching it...", :debug)
       Fontist::Formula.update_formulas_repo
-      fontist_install(manifest, agree)
+      @updated_formulas_repo = true
+      install_fonts_safe(manifest, agree, continue, no_progress)
     end
 
-    def fontist_install(manifest, agree)
+    def fontist_install(manifest, agree, no_progress)
       Fontist::Manifest::Install.from_hash(
         manifest,
-        confirmation: agree ? "yes" : "no"
+        confirmation: agree ? "yes" : "no",
+        no_progress: no_progress
       )
     end
 
+    def fontist_font_locations(options)
+      return nil if missing_fontist_manifest? || options[:no_install_fonts]
+
+      dump_fontist_manifest_locations(@processor.fonts_manifest)
+    rescue Fontist::Errors::FormulaIndexNotFoundError
+      raise unless options[:continue_without_fonts]
+
+      nil
+    end
+
+    def dump_fontist_manifest_locations(manifest)
+      location_manifest = Fontist::Manifest::Locations.from_hash(
+        manifest
+      )
+      location_manifest_file = Tempfile.new(["fontist_locations", ".yml"])
+      location_manifest_file.write location_manifest.to_yaml
+      location_manifest_file.flush
+
+      location_manifest_file
+    end
+
+    def missing_fontist_manifest?
+      !@processor.respond_to?(:fonts_manifest) || @processor.fonts_manifest.nil?
+    end
+
     # @param options [Hash]
     # @return [String]
     def change_output_dir(options)
-      if options[:"output-dir"]
-        File.join options[:"output-dir"], File.basename(options[:filename])
+      if options[:output_dir]
+        File.join options[:output_dir], File.basename(options[:filename])
       else options[:filename]
       end
     end

data/lib/metanorma/input/asciidoc.rb

@@ -14,7 +14,7 @@ module Metanorma
           header_footer: true,
           attributes: [
             "nodoc", "stem", "xrefstyle=short", "docfile=#{filename}",
-            "output_dir=#{options[:"output-dir"]}"
+            "output_dir=#{options[:output_dir]}"
           ]
         }
         unless asciidoctor_validate(file, filename, out_opts)
@@ -70,11 +70,13 @@ module Metanorma
         /\n:title-font: (?<titlefont>[^\n]+)\n/ =~ headerextract
         /\n:i18nyaml: (?<i18nyaml>[^\n]+)\n/ =~ headerextract
         /\n:htmlstylesheet: (?<htmlstylesheet>[^\n]+)\n/ =~ headerextract
+        /\n:htmlstylesheet-override: (?<htmlstylesheet_override>[^\n]+)\n/ =~ headerextract
         /\n:htmlcoverpage: (?<htmlcoverpage>[^\n]+)\n/ =~ headerextract
         /\n:htmlintropage: (?<htmlintropage>[^\n]+)\n/ =~ headerextract
         /\n:scripts: (?<scripts>[^\n]+)\n/ =~ headerextract
         /\n:scripts-pdf: (?<scripts_pdf>[^\n]+)\n/ =~ headerextract
         /\n:wordstylesheet: (?<wordstylesheet>[^\n]+)\n/ =~ headerextract
+        /\n:wordstylesheet-override: (?<wordstylesheet_override>[^\n]+)\n/ =~ headerextract
         /\n:standardstylesheet: (?<standardstylesheet>[^\n]+)\n/ =~ headerextract
         /\n:header: (?<header>[^\n]+)\n/ =~ headerextract
         /\n:wordcoverpage: (?<wordcoverpage>[^\n]+)\n/ =~ headerextract
@@ -102,11 +104,13 @@ module Metanorma
           titlefont: defined?(titlefont) ? titlefont : nil,
           i18nyaml: defined?(i18nyaml) ? i18nyaml : nil,
           htmlstylesheet: defined?(htmlstylesheet) ? htmlstylesheet : nil,
+          htmlstylesheet_override: defined?(htmlstylesheet_override) ? htmlstylesheet_override : nil,
           htmlcoverpage: defined?(htmlcoverpage) ? htmlcoverpage : nil,
           htmlintropage: defined?(htmlintropage) ? htmlintropage : nil,
           scripts: defined?(scripts) ? scripts : nil,
           scripts_pdf: defined?(scripts_pdf) ? scripts_pdf : nil,
           wordstylesheet: defined?(wordstylesheet) ? wordstylesheet : nil,
+          wordstylesheet_override: defined?(wordstylesheet_override) ? wordstylesheet_override : nil,
           standardstylesheet: defined?(standardstylesheet) ? standardstylesheet : nil,
           header: defined?(header) ? header : nil,
           wordcoverpage: defined?(wordcoverpage) ? wordcoverpage : nil,

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.2.4"
+  VERSION = "1.2.8"
 end

data/metanorma.gemspec

@@ -27,19 +27,20 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'htmlentities'
   spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'mn2pdf', "~> 1"
+  spec.add_runtime_dependency 'metanorma-utils', "~> 1.1.0"
   spec.add_runtime_dependency 'pry'
   spec.add_runtime_dependency 'fontist', '~> 1.8'
 
-# get relaton-cli to avoic circular reference with metanorma-standoc
+# get relaton-cli to avoid circular reference with metanorma-standoc
   #spec.add_dependency "relaton-cli"
-  #spec.add_dependency "metanorma-standoc", "~> 1.5.3"
+  #spec.add_dependency "metanorma-standoc"
 
-  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "byebug", "~> 10.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.6.0"
+  spec.add_development_dependency "metanorma-iso", "~> 1.7.0"
   spec.add_development_dependency "sassc", "~> 2.4.0"
-  #spec.add_development_dependency "isodoc", "~> 1.2.1"
+  spec.add_development_dependency "rubocop", "~> 1.5.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.2.8
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-14 00:00:00.000000000 Z
+date: 2021-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: metanorma-utils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: sassc
   requirement: !ruby/object:Gem::Requirement
@@ -192,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.4.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.2
 description: Library to process any Metanorma standard.
 email:
 - open.source@ribose.com
@@ -224,6 +252,7 @@ files:
 - lib/metanorma/asciidoctor_extensions.rb
 - lib/metanorma/asciidoctor_extensions/glob_include_processor.rb
 - lib/metanorma/collection.rb
+- lib/metanorma/collection_fileprocess.rb
 - lib/metanorma/collection_manifest.rb
 - lib/metanorma/collection_renderer.rb
 - lib/metanorma/compile.rb
@@ -261,7 +290,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Metanorma is the standard of standards; the metanorma gem allows you to create