metanorma 1.0.6 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b51503237da3dfd54719591c47f599bca877d414528c4cf9a69abdb19aec9e9
-  data.tar.gz: 1588206551663889040f94194789cee7f2fb9b095775591a08f7b613ce59fbfe
+  metadata.gz: 72a6cdf0d2785bbc70cc907a024ab4e03ce4bdf5cd744c73ccc24ee0e928e092
+  data.tar.gz: bae5a06c2b0b71ecc46ed52699607eb235aa05d327f474328cce113510d4616e
 SHA512:
-  metadata.gz: 6f7dd77c9a6bf129594e3bfdabc1626bcb4c58a9b68454b09e1d216f7c9e43d4ebce62ce980ed762e0ad3e9f5c3c3456b9473fe7b1670e13bdc7c2d286edc1ec
-  data.tar.gz: 9c4125021226bdd561436e37e2fbacd4bd3c3bfa48845607282dd7ce3b4789e218e2aaa835364de7268c684bbf667e5267a6224db5a4c268fce180136b835391
+  metadata.gz: 78f0012cc133961e454a58e28a67c47a527a766ef85d8e995b7495f5095ec149bc7b89849ca565c8b668487e43ef90d1cc2d0c00658deee546d15295280352cd
+  data.tar.gz: 48a352c90ab1d5b4c2cd8e0e8581e58cd8425399d34cde1ba9ceafb333f36d2a6c048f59f739057840ce5bb6c7a0db2c5973dc26113c4d8102daf7300b806605

data/.github/workflows/macos.yml

@@ -29,18 +29,10 @@ jobs:
         uses: actions/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
       - name: Update gems
         run: |
           sudo gem install bundler --force
           bundle install --jobs 4 --retry 3
-      - name: Use Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12'
-      - name: Install Puppeteer
-        run: |
-          npm install -g puppeteer@3.0.1
       - name: Run specs
         run: |
           bundle exec rake

data/.github/workflows/ubuntu.yml

@@ -5,6 +5,8 @@ name: ubuntu
 on:
   push:
     branches: [ master ]
+    tags:
+      - '*'
   pull_request:
     paths-ignore:
       - .github/workflows/macos.yml
@@ -29,32 +31,26 @@ jobs:
         uses: actions/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
       - name: Update gems
         run: |
           gem install bundler
           bundle install --jobs 4 --retry 3
-      - name: Use Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12'
-      - name: Install Puppeteer
-        run: |
-          sudo apt-get update
-          sudo apt-get install libgbm1
-          npm install -g puppeteer@3.0.1
       - name: Run specs
         run: |
           bundle exec rake
-      - name: Trigger dependent repositories
-        if: github.ref == 'refs/heads/master' && matrix.ruby == '2.6'
+      - name: Trigger repositories
+        if: matrix.ruby == '2.6'
         env:
-          GH_USERNAME: ${{ secrets.PAT_USERNAME }}
-          GH_ACCESS_TOKEN: ${{ secrets.PAT_TOKEN }}
+          GH_USERNAME: metanorma-ci
+          GH_ACCESS_TOKEN: ${{ secrets.METANORMA_CI_PAT_TOKEN }}
         run: |
           curl -LO --retry 3 https://raw.githubusercontent.com/metanorma/metanorma-build-scripts/master/trigger-gh-actions.sh
           [[ -f ".github/workflows/dependent_repos.env" ]] && source .github/workflows/dependent_repos.env
-          for repo in $DEPENDENT_REPOS
+          CLIENT_PAYLOAD=$(cat <<EOF
+          "{ "ref": "${GITHUB_REF}", "repo": "${GITHUB_REPOSITORY}" }"
+          EOF
+          )
+          for repo in $REPOS
           do
-            sh trigger-gh-actions.sh $ORGANISATION $repo $GH_USERNAME $GH_ACCESS_TOKEN $GITHUB_REPOSITORY
+            sh trigger-gh-actions.sh $ORGANISATION $repo $GH_USERNAME $GH_ACCESS_TOKEN $GITHUB_REPOSITORY "$CLIENT_PAYLOAD"
           done

data/.github/workflows/windows.yml

@@ -29,20 +29,12 @@ jobs:
         uses: actions/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}
-          architecture: 'x64'
       - name: Update gems
         shell: pwsh
         run: |
           gem install bundler
           bundle config --local path vendor/bundle
           bundle install --jobs 4 --retry 3
-      - name: Use Node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '12'
-      - name: Install Puppeteer
-        run: |
-          npm install -g puppeteer@3.0.1
       - name: Run specs
         run: |
           bundle exec rake

data/.gitignore

@@ -16,3 +16,7 @@
 
 # rspec failure tracking
 .rspec_status
+.rubocop-https---raw-githubusercontent-com-riboseinc-oss-guides-master-ci-rubocop-yml
+Gemfile.lock
+relaton/
+.vscode/

data/.rubocop.yml

@@ -4,7 +4,3 @@
 
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
-AllCops:
-  TargetRubyVersion: 2.3
-Rails:
-  Enabled: true

data/lib/metanorma.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "metanorma/version"
 require "asciidoctor"
 require "metanorma/util"
@@ -8,7 +10,10 @@ require "metanorma/registry"
 require "metanorma/processor"
 require "metanorma/asciidoctor_extensions"
 require "metanorma/compile"
+require "metanorma/collection"
+require "metanorma/collection_renderer"
+require "metanorma/document"
 
+# Metanorma module
 module Metanorma
-
 end

data/lib/metanorma/collection.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require "relaton"
+require "relaton/cli"
+require "metanorma/collection_manifest"
+
+module Metanorma
+  # Metanorma collection of documents
+  class Collection
+    # @return [String]
+    attr_reader :file
+
+    # @return [Array<String>] documents-inline to inject the XML into
+    #   the collection manifest; documents-external to keeps them outside
+    attr_reader :directives
+
+    # @return [Hash<String, Metanorma::Document>]
+    attr_reader :documents
+
+    # @param file [String] path to source file
+    # @param directives [Array<String>] documents-inline to inject the XML into
+    #   the collection manifest; documents-external to keeps them outside
+    # @param bibdata [RelatonBib::BibliographicItem]
+    # @param manifest [Metanorma::CollectionManifest]
+    # @param documents [Hash<String, Metanorma::Document>]
+    # @param prefatory [String]
+    # @param final [String]
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+    def initialize(**args)
+      @file = args[:file]
+      @directives = args[:directives] || []
+      @bibdata = args[:bibdata]
+      @manifest = args[:manifest]
+      @manifest.collection = self
+      @documents = args[:documents] || {}
+      if @documents.any? && !@directives.include?("documents-inline")
+        @directives << "documents-inline"
+      end
+      @documents.merge! @manifest.documents(File.dirname(@file))
+      @prefatory = args[:prefatory]
+      @final = args[:final]
+    end
+    # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
+
+    # @return [String] XML
+    def to_xml
+      Nokogiri::XML::Builder.new do |xml|
+        xml.send("metanorma-collection",
+                 "xmlns" => "http://metanorma.org") do |mc|
+          @bibdata.to_xml mc, bibdata: true, date_format: :full
+          @manifest.to_xml mc
+          content_to_xml "prefatory", mc
+          doccontainer mc
+          content_to_xml "final", mc
+        end
+      end.to_xml
+    end
+
+    def render(opts)
+      CollectionRenderer.render self, opts
+    end
+
+    class << self
+      # @param file [String]
+      # @return [RelatonBib::BibliographicItem,
+      #   RelatonIso::IsoBibliographicItem]
+      def parse(file)
+        case file
+        when /\.xml$/ then parse_xml(file)
+        when /.ya?ml$/ then parse_yaml(file)
+        end
+      end
+
+      private
+
+      def parse_xml(file)
+        xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
+        if (b = xml.at("/xmlns:metanorma-collection/xmlns:bibdata"))
+          bd = Relaton::Cli.parse_xml b
+        end
+        mnf_xml = xml.at("/xmlns:metanorma-collection/xmlns:manifest")
+        mnf = CollectionManifest.from_xml mnf_xml
+        pref = pref_final_content xml.at("//xmlns:prefatory-content")
+        fnl = pref_final_content xml.at("//xmlns:final-content")
+        new(file: file, bibdata: bd, manifest: mnf,
+            documents: docs_from_xml(xml, mnf), prefatory: pref, final: fnl)
+      end
+
+      def parse_yaml(file)
+        yaml = YAML.load_file file
+        if yaml["bibdata"]
+          bd = Relaton::Cli::YAMLConvertor.convert_single_file yaml["bibdata"]
+        end
+        mnf = CollectionManifest.from_yaml yaml["manifest"]
+        dirs = yaml["directives"]
+        pref = yaml["prefatory-content"]
+        fnl = yaml["final-content"]
+        new(file: file, directives: dirs, bibdata: bd, manifest: mnf,
+            prefatory: pref, final: fnl)
+      end
+
+      # @param xml [Nokogiri::XML::Document]
+      # @parma mnf [Metanorma::CollectionManifest]
+      # @return [Hash{String=>Metanorma::Document}]
+      def docs_from_xml(xml, mnf)
+        xml.xpath("//xmlns:doc-container/*/xmlns:bibdata")
+          .each_with_object({}) do |b, m|
+          bd = Relaton::Cli.parse_xml b
+          docref = mnf.docref_by_id bd.docidentifier.first.id
+          m[docref["identifier"]] = Document.new bd, docref["fileref"]
+          m
+        end
+      end
+
+      # @param xml [Nokogiri::XML::Element, nil]
+      # @return [String, nil]
+      def pref_final_content(xml)
+        return unless xml
+
+        <<~CONT
+
+          == #{xml.at('title')&.text}
+          #{xml.at('p')&.text}
+        CONT
+      end
+    end
+
+    private
+
+    # @return [String, nil]
+    attr_reader :prefatory, :final
+
+    # @return [String]
+    def dummy_header
+      <<~DUMMY
+        = X
+        A
+
+      DUMMY
+    end
+
+    # @param elm [String] 'prefatory' or 'final'
+    # @param builder [Nokogiri::XML::Builder]
+    def content_to_xml(elm, builder)
+      return unless (cnt = send(elm))
+
+      require "metanorma-#{doctype}"
+      out = sections(dummy_header + cnt)
+      builder.send(elm + "-content") { |b| b << out }
+    end
+
+    # @param cnt [String] prefatory/final content
+    # @return [String] XML
+    def sections(cnt)
+      c = Asciidoctor.convert(cnt, backend: doctype.to_sym, header_footer: true)
+      Nokogiri::XML(c).at("//xmlns:sections").children.to_xml
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def doccontainer(builder)
+      return unless Array(@directives).include? "documents-inline"
+
+      documents.each_with_index do |(_, d), i|
+        id = format("doc%<index>09d", index: i)
+        builder.send("doc-container", id: id) { |b| d.to_xml b }
+      end
+    end
+
+    # @return [String]
+    def doctype
+      @doctype ||= fetch_doctype || "standoc"
+    end
+
+    # @return [String]
+    def fetch_doctype
+      docid = @bibdata.docidentifier.first
+      return unless docid
+
+      docid.type&.downcase || docid.id&.sub(/\s.*$/, "")&.downcase
+    end
+  end
+end

data/lib/metanorma/collection_manifest.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module Metanorma
+  # Metanorma collection's manifest
+  class CollectionManifest
+    # @return [Metanorma::Collection]
+    attr_reader :collection
+
+    # @param level [String]
+    # @param title [String, nil]
+    # @param docref [Array<Hash{String=>String}>]
+    # @param manifest [Array<Metanorma::CollectionManifest>]
+    def initialize(level, title = nil, docref = [], manifest = [])
+      @level = level
+      @title = title
+      @docref = docref
+      @manifest = manifest
+    end
+
+    class << self
+      # @param mnf [Nokogiri::XML::Element]
+      # @return [Metanorma::CollectionManifest]
+      def from_yaml(mnf)
+        manifest = RelatonBib::HashConverter.array(mnf["manifest"]).map do |m|
+          from_yaml m
+        end
+        docref = RelatonBib::HashConverter.array mnf["docref"]
+        new(mnf["level"], mnf["title"], docref, manifest)
+      end
+
+      # @param mnf [Nokogiri::XML::Element]
+      # @return [Metanorma::CollectionManifest]
+      def from_xml(mnf)
+        level = mnf.at("level").text
+        title = mnf.at("title")&.text
+        manifest = mnf.xpath("xmlns:manifest").map { |m| from_xml(m) }
+        new(level, title, parse_docref(mnf), manifest)
+      end
+
+      private
+
+      # @param mnf [Nokogiri::XML::Element]
+      # @return [Hash{String=>String}]
+      def parse_docref(mnf)
+        mnf.xpath("xmlns:docref").map do |dr|
+          h = { "identifier" => dr.at("identifier").text }
+          h["fileref"] = dr[:fileref] if dr[:fileref]
+          h
+        end
+      end
+    end
+
+    # @param col [Metanorma::Collection]
+    def collection=(col)
+      @collection = col
+      @manifest.each { |mnf| mnf.collection = col }
+    end
+
+    # @param dir [String] path to coolection
+    # @return [Hash<String, Metanorma::Document>]
+    def documents(dir = "")
+      docs = @docref.each_with_object({}) do |dr, m|
+        next m unless dr["fileref"]
+
+        m[dr["identifier"]] = Document.parse_file File.join(dir, dr["fileref"])
+        m
+      end
+      @manifest.reduce(docs) do |mem, mnf|
+        mem.merge mnf.documents(dir)
+      end
+    end
+
+    # @param builder [Nokogiri::XML::Builder]
+    def to_xml(builder)
+      builder.manifest do |b|
+        b.level @level
+        b.title @title if @title
+        docref_to_xml b
+        @manifest.each { |m| m.to_xml b }
+      end
+    end
+
+    # @return [Array<Hash{String=>String}>]
+    def docrefs
+      return @docrefs if @docrefs
+
+      drfs = @docref.map { |dr| dr }
+      @manifest.reduce(drfs) { |mem, mnf| mem + mnf.docrefs }
+    end
+
+    def docref_by_id(docid)
+      refs = docrefs
+      dref = refs.detect { |k| k["identifier"] == docid }
+      dref || docrefs.detect { |k| /^#{k["identifier"]}/ =~ docid }
+    end
+
+    private
+
+    # @param builder [Nokogiri::XML::Builder]
+    def docref_to_xml(builder)
+      @docref.each do |dr|
+        drf = builder.docref { |b| b.identifier dr["identifier"] }
+        drf[:fileref] = dr["fileref"]
+        if collection.directives.include?("documents-inline")
+          id = collection.documents.find_index { |k, _| k == dr["identifier"] }
+          drf[:id] = format("doc%<index>09d", index: id)
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/collection_renderer.rb

@@ -0,0 +1,312 @@
+# frozen_string_literal: true
+
+require "isodoc"
+
+module Metanorma
+  # XML collection renderer
+  class CollectionRenderer
+    FORMATS = %i[html xml doc pdf].freeze
+
+    # This is only going to render the HTML collection
+    # @param xml [Metanorma::Collection] input XML collection
+    # @param folder [String] input folder
+    # @param options [Hash]
+    # @option options [String] :coverpage cover page HTML (Liquid template)
+    # @option options [Array<Symbol>] :format list of formats (xml,html,doc,pdf)
+    # @option options [String] :ourput_folder output directory
+    #
+    # We presuppose that the bibdata of the document is equivalent to that of
+    # the collection, and that the flavour gem can sensibly process it. We may
+    # need to enhance metadata in the flavour gems isodoc/metadata.rb with
+    # collection metadata
+    def initialize(xml, folder, options = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      check_options options
+      @xml = Nokogiri::XML xml # @xml is the collection manifest
+      @lang = @xml&.at(ns("//bibdata/language"))&.text || "en"
+      @script = @xml&.at(ns("//bibdata/script"))&.text || "Latn"
+      @doctype = doctype
+      require "metanorma-#{@doctype}"
+
+      # output processor for flavour
+      @isodoc = isodoc
+
+      @outdir = options[:output_folder]
+      @coverpage = options[:coverpage]
+      @format = options[:format]
+
+      # list of files in the collection
+      @files = read_files folder
+      FileUtils.rm_rf @outdir
+      FileUtils.mkdir_p @outdir
+    end
+
+    # @param col [Metanorma::Collection] XML collection
+    # @param options [Hash]
+    # @option options [String] :coverpage cover page HTML (Liquid template)
+    # @option options [Array<Synbol>] :format list of formats
+    # @option options [Strong] :ourput_folder output directory
+    def self.render(col, options = {})
+      folder = File.dirname col.file
+      cr = new(col.to_xml, folder, options)
+      cr.files
+      cr.coverpage if options[:format]&.include?(:html)
+    end
+
+    # Dummy class
+    class Dummy
+      def attr(_xyz); end
+    end
+
+    # The isodoc class for the metanorma flavour we are using
+    def isodoc # rubocop:disable Metrics/MethodLength
+      x = Asciidoctor.load nil, backend: @doctype.to_sym
+      isodoc = x.converter.html_converter(Dummy.new)
+      isodoc.i18n_init(@lang, @script) # read in internationalisation
+      # create the @meta class of isodoc, with "navigation" set to the index bar
+      # extracted from the manifest
+      nav = indexfile(@xml.at(ns("//manifest")))
+      i18n = isodoc.i18n
+      i18n.set(:navigation, nav)
+      isodoc.metadata_init(@lang, @script, i18n)
+      # populate the @meta class of isodoc with the various metadata fields
+      # native to the flavour; used to populate Liquid
+      isodoc.info(@xml, nil)
+      isodoc
+    end
+
+    # infer the flavour from the first document identifier; relaton does that
+    def doctype
+      if (docid = @xml&.at(ns("//bibdata/docidentifier/@type"))&.text)
+        dt = docid.downcase
+      elsif (docid = @xml&.at(ns("//bibdata/docidentifier"))&.text)
+        dt = docid.sub(/\s.*$/, "").lowercase
+      else return "standoc"
+      end
+      @registry = Metanorma::Registry.instance
+      @registry.alias(dt.to_sym)&.to_s || dt
+    end
+
+    def ns(xpath)
+      IsoDoc::Convert.new({}).ns(xpath)
+    end
+
+    # hash for each document in collection of document identifier to:
+    # document reference (fileref or id), type of document reference,
+    # and bibdata entry for that file
+    # @param path [String] path to collection
+    # @return [Hash{String=>Hash}]
+    def read_files(path) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      files = {}
+      @xml.xpath(ns("//docref")).each do |d|
+        identifier = d.at(ns("./identifier")).text
+        files[identifier] = if d["fileref"]
+                              { type: "fileref",
+                                ref: File.join(path, d["fileref"]) }
+                            else { type: "id", ref: d["id"] }
+                            end
+        file, _filename = targetfile(files[identifier], true)
+        xml = Nokogiri::XML(file)
+        files[identifier][:anchors] = read_anchors(xml)
+        files[identifier][:bibdata] = xml.at(ns("//bibdata"))
+      end
+      files
+    end
+
+    # map locality type and label (e.g. "clause" "1") to id = anchor for
+    # a document
+    def read_anchors(xml)
+      ret = {}
+      xrefs = @isodoc.xref_init(@lang, @script, @isodoc, @isodoc.i18n, {})
+      xrefs.parse xml
+      xrefs.get.each do |k, v|
+        v[:label] && v[:type] || next
+        ret[v[:type]] ||= {}
+        ret[v[:type]][v[:label]] = k
+      end
+      ret
+    end
+
+    # populate liquid template of ARGV[1] with metadata extracted from
+    # collection manifest
+    def coverpage
+      File.open(File.join(@outdir, "index.html"), "w:UTF-8") do |f|
+        f.write @isodoc.populate_template(File.read(@coverpage))
+      end
+    end
+
+    # @param elm [Nokogiri::XML::Element]
+    # @return [String]
+    def indexfile_title(elm)
+      lvl = elm&.at(ns("./level"))&.text&.capitalize
+      lbl = elm&.at(ns("./title"))&.text
+      "#{lvl}#{lvl && lbl ? ': ' : ''}#{lbl}"
+    end
+
+    # uses the identifier to label documents; other attributes (title) can be
+    # looked up in @files[id][:bibdata]
+    #
+    # @param elm [Nokogiri::XML::Element]
+    # @param builder [Nokogiri::XML::Builder]
+    def indexfile_docref(elm, builder)
+      return "" unless elm.at(ns("./docref"))
+
+      builder.ul { |b| docrefs(elm, b) }
+    end
+
+    # @param elm [Nokogiri::XML::Element]
+    # @param builder [Nokogiri::XML::Builder]
+    def docrefs(elm, builder)
+      elm.xpath(ns("./docref")).each do |d|
+        identifier = d.at(ns("./identifier")).text
+        link = if d["fileref"] then d["fileref"].sub(/\.xml$/, ".html")
+               else d["id"] + ".html"
+               end
+        builder.li { builder.a identifier, href: link }
+      end
+    end
+
+    # single level navigation list, with hierarchical nesting
+    # if multiple lists are needed as separate HTML fragments, multiple
+    # instances of this function will be needed,
+    # and associated to different variables in the call to @isodoc.metadata_init
+    # (including possibly an array of HTML fragments)
+    #
+    # @param elm [Nokogiri::XML::Element]
+    # @return [String] XML
+    def indexfile(elm)
+      Nokogiri::HTML::Builder.new do |b|
+        b.ul do
+          b.li indexfile_title(elm)
+          indexfile_docref(elm, b)
+          elm.xpath(ns("./manifest")).each do |d|
+            b << indexfile(d)
+          end
+        end
+      end.doc.root.to_html
+    end
+
+    # return file contents + output filename for each file in the collection,
+    # given a docref entry
+    # @param data [Hash]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def targetfile(data, read = false)
+      if data[:type] == "fileref" then ref_file data[:ref], read
+      else xml_file data[:id], read
+      end
+    end
+
+    # @param ref [String]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def ref_file(ref, read)
+      file = File.read(ref, encoding: "utf-8") if read
+      filename = ref.sub(/\.xml$/, ".html")
+      [file, filename]
+    end
+
+    # @param id [String]
+    # @param read [Boolean]
+    # @return [Array<String, nil>]
+    def xml_file(id, read)
+      file = @xml.at(ns("//doc-container[@id = '#{id}']")).to_xml if read
+      filename = id + ".html"
+      [file, filename]
+    end
+
+    # @param bib [Nokogiri::XML::Element]
+    # @param identifier [String]
+    def update_bibitem(bib, identifier) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      docid = bib&.at(ns("./docidentifier"))&.text
+      unless @files[docid]
+        warn "Cannot find crossreference to document #{docid} in document "\
+        "#{identifier}!"
+        abort
+      end
+      id = bib["id"]
+      newbib = bib.replace(@files[docid][:bibdata])
+      newbib.name = "bibitem"
+      newbib["id"] = id
+      newbib&.at(ns("./ext"))&.remove
+      _file, url = targetfile(@files[docid], false)
+      uri_node = Nokogiri::XML::Node.new "uri", newbib
+      uri_node[:type] = "citation"
+      uri_node.content = url
+      newbib.at(ns("./docidentifier")).previous = uri_node
+    end
+
+    # TODO: update crossreferences to other files in the selection
+    # repo(current-metanorma-collection/ISO 17301-1:2016)
+    # replaced by
+    # bibdata of "ISO 17301-1:2016" in situ as bibitem
+    # Any erefs to that bibitem id are replaced with relative URL
+    # Preferably with anchor, and is a job to realise dynamic lookup of
+    # localities
+    # @param file [String] XML content
+    # @param identifier [String] docid
+    # @return [String] XML content
+    def update_xrefs(file, identifier)
+      docxml = Nokogiri::XML(file)
+      docxml.xpath(ns("//bibitem[not(ancestor::bibitem)]")).each do |b|
+        docid = b&.at(ns("./docidentifier[@type = 'repository']"))&.text
+        next unless docid && %r{^current-metanorma-collection/}.match(docid)
+
+        update_bibitem(b, identifier)
+        update_anchors(b, docxml, docid)
+      end
+      docxml.to_xml
+    end
+
+    # if there is a crossref to another document, with no anchor, retrieve the
+    # anchor given the locality, and insert it into the crossref
+    def update_anchors(bib, docxml, _id) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      docid = bib&.at(ns("./docidentifier"))&.text
+      docxml.xpath("//xmlns:eref[@citeas = '#{docid}']").each do |e|
+        e.at(ns(".//locality[@type = 'anchor']")).nil? || next
+        ins = e.at(ns("./localityStack")) || next
+        type = ins&.at(ns("./locality/@type"))&.text
+        ref = ins&.at(ns("./locality/referenceFrom"))&.text
+        (anchor = @files[docid][:anchors][type][ref]) || next
+        ref_from = Nokogiri::XML::Node.new "referenceFrom", bib
+        ref_from.content = anchor.sub(/^_/, "")
+        locality = Nokogiri::XML::Node.new "locality", bib
+        locality[:type] = "anchor"
+        locality.add_child ref_from
+        ins << locality
+      end
+    end
+
+    # process each file in the collection
+    # files are held in memory, and altered as postprocessing
+    def files # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      @files.each do |identifier, x|
+        file, filename = targetfile(x, true)
+        file = update_xrefs(file, identifier)
+        Tempfile.open(["collection", ".xml"], encoding: "utf-8") do |f|
+          f.write(file)
+          f.close
+          # warn "metanorma compile -x html #{f.path}"
+          c = Compile.new
+          c.compile f.path, format: :asciidoc, extension_keys: @format
+          @format.each do |ext|
+            fn = File.basename(filename).sub(/(?<=\.)[^\.]+$/, ext.to_s)
+            FileUtils.mv f.path.sub(/\.xml$/, ".#{ext}"), File.join(@outdir, fn)
+          end
+        end
+      end
+    end
+
+    private
+
+    # @param options [Hash]
+    # @raise [ArgumentError]
+    def check_options(options)
+      unless options[:format].is_a?(Array) && (FORMATS & options[:format]).any?
+        raise ArgumentError, "Need to specify formats (xml,html,pdf,doc)"
+      end
+      return if !options[:format].include?(:html) || options[:coverpage]
+
+      raise ArgumentError, "Need to specify a coverpage to render HTML"
+    end
+  end
+end

data/lib/metanorma/compile.rb

@@ -4,14 +4,18 @@ require "htmlentities"
 
 module Metanorma
   class Compile
+    # @return [Array<String>]
+    attr_reader :errors
+
     def initialize
       @registry = Metanorma::Registry.instance
+      @errors = []
     end
 
     def compile(filename, options = {})
       require_libraries(options)
       options = options_extract(filename, options)
-      validate(options) or return nil
+      validate_type(options) && validate_format(options) || (return nil)
       @processor = @registry.find_processor(options[:type].to_sym)
       extensions = get_extensions(options) or return nil
       (file, isodoc = process_input(filename, options)) or return nil
@@ -43,6 +47,7 @@ module Metanorma
       o = Metanorma::Input::Asciidoc.new.extract_metanorma_options(content)
       o = o.merge(xml_options_extract(content))
       options[:type] ||= o[:type]&.to_sym
+      t = @registry.alias(options[:type]) and options[:type] = t
       dir = filename.sub(%r(/[^/]+$), "/")
       options[:relaton] ||= "#{dir}/#{o[:relaton]}" if o[:relaton]
       options[:sourcecode] ||= "#{dir}/#{o[:sourcecode]}" if o[:sourcecode]
@@ -53,10 +58,6 @@ module Metanorma
       options
     end
 
-    def validate(options)
-      validate_type(options) && validate_format(options)
-    end
-
     def validate_type(options)
       unless options[:type]
         Util.log("[metanorma] Error: Please specify a standard type: #{@registry.supported_backends}.", :error)
@@ -89,14 +90,22 @@ module Metanorma
     end
 
     def get_extensions(options)
-      options[:extension_keys] ||= @processor.output_formats.inject([]) do |memo, (k, _)|
-        memo << k; memo
+      options[:extension_keys] ||= @processor.output_formats.reduce([]) do |memo, (k, _)|
+        memo << k
       end
-      extensions = options[:extension_keys].inject([]) do |memo, e|
-        @processor.output_formats[e] and memo << e or
-          Util.log("[metanorma] Error: #{e} format is not supported for this standard.", :error)
-        memo
+      extensions = options[:extension_keys].reduce([]) do |memo, e|
+        if @processor.output_formats[e]
+          memo << e
+        else
+          message = "[metanorma] Error: #{e} format is not supported for this standard."
+          @errors << message
+          Util.log(message, :error)
+          memo
+        end
       end
+      if !extensions.include?(:presentation) and extensions.any? { |e| @processor.use_presentation_xml(e) }
+        extensions << :presentation
+        end
       extensions
     end
 
@@ -110,7 +119,7 @@ module Metanorma
         dir = File.dirname(filename)
         dir != '.' and
           file.gsub!(/^include::/, "include::#{dir}/")
-        [file, @processor.input_to_isodoc(file, filename)]
+        [file, @processor.input_to_isodoc(file, filename, options)]
       when ".xml"
         Util.log("[metanorma] Processing: Metanorma XML input.", :info)
         # TODO NN: this is a hack -- we should provide/bridge the
@@ -195,28 +204,62 @@ module Metanorma
       end
     end
 
+    # dependency ordering
+    def sort_extensions_execution(ext)
+       case ext
+       when :xml then 0
+       when :rxl then 1
+       when :presentation then 2
+       else
+         99
+       end
+    end
+
+    def wrap_html(options, file_extension, outfilename)
+      if options[:wrapper] and /html$/.match file_extension
+          outfilename = outfilename.sub(/\.html$/, "")
+          FileUtils.mkdir_p outfilename
+          FileUtils.mv "#{outfilename}.html", outfilename
+          FileUtils.mv "#{outfilename}_images", outfilename, force: true
+        end
+    end
+
+    # isodoc is Raw Metanorma XML
     def process_extensions(extensions, file, isodoc, options)
-      extensions.each do |ext|
+      f = change_output_dir options
+      xml_name = f.sub(/\.[^.]+$/, ".xml")
+      presentationxml_name = f.sub(/\.[^.]+$/, ".presentation.xml")
+      extensions.sort do |a, b|
+        sort_extensions_execution(a) <=> sort_extensions_execution(b)
+      end.each do |ext|
         isodoc_options = @processor.extract_options(file)
         isodoc_options[:datauriimage] = true if options[:datauriimage]
         file_extension = @processor.output_formats[ext]
-        outfilename = options[:filename].sub(/\.[^.]+$/, ".#{file_extension}")
+        outfilename = f.sub(/\.[^.]+$/, ".#{file_extension}")
         if ext == :rxl
           options[:relaton] = outfilename
           relaton_export(isodoc, options)
         else
           begin
-          @processor.output(isodoc, outfilename, ext, isodoc_options)
-          rescue StandardError => e  
+            @processor.use_presentation_xml(ext) ?
+              @processor.output(nil, presentationxml_name, outfilename, ext, isodoc_options) :
+              @processor.output(isodoc, xml_name, outfilename, ext, isodoc_options)
+          rescue StandardError => e
             puts e.message
           end
         end
-        if options[:wrapper] and /html$/.match file_extension
-          outfilename = outfilename.sub(/\.html$/, "")
-          FileUtils.mkdir_p outfilename
-          FileUtils.mv "#{outfilename}.html", outfilename
-          FileUtils.mv "#{outfilename}_images", outfilename, force: true
-        end
+        wrap_html(options, file_extension, outfilename)
+      end
+    end
+
+    private
+
+    # @param options [Hash]
+    # @return [String]
+    def change_output_dir(options)
+      if options[:"output-dir"]
+        File.join options[:"output-dir"], File.basename(options[:filename])
+      else options[:filename]
       end
     end
   end

data/lib/metanorma/document.rb

@@ -1,12 +1,82 @@
 module Metanorma
   class Document
+    # @return [Strin]
+    attr_reader :file
 
-    def initialize
-      @input_file
-      @input_format
-      @document_type
-      @output_formats
+    # @param bibitem [RelatonBib::BibliographicItem]
+    def initialize(bibitem, file)
+      @bibitem = bibitem
+      @file = file
     end
 
+    class << self
+      # @param file [String] file path
+      # @return [Metanorma::Document]
+      def parse_file(file)
+        new bibitem(file), file
+      end
+
+      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
+      # @return [Metanorma::Document]
+      def parse_xml(xml)
+        new from_xml(xml)
+      end
+
+      private
+
+      # #param xml [Nokogiri::XML::Document, Nokogiri::XML::Element]
+      # @return [RelatonBib::BibliographicItem,RelatonIso::IsoBibliographicItem]
+      def from_xml(xml)
+        Relaton::Cli.parse_xml xml.at("//xmlns:bibitem|//xmlns:bibdata")
+      end
+
+      # @param file [String]
+      # @return [Symbol] file type
+      def format(file)
+        case file
+        when /\.xml$/ then :xml
+        when /.ya?ml$/ then :yaml
+        end
+      end
+
+      # @param file [String]
+      # @return [RelatonBib::BibliographicItem,
+      #   RelatonIso::IsoBibliographicItem]
+      def bibitem(file)
+        case format(file)
+        when :xml
+          from_xml Nokogiri::XML(File.read(file, encoding: "UTF-8"))
+        when :yaml
+          yaml = File.read(file, ecoding: "UTF-8")
+          Relaton::Cli::YAMLConvertor.convert_single_file(yaml)
+        end
+      end
+    end
+
+    # @param builder [Nokogiri::XML::Builder, nil]
+    # @return [Nokogiri::XML::Builder, String]
+    def to_xml(builder = nil)
+      if builder
+        render_xml builder
+      else
+        Nokogiri::XML::Builder.new do |b|
+          root = render_xml b
+          root["xmlns"] = "http://metanorma.org"
+        end.to_xml
+      end
+    end
+
+    # @return [String]
+    def type
+      @type ||= (@bibitem.docidentifier.first&.type ||
+        @bibitem.docidentifier.first&.id&.match(/^[^\s]+/)&.to_s)&.downcase ||
+        "standoc"
+    end
+
+    private
+
+    def render_xml(builder)
+      builder.send(type + "-standard") { |b| @bibitem.to_xml b, bibdata: true }
+    end
   end
 end

data/lib/metanorma/input/asciidoc.rb

@@ -5,7 +5,7 @@ module Metanorma
 
     class Asciidoc < Base
 
-      def process(file, filename, type)
+      def process(file, filename, type, options = {})
         require "asciidoctor"
         ::Asciidoctor.convert(
           file,
@@ -13,7 +13,10 @@ module Metanorma
           safe: :safe,
           backend: type,
           header_footer: true,
-          attributes: ["nodoc", "stem", "xrefstyle=short", "docfile=#{filename}"]
+          attributes: [
+            "nodoc", "stem", "xrefstyle=short", "docfile=#{filename}",
+            "output_dir=#{options[:"output-dir"]}"
+          ],
         )
       end
 

data/lib/metanorma/output/xslfo.rb

@@ -7,7 +7,13 @@ module Metanorma
       def convert(url_path, output_path, xsl_stylesheet)
         return if url_path.nil? || output_path.nil? || xsl_stylesheet.nil?
 
-        Mn2pdf.convert(url_path, output_path, xsl_stylesheet)
+        Mn2pdf.convert(quote(url_path), quote(output_path), quote(xsl_stylesheet))
+      end
+
+      def quote(x)
+        return x if /^'.*'$/.match(x)
+        return x if /^".*"$/.match(x)
+        %("#{x}")
       end
     end
   end

data/lib/metanorma/processor.rb

@@ -15,15 +15,28 @@ module Metanorma
     def output_formats
       {
         xml: "xml",
+        presentation: "presentation.xml",
         rxl: "rxl"
       }
     end
 
-    def input_to_isodoc(file, filename)
-      raise "This is an abstract class!"
+    def input_to_isodoc(file, filename, options = {})
+      Metanorma::Input::Asciidoc.new.process(file, filename, @asciidoctor_backend, options)
+    end
+
+    # def input_to_isodoc(file, filename)
+    #   raise "This is an abstract class!"
+    # end
+
+    def use_presentation_xml(ext)
+      case ext
+      when :html, :doc, :pdf then true
+      else
+        false
+      end
     end
 
-    def output(isodoc_node, outname, format, options={})
+    def output(isodoc_node, inname, outname, format, options={})
       File.open(outname, "w:UTF-8") { |f| f.write(isodoc_node) }
     end
 

data/lib/metanorma/registry.rb

@@ -14,14 +14,23 @@ module Metanorma
 
     def initialize
       @processors = {}
+      @aliases = {csd: :cc, m3d: :m3aawg, mpfd: :mpfa, csand: :csa}
+    end
+
+    def alias(x)
+      @aliases[x]
     end
 
     def register processor
       raise Error unless processor < ::Metanorma::Processor
       p = processor.new
-      Array(p.short).each do |s|
-        @processors[s] = p
+      # p.short[-1] is the canonical name
+      short = Array(p.short)
+      @processors[short[-1]] = p
+      short.each do |s|
+        @aliases[s] = short[-1]
       end
+      Array(p.short)
       Util.log("[metanorma] processor \"#{Array(p.short)[0]}\" registered", :info)
     end
 

data/lib/metanorma/version.rb

@@ -1,3 +1,3 @@
 module Metanorma
-  VERSION = "1.0.6"
+  VERSION = "1.1.4"
 end

data/metanorma.gemspec

@@ -27,11 +27,13 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'htmlentities'
   spec.add_runtime_dependency 'nokogiri'
   spec.add_runtime_dependency 'mn2pdf', "~> 1"
+  spec.add_dependency "relaton-cli", "~> 1.3.0"
 
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "byebug", "~> 10.0"
   spec.add_development_dependency "rspec-command", "~> 1.0"
   spec.add_development_dependency "equivalent-xml", "~> 0.6"
-  spec.add_development_dependency "metanorma-iso", "~> 1.3"
+  spec.add_development_dependency "metanorma-iso", "~> 1.5"
+  spec.add_development_dependency "isodoc", "~> 1.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metanorma
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.1.4
 platform: ruby
 authors:
 - Ribose Inc.
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: relaton-cli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -142,14 +156,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: isodoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 1.2.1
 description: Library to process any Metanorma standard.
 email:
 - open.source@ribose.com
@@ -182,6 +210,9 @@ files:
 - lib/metanorma.rb
 - lib/metanorma/asciidoctor_extensions.rb
 - lib/metanorma/asciidoctor_extensions/glob_include_processor.rb
+- lib/metanorma/collection.rb
+- lib/metanorma/collection_manifest.rb
+- lib/metanorma/collection_renderer.rb
 - lib/metanorma/compile.rb
 - lib/metanorma/config.rb
 - lib/metanorma/document.rb
@@ -202,7 +233,7 @@ homepage: https://github.com/metanorma/metanorma
 licenses:
 - BSD-2-Clause
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -217,9 +248,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.0.3
+signing_key:
 specification_version: 4
 summary: Metanorma is the standard of standards; the metanorma gem allows you to create
   any standard document type supported by Metanorma.