metanorma-iso 1.10.6 → 2.0.0

data/lib/metanorma/iso/cleanup.rb

@@ -0,0 +1,176 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      PRE_NORMREF_FOOTNOTES = "//preface//fn | "\
+                              "//clause[@type = 'scope']//fn".freeze
+
+      NORMREF_FOOTNOTES =
+        "//references[@normative = 'true']//fn".freeze
+
+      POST_NORMREF_FOOTNOTES =
+        "//sections//clause[not(@type = 'scope')]//fn | "\
+        "//annex//fn | "\
+        "//references[@normative = 'false']//fn".freeze
+
+      def other_footnote_renumber(xmldoc)
+        seen = {}
+        i = 0
+        [PRE_NORMREF_FOOTNOTES, NORMREF_FOOTNOTES,
+         POST_NORMREF_FOOTNOTES].each do |xpath|
+          xmldoc.xpath(xpath).each do |fn|
+            i, seen = other_footnote_renumber1(fn, i, seen)
+          end
+        end
+      end
+
+      def id_prefix(prefix, id)
+        # we're just inheriting the prefixes from parent doc
+        return id.text if @amd
+
+        prefix.join("/") + (id.text.match?(%{^/}) ? "" : " ") + id.text
+      end
+
+      def get_id_prefix(xmldoc)
+        prefix = []
+        xmldoc.xpath("//bibdata/contributor[role/@type = 'publisher']"\
+                     "/organization").each do |x|
+          x1 = x.at("abbreviation")&.text || x.at("name")&.text
+          (x1 == "ISO" and prefix.unshift("ISO")) or prefix << x1
+        end
+        prefix
+      end
+
+      # ISO as a prefix goes first
+      def docidentifier_cleanup(xmldoc)
+        prefix = get_id_prefix(xmldoc)
+        id = xmldoc.at("//bibdata/docidentifier[@type = 'ISO']") or return
+        id.content = id_prefix(prefix, id)
+        id = xmldoc.at("//bibdata/ext/structuredidentifier/project-number") and
+          id.content = id_prefix(prefix, id)
+        id = xmldoc.at("//bibdata/docidentifier[@type = 'iso-with-lang']") and
+          id.content = id_prefix(prefix, id)
+        id = xmldoc.at("//bibdata/docidentifier[@type = 'iso-reference']") and
+          id.content = id_prefix(prefix, id)
+      end
+
+      def format_ref(ref, type)
+        ref = ref.sub(/ \(All Parts\)/i, "")
+        super
+      end
+
+      TERM_CLAUSE =
+        "//sections//terms | "\
+        "//sections//clause[descendant::terms][not(descendant::definitions)]"
+          .freeze
+
+      PUBLISHER = "./contributor[role/@type = 'publisher']/organization".freeze
+
+      OTHERIDS = "@type = 'DOI' or @type = 'metanorma' or @type = 'ISSN' or "\
+                 "@type = 'ISBN'".freeze
+
+      def pub_class(bib)
+        return 1 if bib.at("#{PUBLISHER}[abbreviation = 'ISO']")
+        return 1 if bib.at("#{PUBLISHER}[name = 'International Organization "\
+                           "for Standardization']")
+        return 2 if bib.at("#{PUBLISHER}[abbreviation = 'IEC']")
+        return 2 if bib.at("#{PUBLISHER}[name = 'International "\
+                           "Electrotechnical Commission']")
+        return 3 if bib.at("./docidentifier[@type][not(#{OTHERIDS})]")
+
+        4
+      end
+
+      def sort_biblio(bib)
+        bib.sort do |a, b|
+          sort_biblio_key(a) <=> sort_biblio_key(b)
+        end
+      end
+
+      # TODO sort by authors
+      # sort by: doc class (ISO, IEC, other standard (not DOI &c), other
+      # then standard class (docid class other than DOI &c)
+      # then docnumber if present, numeric sort
+      #      else alphanumeric metanorma id (abbreviation)
+      # then doc part number if present, numeric sort
+      # then doc id (not DOI &c)
+      # then title
+      def sort_biblio_key(bib)
+        pubclass = pub_class(bib)
+        num = bib&.at("./docnumber")&.text
+        id = bib&.at("./docidentifier[not(#{OTHERIDS})]")
+        metaid = bib&.at("./docidentifier[@type = 'metanorma']")&.text
+        abbrid = metaid unless /^\[\d+\]$/.match?(metaid)
+        /\d-(?<partid>\d+)/ =~ id&.text
+        type = id["type"] if id
+        title = bib&.at("./title[@type = 'main']")&.text ||
+          bib&.at("./title")&.text || bib&.at("./formattedref")&.text
+        "#{pubclass} :: #{type} :: "\
+          "#{num.nil? ? abbrid : sprintf('%09d', num.to_i)} :: "\
+          "#{sprintf('%09d', partid.to_i)} :: #{id&.text} :: #{title}"
+      end
+
+      def sections_cleanup(xml)
+        super
+        return unless @amd
+
+        xml.xpath("//*[@inline-header]").each do |h|
+          h.delete("inline-header")
+        end
+      end
+
+      def boilerplate_file(_xmldoc)
+        file = @lang == "fr" ? "boilerplate-fr.xml" : "boilerplate.xml"
+        File.join(@libdir, file)
+      end
+
+      def footnote_cleanup(xmldoc)
+        unpub_footnotes(xmldoc)
+        super
+      end
+
+      def unpub_footnotes(xmldoc)
+        xmldoc.xpath("//bibitem/note[@type = 'Unpublished-Status']").each do |n|
+          id = n.parent["id"]
+          e = xmldoc.at("//eref[@bibitemid = '#{id}']") or next
+          fn = n.children.to_xml
+          n&.elements&.first&.name == "p" or fn = "<p>#{fn}</p>"
+          e.next = "<fn>#{fn}</fn>"
+        end
+      end
+
+      def bibitem_cleanup(xmldoc)
+        super
+        unpublished_note(xmldoc)
+      end
+
+      def unpublished_note(xmldoc)
+        xmldoc.xpath("//bibitem[not(note[@type = 'Unpublished-Status'])]")
+          .each do |b|
+          next if pub_class(b) > 2
+          next unless (s = b.at("./status/stage")) && (s.text.to_i < 60)
+
+          id = b.at("docidentifier").text
+          b.at("./language | ./script | ./abstract | ./status")
+            .previous = %(<note type="Unpublished-Status">
+                          <p>#{@i18n.under_preparation.sub(/%/, id)}</p></note>)
+        end
+      end
+
+      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
+        once = true
+        super
+      end
+
+      def term_defs_boilerplate_cont(src, term, isodoc)
+        @vocab and src.empty? and return
+        super
+      end
+    end
+  end
+end

data/lib/metanorma/iso/converter.rb

@@ -0,0 +1,18 @@
+require "asciidoctor"
+require "metanorma-standoc"
+require "metanorma/iso/version"
+require "metanorma/iso/base"
+require "metanorma/iso/front"
+require "metanorma/iso/section"
+require "metanorma/iso/validate"
+require "metanorma/iso/cleanup"
+
+module Metanorma
+  module ISO
+    # A {Converter} implementation that generates ISO output, and a document
+    # schema encapsulation of the document for validation
+    class Converter < ::Metanorma::Standoc::Converter
+      register_for "iso"
+    end
+  end
+end

data/lib/metanorma/iso/front.rb

@@ -0,0 +1,170 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "open-uri"
+require_relative "front_id"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def metadata_ext(node, xml)
+        super
+        structured_id(node, xml)
+        xml.stagename stage_name(get_stage(node), get_substage(node),
+                                 doctype(node), node.attr("iteration"))
+        @amd && a = node.attr("updates-document-type") and
+          xml.updates_document_type a
+      end
+
+      def metadata_subdoctype(node, xml)
+        super
+        a = node.attr("horizontal") and xml.horizontal a
+      end
+
+      def org_abbrev
+        { "International Organization for Standardization" => "ISO",
+          "International Electrotechnical Commission" => "IEC" }
+      end
+
+      def metadata_author(node, xml)
+        publishers = node.attr("publisher") || "ISO"
+        csv_split(publishers).each do |p|
+          xml.contributor do |c|
+            c.role **{ type: "author" }
+            c.organization do |a|
+              organization(a, p, false, node, !node.attr("publisher"))
+            end
+          end
+        end
+      end
+
+      def metadata_publisher(node, xml)
+        publishers = node.attr("publisher") || "ISO"
+        csv_split(publishers).each do |p|
+          xml.contributor do |c|
+            c.role **{ type: "publisher" }
+            c.organization do |a|
+              organization(a, p, true, node, !node.attr("publisher"))
+            end
+          end
+        end
+      end
+
+      def metadata_copyright(node, xml)
+        publishers = node.attr("copyright-holder") || node.attr("publisher") ||
+          "ISO"
+        csv_split(publishers).each do |p|
+          xml.copyright do |c|
+            c.from (node.attr("copyright-year") || Date.today.year)
+            c.owner do |owner|
+              owner.organization do |o|
+                organization(
+                  o, p, true, node,
+                  !(node.attr("copyright-holder") || node.attr("publisher"))
+                )
+              end
+            end
+          end
+        end
+      end
+
+      def metadata_status(node, xml)
+        stage = get_stage(node)
+        substage = get_substage(node)
+        xml.status do |s|
+          s.stage stage, **attr_code(abbreviation: cover_stage_abbr(node))
+          s.substage substage
+          node.attr("iteration") && (s.iteration node.attr("iteration"))
+        end
+      end
+
+      def metadata_committee(node, xml)
+        xml.editorialgroup do |a|
+          committee_component("technical-committee", node, a)
+          committee_component("subcommittee", node, a)
+          committee_component("workgroup", node, a)
+          node.attr("secretariat") && a.secretariat(node.attr("secretariat"))
+        end
+      end
+
+      def title_intro(node, xml, lang, at)
+        return unless node.attr("title-intro-#{lang}")
+
+        xml.title(**attr_code(at.merge(type: "title-intro"))) do |t1|
+          t1 << Metanorma::Utils::asciidoc_sub(node.attr("title-intro-#{lang}"))
+        end
+      end
+
+      def title_main(node, xml, lang, at)
+        xml.title **attr_code(at.merge(type: "title-main")) do |t1|
+          t1 << Metanorma::Utils::asciidoc_sub(node.attr("title-main-#{lang}"))
+        end
+      end
+
+      def title_part(node, xml, lang, at)
+        return unless node.attr("title-part-#{lang}")
+
+        xml.title(**attr_code(at.merge(type: "title-part"))) do |t1|
+          t1 << Metanorma::Utils::asciidoc_sub(node.attr("title-part-#{lang}"))
+        end
+      end
+
+      def title_amd(node, xml, lang, at)
+        return unless node.attr("title-amendment-#{lang}")
+
+        xml.title(**attr_code(at.merge(type: "title-amd"))) do |t1|
+          t1 << Metanorma::Utils::asciidoc_sub(
+            node.attr("title-amendment-#{lang}"),
+          )
+        end
+      end
+
+      def title_full(node, xml, lang, at)
+        title = node.attr("title-main-#{lang}")
+        intro = node.attr("title-intro-#{lang}")
+        part = node.attr("title-part-#{lang}")
+        amd = node.attr("title-amendment-#{lang}")
+        title = "#{intro} -- #{title}" if intro
+        title = "#{title} -- #{part}" if part
+        title = "#{title} -- #{amd}" if amd && @amd
+        xml.title **attr_code(at.merge(type: "main")) do |t1|
+          t1 << Metanorma::Utils::asciidoc_sub(title)
+        end
+      end
+
+      def title(node, xml)
+        ["en", "fr"].each do |lang|
+          at = { language: lang, format: "text/plain" }
+          title_full(node, xml, lang, at)
+          title_intro(node, xml, lang, at)
+          title_main(node, xml, lang, at)
+          title_part(node, xml, lang, at)
+          title_amd(node, xml, lang, at) if @amd
+        end
+      end
+
+      def relaton_relations
+        super + %w(obsoletes successor-of manifestation-of related annotation-of)
+      end
+
+      def relaton_relation_descriptions
+        super.merge(
+          "amends" => "updates", "revises" => "updates",
+          "replaces" => "obsoletes",
+          "supersedes" => "obsoletes",
+          "corrects" => "updates",
+          "informatively-cited-in" => "isCitedIn",
+          "informatively-cites" => "cites",
+          "normatively-cited in" => "isCitedIn",
+          "normatively-cites" => "cites",
+          "identical-adopted-from" => "adoptedFrom",
+          "modified-adopted-from" => "adoptedFrom",
+          "related-directive" => "related",
+          "related-mandate" => "related",
+        )
+      end
+    end
+  end
+end

data/lib/metanorma/iso/front_id.rb

@@ -0,0 +1,225 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "open-uri"
+require "twitter_cldr"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      STAGE_ABBRS = {
+        "00": "PWI",
+        "10": "NP",
+        "20": "WD",
+        "30": "CD",
+        "40": "DIS",
+        "50": "FDIS",
+        "60": "IS",
+        "90": "(Review)",
+        "95": "(Withdrawal)",
+      }.freeze
+
+      STAGE_NAMES = {
+        "00": "Preliminary work item",
+        "10": "New work item proposal",
+        "20": "Working draft",
+        "30": "Committee draft",
+        "40": "Draft",
+        "50": "Final draft",
+        "60": "International standard",
+        "90": "Review",
+        "95": "Withdrawal",
+      }.freeze
+
+      def stage_abbr(stage, substage, doctype)
+        return nil if stage.to_i > 60
+
+        ret = STAGE_ABBRS[stage.to_sym]
+        ret = "PRF" if stage == "60" && substage == "00"
+        ret = "AWI" if stage == "10" && substage == "99"
+        if %w(amendment technical-corrigendum technical-report
+              technical-specification).include?(doctype)
+          ret = "D" if stage == "40" && doctype == "amendment"
+          ret = "FD" if stage == "50" && %w(amendment technical-corrigendum)
+            .include?(doctype)
+        end
+        ret
+      end
+
+      def stage_name(stage, substage, _doctype, iteration = nil)
+        return "Proof" if stage == "60" && substage == "00"
+
+        ret = STAGE_NAMES[stage.to_sym]
+        if iteration && %w(20 30).include?(stage)
+          prefix = iteration.to_i.localize(@lang.to_sym)
+            .to_rbnf_s("SpelloutRules", "spellout-ordinal")
+          ret = "#{prefix.capitalize} #{ret.downcase}"
+        end
+        ret
+      end
+
+      def metadata_id(node, xml)
+        iso_id(node, xml)
+        node&.attr("tc-docnumber")&.split(/,\s*/)&.each do |n|
+          xml.docidentifier(n, **attr_code(type: "iso-tc"))
+        end
+        xml.docnumber node&.attr("docnumber")
+      end
+
+      def iso_id(node, xml)
+        (!@amd && node.attr("docnumber")) || (@amd && node.attr("updates")) or
+          return
+
+        dn = iso_id1(node)
+        dn1 = id_stage_prefix(dn, node, false)
+        dn2 = id_stage_prefix(dn, node, true)
+        xml.docidentifier dn1, **attr_code(type: "ISO")
+        xml.docidentifier(id_langsuffix(dn1, node),
+                          **attr_code(type: "iso-with-lang"))
+        xml.docidentifier(id_langsuffix(dn2, node),
+                          **attr_code(type: "iso-reference"))
+      end
+
+      def iso_id1(node)
+        if @amd
+          dn = node.attr("updates")
+          add_amd_parts(dn, node)
+        else
+          part, subpart = node&.attr("partnumber")&.split(/-/)
+          add_id_parts(node.attr("docnumber"), part, subpart)
+        end
+      end
+
+      def add_amd_parts(docnum, node)
+        case doctype(node)
+        when "amendment"
+          "#{docnum}/Amd #{node.attr('amendment-number')}"
+        when "technical-corrigendum"
+          "#{docnum}/Cor.#{node.attr('corrigendum-number')}"
+        end
+      end
+
+      def id_langsuffix(docnum, node)
+        lang = node.attr("language") || "en"
+        suffix = case lang
+                 when "en" then "(E)"
+                 when "fr" then "(F)"
+                 else
+                   "(X)"
+                 end
+        "#{docnum}#{suffix}"
+      end
+
+      def structured_id(node, xml)
+        return unless node.attr("docnumber")
+
+        part, subpart = node&.attr("partnumber")&.split(/-/)
+        xml.structuredidentifier do |i|
+          i.project_number(node.attr("docnumber"), **attr_code(
+            part: part, subpart: subpart,
+            amendment: node.attr("amendment-number"),
+            corrigendum: node.attr("corrigendum-number"),
+            origyr: node.attr("created-date")
+          ))
+        end
+      end
+
+      def add_id_parts(docnum, part, subpart)
+        docnum += "-#{part}" if part
+        docnum += "-#{subpart}" if subpart
+        docnum
+      end
+
+      def id_stage_abbr(stage, substage, node, bare = false)
+        ret = id_stage_abbr1(stage, substage, node, bare)
+        if %w(amendment technical-corrigendum technical-report
+              technical-specification).include?(doctype(node)) &&
+            !%w(D FD).include?(ret)
+          ret = "#{ret} "
+        end
+        ret
+      end
+
+      def id_stage_abbr1(stage, substage, node, bare)
+        if bare
+          IsoDoc::Iso::Metadata.new("en", "Latn", @i18n)
+            .status_abbrev(stage_abbr(stage, substage, doctype(node)),
+                           substage, nil, nil, doctype(node))
+        else
+          IsoDoc::Iso::Metadata.new("en", "Latn", @i18n)
+            .status_abbrev(stage_abbr(stage, substage, doctype(node)),
+                           substage, node.attr("iteration"),
+                           node.attr("draft"), doctype(node))
+        end
+      end
+
+      def cover_stage_abbr(node)
+        stage = get_stage(node)
+        abbr = id_stage_abbr(get_stage(node), get_substage(node), node, true)
+        typeabbr = get_typeabbr(node, true)
+        if stage.to_i > 50 || (stage.to_i == 60 && get_substage(node).to_i < 60)
+          typeabbr = ""
+        end
+        "#{abbr}#{typeabbr}".strip
+      end
+
+      def id_stage_prefix(docnum, node, force_year)
+        stage = get_stage(node)
+        typeabbr = get_typeabbr(node)
+        if stage && (stage.to_i < 60)
+          docnum = unpub_stage_prefix(docnum, stage, typeabbr, node)
+        elsif typeabbr == "DIR " then docnum = "#{typeabbr}#{docnum}"
+        elsif typeabbr && !@amd then docnum = "/#{typeabbr}#{docnum}"
+        end
+        (force_year || !(stage && (stage.to_i < 60))) and
+          docnum = id_add_year(docnum, node)
+        docnum
+      end
+
+      def unpub_stage_prefix(docnum, stage, typeabbr, node)
+        abbr = id_stage_abbr(stage, get_substage(node), node)
+        %w(40 50).include?(stage) && i = node.attr("iteration") and
+          itersuffix = ".#{i}"
+        return docnum if abbr.nil? || abbr.empty? # prefixes added in cleanup
+        return "/#{abbr}#{typeabbr} #{docnum}#{itersuffix}" unless @amd
+
+        a = docnum.split(%r{/})
+        a[-1] = "#{abbr}#{a[-1]}#{itersuffix}"
+        a.join("/")
+      end
+
+      def id_add_year(docnum, node)
+        year = node.attr("copyright-year")
+        @amd and year ||= node.attr("updated-date")&.sub(/-.*$/, "")
+        docnum += ":#{year}" if year
+        docnum
+      end
+
+      def get_stage(node)
+        a = node.attr("status")
+        a = node.attr("docstage") if a.nil? || a.empty?
+        a = "60" if a.nil? || a.empty?
+        a
+      end
+
+      def get_substage(node)
+        stage = get_stage(node)
+        ret = node.attr("docsubstage")
+        ret = (stage == "60" ? "60" : "00") if ret.nil? || ret.empty?
+        ret
+      end
+
+      def get_typeabbr(node, amd = false)
+        case doctype(node)
+        when "directive" then "DIR "
+        when "technical-report" then "TR "
+        when "technical-specification" then "TS "
+        when "amendment" then (amd ? "Amd " : "")
+        when "technical-corrigendum" then (amd ? "Cor " : "")
+        end
+      end
+    end
+  end
+end

data/lib/{asciidoctor → metanorma}/iso/isodoc.rng

@@ -32,6 +32,19 @@
         <ref name="DocumentType"/>
       </element>
     </define>
+    <define name="bibitem">
+      <element name="bibitem">
+        <attribute name="id">
+          <data type="ID"/>
+        </attribute>
+        <optional>
+          <attribute name="hidden">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
+        <ref name="BibliographicItem"/>
+      </element>
+    </define>
     <define name="section-title">
       <element name="title">
         <zeroOrMore>
@@ -690,6 +703,7 @@
             <ref name="terms"/>
             <ref name="term-clause"/>
             <ref name="definitions"/>
+            <ref name="floating-title"/>
           </choice>
         </oneOrMore>
       </element>
@@ -1680,6 +1694,7 @@
           <ref name="clause-subsection"/>
           <ref name="terms"/>
           <ref name="definitions"/>
+          <ref name="floating-title"/>
         </choice>
       </oneOrMore>
     </choice>
@@ -1722,6 +1737,7 @@
           <ref name="terms"/>
           <ref name="definitions"/>
           <ref name="references"/>
+          <ref name="floating-title"/>
         </choice>
       </zeroOrMore>
     </group>
@@ -2534,4 +2550,17 @@
       </oneOrMore>
     </element>
   </define>
+  <define name="floating-title">
+    <element name="floating-title">
+      <attribute name="id">
+        <data type="ID"/>
+      </attribute>
+      <attribute name="depth">
+        <data type="int"/>
+      </attribute>
+      <zeroOrMore>
+        <ref name="TextElement"/>
+      </zeroOrMore>
+    </element>
+  </define>
 </grammar>