metanorma-iso 1.10.3 → 2.0.0

data/lib/metanorma/iso/front_id.rb

@@ -0,0 +1,225 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "open-uri"
+require "twitter_cldr"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      STAGE_ABBRS = {
+        "00": "PWI",
+        "10": "NP",
+        "20": "WD",
+        "30": "CD",
+        "40": "DIS",
+        "50": "FDIS",
+        "60": "IS",
+        "90": "(Review)",
+        "95": "(Withdrawal)",
+      }.freeze
+
+      STAGE_NAMES = {
+        "00": "Preliminary work item",
+        "10": "New work item proposal",
+        "20": "Working draft",
+        "30": "Committee draft",
+        "40": "Draft",
+        "50": "Final draft",
+        "60": "International standard",
+        "90": "Review",
+        "95": "Withdrawal",
+      }.freeze
+
+      def stage_abbr(stage, substage, doctype)
+        return nil if stage.to_i > 60
+
+        ret = STAGE_ABBRS[stage.to_sym]
+        ret = "PRF" if stage == "60" && substage == "00"
+        ret = "AWI" if stage == "10" && substage == "99"
+        if %w(amendment technical-corrigendum technical-report
+              technical-specification).include?(doctype)
+          ret = "D" if stage == "40" && doctype == "amendment"
+          ret = "FD" if stage == "50" && %w(amendment technical-corrigendum)
+            .include?(doctype)
+        end
+        ret
+      end
+
+      def stage_name(stage, substage, _doctype, iteration = nil)
+        return "Proof" if stage == "60" && substage == "00"
+
+        ret = STAGE_NAMES[stage.to_sym]
+        if iteration && %w(20 30).include?(stage)
+          prefix = iteration.to_i.localize(@lang.to_sym)
+            .to_rbnf_s("SpelloutRules", "spellout-ordinal")
+          ret = "#{prefix.capitalize} #{ret.downcase}"
+        end
+        ret
+      end
+
+      def metadata_id(node, xml)
+        iso_id(node, xml)
+        node&.attr("tc-docnumber")&.split(/,\s*/)&.each do |n|
+          xml.docidentifier(n, **attr_code(type: "iso-tc"))
+        end
+        xml.docnumber node&.attr("docnumber")
+      end
+
+      def iso_id(node, xml)
+        (!@amd && node.attr("docnumber")) || (@amd && node.attr("updates")) or
+          return
+
+        dn = iso_id1(node)
+        dn1 = id_stage_prefix(dn, node, false)
+        dn2 = id_stage_prefix(dn, node, true)
+        xml.docidentifier dn1, **attr_code(type: "ISO")
+        xml.docidentifier(id_langsuffix(dn1, node),
+                          **attr_code(type: "iso-with-lang"))
+        xml.docidentifier(id_langsuffix(dn2, node),
+                          **attr_code(type: "iso-reference"))
+      end
+
+      def iso_id1(node)
+        if @amd
+          dn = node.attr("updates")
+          add_amd_parts(dn, node)
+        else
+          part, subpart = node&.attr("partnumber")&.split(/-/)
+          add_id_parts(node.attr("docnumber"), part, subpart)
+        end
+      end
+
+      def add_amd_parts(docnum, node)
+        case doctype(node)
+        when "amendment"
+          "#{docnum}/Amd #{node.attr('amendment-number')}"
+        when "technical-corrigendum"
+          "#{docnum}/Cor.#{node.attr('corrigendum-number')}"
+        end
+      end
+
+      def id_langsuffix(docnum, node)
+        lang = node.attr("language") || "en"
+        suffix = case lang
+                 when "en" then "(E)"
+                 when "fr" then "(F)"
+                 else
+                   "(X)"
+                 end
+        "#{docnum}#{suffix}"
+      end
+
+      def structured_id(node, xml)
+        return unless node.attr("docnumber")
+
+        part, subpart = node&.attr("partnumber")&.split(/-/)
+        xml.structuredidentifier do |i|
+          i.project_number(node.attr("docnumber"), **attr_code(
+            part: part, subpart: subpart,
+            amendment: node.attr("amendment-number"),
+            corrigendum: node.attr("corrigendum-number"),
+            origyr: node.attr("created-date")
+          ))
+        end
+      end
+
+      def add_id_parts(docnum, part, subpart)
+        docnum += "-#{part}" if part
+        docnum += "-#{subpart}" if subpart
+        docnum
+      end
+
+      def id_stage_abbr(stage, substage, node, bare = false)
+        ret = id_stage_abbr1(stage, substage, node, bare)
+        if %w(amendment technical-corrigendum technical-report
+              technical-specification).include?(doctype(node)) &&
+            !%w(D FD).include?(ret)
+          ret = "#{ret} "
+        end
+        ret
+      end
+
+      def id_stage_abbr1(stage, substage, node, bare)
+        if bare
+          IsoDoc::Iso::Metadata.new("en", "Latn", @i18n)
+            .status_abbrev(stage_abbr(stage, substage, doctype(node)),
+                           substage, nil, nil, doctype(node))
+        else
+          IsoDoc::Iso::Metadata.new("en", "Latn", @i18n)
+            .status_abbrev(stage_abbr(stage, substage, doctype(node)),
+                           substage, node.attr("iteration"),
+                           node.attr("draft"), doctype(node))
+        end
+      end
+
+      def cover_stage_abbr(node)
+        stage = get_stage(node)
+        abbr = id_stage_abbr(get_stage(node), get_substage(node), node, true)
+        typeabbr = get_typeabbr(node, true)
+        if stage.to_i > 50 || (stage.to_i == 60 && get_substage(node).to_i < 60)
+          typeabbr = ""
+        end
+        "#{abbr}#{typeabbr}".strip
+      end
+
+      def id_stage_prefix(docnum, node, force_year)
+        stage = get_stage(node)
+        typeabbr = get_typeabbr(node)
+        if stage && (stage.to_i < 60)
+          docnum = unpub_stage_prefix(docnum, stage, typeabbr, node)
+        elsif typeabbr == "DIR " then docnum = "#{typeabbr}#{docnum}"
+        elsif typeabbr && !@amd then docnum = "/#{typeabbr}#{docnum}"
+        end
+        (force_year || !(stage && (stage.to_i < 60))) and
+          docnum = id_add_year(docnum, node)
+        docnum
+      end
+
+      def unpub_stage_prefix(docnum, stage, typeabbr, node)
+        abbr = id_stage_abbr(stage, get_substage(node), node)
+        %w(40 50).include?(stage) && i = node.attr("iteration") and
+          itersuffix = ".#{i}"
+        return docnum if abbr.nil? || abbr.empty? # prefixes added in cleanup
+        return "/#{abbr}#{typeabbr} #{docnum}#{itersuffix}" unless @amd
+
+        a = docnum.split(%r{/})
+        a[-1] = "#{abbr}#{a[-1]}#{itersuffix}"
+        a.join("/")
+      end
+
+      def id_add_year(docnum, node)
+        year = node.attr("copyright-year")
+        @amd and year ||= node.attr("updated-date")&.sub(/-.*$/, "")
+        docnum += ":#{year}" if year
+        docnum
+      end
+
+      def get_stage(node)
+        a = node.attr("status")
+        a = node.attr("docstage") if a.nil? || a.empty?
+        a = "60" if a.nil? || a.empty?
+        a
+      end
+
+      def get_substage(node)
+        stage = get_stage(node)
+        ret = node.attr("docsubstage")
+        ret = (stage == "60" ? "60" : "00") if ret.nil? || ret.empty?
+        ret
+      end
+
+      def get_typeabbr(node, amd = false)
+        case doctype(node)
+        when "directive" then "DIR "
+        when "technical-report" then "TR "
+        when "technical-specification" then "TS "
+        when "amendment" then (amd ? "Amd " : "")
+        when "technical-corrigendum" then (amd ? "Cor " : "")
+        end
+      end
+    end
+  end
+end

data/lib/{asciidoctor → metanorma}/iso/isodoc.rng

@@ -32,6 +32,19 @@
         <ref name="DocumentType"/>
       </element>
     </define>
+    <define name="bibitem">
+      <element name="bibitem">
+        <attribute name="id">
+          <data type="ID"/>
+        </attribute>
+        <optional>
+          <attribute name="hidden">
+            <data type="boolean"/>
+          </attribute>
+        </optional>
+        <ref name="BibliographicItem"/>
+      </element>
+    </define>
     <define name="section-title">
       <element name="title">
         <zeroOrMore>
@@ -690,6 +703,7 @@
             <ref name="terms"/>
             <ref name="term-clause"/>
             <ref name="definitions"/>
+            <ref name="floating-title"/>
           </choice>
         </oneOrMore>
       </element>
@@ -1680,6 +1694,7 @@
           <ref name="clause-subsection"/>
           <ref name="terms"/>
           <ref name="definitions"/>
+          <ref name="floating-title"/>
         </choice>
       </oneOrMore>
     </choice>
@@ -1722,6 +1737,7 @@
           <ref name="terms"/>
           <ref name="definitions"/>
           <ref name="references"/>
+          <ref name="floating-title"/>
         </choice>
       </zeroOrMore>
     </group>
@@ -1796,6 +1812,20 @@
           <data type="ID"/>
         </attribute>
       </optional>
+      <optional>
+        <attribute name="language"/>
+      </optional>
+      <optional>
+        <attribute name="script"/>
+      </optional>
+      <optional>
+        <attribute name="tag"/>
+      </optional>
+      <optional>
+        <attribute name="multilingual-rendering">
+          <ref name="MultilingualRenderingType"/>
+        </attribute>
+      </optional>
       <oneOrMore>
         <ref name="preferred"/>
       </oneOrMore>
@@ -1959,7 +1989,11 @@
       </optional>
       <element name="name">
         <zeroOrMore>
-          <ref name="PureTextElement"/>
+          <choice>
+            <ref name="PureTextElement"/>
+            <ref name="stem"/>
+            <ref name="index"/>
+          </choice>
         </zeroOrMore>
       </element>
       <optional>
@@ -1973,7 +2007,7 @@
         </element>
       </optional>
       <optional>
-        <element name="grammar-info">
+        <element name="grammar">
           <ref name="Grammar"/>
         </element>
       </optional>
@@ -2516,4 +2550,17 @@
       </oneOrMore>
     </element>
   </define>
+  <define name="floating-title">
+    <element name="floating-title">
+      <attribute name="id">
+        <data type="ID"/>
+      </attribute>
+      <attribute name="depth">
+        <data type="int"/>
+      </attribute>
+      <zeroOrMore>
+        <ref name="TextElement"/>
+      </zeroOrMore>
+    </element>
+  </define>
 </grammar>

data/lib/{asciidoctor → metanorma}/iso/isostandard.rng

@@ -159,6 +159,20 @@
             <data type="ID"/>
           </attribute>
         </optional>
+        <optional>
+          <attribute name="language"/>
+        </optional>
+        <optional>
+          <attribute name="script"/>
+        </optional>
+        <optional>
+          <attribute name="tag"/>
+        </optional>
+        <optional>
+          <attribute name="multilingual-rendering">
+            <ref name="MultilingualRenderingType"/>
+          </attribute>
+        </optional>
         <ref name="preferred"/>
         <zeroOrMore>
           <ref name="admitted"/>

data/lib/metanorma/iso/section.rb

@@ -0,0 +1,49 @@
+require "htmlentities"
+require "uri" if /^2\./.match?(RUBY_VERSION)
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def clause_parse(attrs, xml, node)
+        node.option? "appendix" and return appendix_parse(attrs, xml, node)
+        super
+      end
+
+      def scope_parse(attrs, xml, node)
+        attrs = attrs.merge(type: "scope") unless @amd
+        clause_parse(attrs, xml, node)
+      end
+
+      def appendix_parse(attrs, xml, node)
+        attrs[:"inline-header"] = node.option? "inline-header"
+        set_obligation(attrs, node)
+        xml.appendix **attr_code(attrs) do |xml_section|
+          xml_section.title { |name| name << node.title }
+          xml_section << node.content
+        end
+      end
+
+      def patent_notice_parse(xml, node)
+        # xml.patent_notice do |xml_section|
+        #  xml_section << node.content
+        # end
+        xml << node.content
+      end
+
+      def sectiontype(node, level = true)
+        return nil if @amd
+
+        ret = sectiontype_streamline(sectiontype1(node))
+        return ret if ret == "terms and definitions" && @vocab
+
+        super
+      end
+
+      def term_def_subclause_parse(attrs, xml, node)
+        node.role == "term" and
+          return term_def_subclause_parse1(attrs, xml, node)
+        super
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate.rb

@@ -0,0 +1,172 @@
+require "metanorma-standoc"
+require_relative "./validate_style"
+require_relative "./validate_requirements"
+require_relative "./validate_section"
+require_relative "./validate_title"
+require_relative "./validate_image"
+require "nokogiri"
+require "jing"
+require "iev"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def isosubgroup_validate(root)
+        root.xpath("//technical-committee/@type").each do |t|
+          unless %w{TC PC JTC JPC}.include? t.text
+            @log.add("Document Attributes", nil,
+                     "invalid technical committee type #{t}")
+          end
+        end
+        root.xpath("//subcommittee/@type").each do |t|
+          unless %w{SC JSC}.include? t.text
+            @log.add("Document Attributes", nil,
+                     "invalid subcommittee type #{t}")
+          end
+        end
+      end
+
+      # ISO/IEC DIR 2, 15.5.3
+      # does not deal with preceding text marked up
+      def see_xrefs_validate(root)
+        root.xpath("//xref").each do |t|
+          preceding = t.at("./preceding-sibling::text()[last()]")
+          next unless !preceding.nil? &&
+            /\b(see| refer to)\s*\Z/mi.match(preceding)
+
+          (target = root.at("//*[@id = '#{t['target']}']")) || next
+          if target&.at("./ancestor-or-self::*[@obligation = 'normative']")
+            @log.add("Style", t,
+                     "'see #{t['target']}' is pointing to a normative section")
+          end
+        end
+      end
+
+      # ISO/IEC DIR 2, 15.5.3
+      def see_erefs_validate(root)
+        root.xpath("//eref").each do |t|
+          prec = t.at("./preceding-sibling::text()[last()]")
+          next unless !prec.nil? && /\b(see|refer to)\s*\Z/mi.match(prec)
+
+          unless target = root.at("//*[@id = '#{t['bibitemid']}']")
+            @log.add("Bibliography", t,
+                     "'#{t} is not pointing to a real reference")
+            next
+          end
+          target.at("./ancestor::references[@normative = 'true']") and
+            @log.add("Style", t,
+                     "'see #{t}' is pointing to a normative reference")
+        end
+      end
+
+      # ISO/IEC DIR 2, 10.4
+      def locality_erefs_validate(root)
+        root.xpath("//eref[descendant::locality]").each do |t|
+          if /^(ISO|IEC)/.match?(t["citeas"]) &&
+              !/: ?(\d+{4}|–)$/.match?(t["citeas"])
+            @log.add("Style", t,
+                     "undated reference #{t['citeas']} should not contain "\
+                     "specific elements")
+          end
+        end
+      end
+
+      def termdef_warn(text, regex, elem, term, msg)
+        regex.match(text) && @log.add("Style", elem, "#{term}: #{msg}")
+      end
+
+      # ISO/IEC DIR 2, 16.5.6
+      def termdef_style(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          para = t.at("./definition/verbal-definition") || return
+          term = t.at("./preferred//name").text
+          termdef_warn(para.text, /\A(the|a)\b/i, t, term,
+                       "term definition starts with article")
+          termdef_warn(para.text, /\.\Z/i, t, term,
+                       "term definition ends with period")
+        end
+      end
+
+      def doctype_validate(xmldoc)
+        doctype = xmldoc&.at("//bibdata/ext/doctype")&.text
+        %w(international-standard technical-specification technical-report
+           publicly-available-specification international-workshop-agreement
+           guide amendment technical-corrigendum).include? doctype or
+          @log.add("Document Attributes", nil,
+                   "#{doctype} is not a recognised document type")
+      end
+
+      def script_validate(xmldoc)
+        script = xmldoc&.at("//bibdata/script")&.text
+        script == "Latn" or
+          @log.add("Document Attributes", nil,
+                   "#{script} is not a recognised script")
+      end
+
+      def stage_validate(xmldoc)
+        stage = xmldoc&.at("//bibdata/status/stage")&.text
+        %w(00 10 20 30 40 50 60 90 95).include? stage or
+          @log.add("Document Attributes", nil,
+                   "#{stage} is not a recognised stage")
+      end
+
+      def substage_validate(xmldoc)
+        substage = xmldoc&.at("//bibdata/status/substage")&.text or return
+        %w(00 20 60 90 92 93 98 99).include? substage or
+          @log.add("Document Attributes", nil,
+                   "#{substage} is not a recognised substage")
+      end
+
+      def iteration_validate(xmldoc)
+        iteration = xmldoc&.at("//bibdata/status/iteration")&.text or return
+        /^\d+/.match(iteration) or
+          @log.add("Document Attributes", nil,
+                   "#{iteration} is not a recognised iteration")
+      end
+
+      def bibdata_validate(doc)
+        doctype_validate(doc)
+        script_validate(doc)
+        stage_validate(doc)
+        substage_validate(doc)
+        iteration_validate(doc)
+      end
+
+      def content_validate(doc)
+        super
+        title_validate(doc.root)
+        isosubgroup_validate(doc.root)
+        onlychild_clause_validate(doc.root)
+        termdef_style(doc.root)
+        see_xrefs_validate(doc.root)
+        see_erefs_validate(doc.root)
+        locality_erefs_validate(doc.root)
+        bibdata_validate(doc.root)
+        bibitem_validate(doc.root)
+        figure_validate(doc.root)
+      end
+
+      def bibitem_validate(xmldoc)
+        xmldoc.xpath("//bibitem[date/on = '–']").each do |b|
+          b.at("./note[@type = 'Unpublished-Status']") or
+            @log.add("Style", b,
+                     "Reference #{b&.at('./@id')&.text} does not have an "\
+                     "associated footnote indicating unpublished status")
+        end
+      end
+
+      def validate(doc)
+        content_validate(doc)
+        doctype = doc&.at("//bibdata/ext/doctype")&.text
+        schema = case doctype
+                 when "amendment", "technical-corrigendum" # @amd
+                   "isostandard-amd.rng"
+                 else
+                   "isostandard.rng"
+                 end
+        schema_validate(formattedstr_strip(doc.dup),
+                        File.join(File.dirname(__FILE__), schema))
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_image.rb

@@ -0,0 +1,97 @@
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      # DRG directives 3.7; but anticipated by standoc
+      def subfigure_validate(xmldoc)
+        xmldoc.xpath("//figure//figure").each do |f|
+          { footnote: "fn", note: "note", key: "dl" }.each do |k, v|
+            f.xpath(".//#{v}").each do |n|
+              @log.add("Style", n, "#{k} is not permitted in a subfigure")
+            end
+          end
+        end
+      end
+
+      def image_name_prefix(xmldoc)
+        std = xmldoc&.at("//bibdata/ext/structuredidentifier/project-number") or
+          return
+        num = xmldoc&.at("//bibdata/docnumber")&.text or return
+        ed = xmldoc&.at("//bibdata/edition")&.text || "1"
+        prefix = num
+        std["part"] and prefix += "-#{std['part']}"
+        prefix += "_ed#{ed}"
+        amd = std["amendment"] and prefix += "amd#{amd}"
+        prefix
+      end
+
+      def image_name_suffix(xmldoc)
+        case xmldoc&.at("//bibdata/language")&.text
+        when "fr" then "_f"
+        when "de" then "_d"
+        when "ru" then "_r"
+        when "es" then "_s"
+        when "ar" then "_a"
+        # when "en" then "_e"
+        else
+          "_e"
+        end
+      end
+
+      def disjunct_error(img, cond1, cond2, msg1, msg2)
+        cond1 && !cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg1}")
+        !cond1 && cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg2}")
+      end
+
+      def image_name_parse(img, prefix)
+        m = %r[(SL)?#{prefix}fig(?<tab>Tab)?(?<annex>[A-Z])?(Text)?(?<num>\d+)
+            (?<subfig>[a-z])?(?<key>_key\d+)?(?<lang>_[a-z])?$]x
+          .match(File.basename(img["src"], ".*"))
+        m.nil? and
+          @log.add("Style", img,
+                   "image name #{img['src']} does not match DRG requirements")
+        m
+      end
+
+      def image_name_validate1(i, prefix)
+        m = image_name_parse(i, prefix) or return
+        warn i["src"]
+        disjunct_error(i, i.at("./ancestor::table"), !m[:tab].nil?,
+                       "is under a table but is not so labelled",
+                       "is labelled as under a table but is not")
+        disjunct_error(i, i.at("./ancestor::annex"), !m[:annex].nil?,
+                       "is under an annex but is not so labelled",
+                       "is labelled as under an annex but is not")
+        disjunct_error(i, i.xpath("./ancestor::figure").size > 1, !m[:subfig].nil?,
+                       "does not have a subfigure letter but is a subfigure",
+                       "has a subfigure letter but is not a subfigure")
+        lang = image_name_suffix(i.document.root)
+        (m[:lang] || "_e") == lang or
+          @log.add("Style", i,
+                   "image name #{i['src']} expected to have suffix #{lang}")
+      end
+
+      # DRG directives 3.2
+      def image_name_validate(xmldoc)
+        prefix = image_name_prefix(xmldoc) or return
+        xmldoc.xpath("//image").each do |i|
+          next if i["src"].start_with?("data:")
+
+          if /^ISO_\d+_/.match?(File.basename(i["src"]))
+          elsif /^(SL)?#{prefix}fig/.match?(File.basename(i["src"]))
+            image_name_validate1(i, prefix)
+          else
+            @log.add("Style", i,
+                     "image name #{i['src']} does not match DRG requirements: expect #{prefix}fig")
+          end
+        end
+      end
+
+      def figure_validate(xmldoc)
+        image_name_validate(xmldoc)
+        subfigure_validate(xmldoc)
+      end
+    end
+  end
+end