metanorma-iso 1.10.5 → 2.0.2

data/lib/metanorma/iso/validate_section.rb

@@ -0,0 +1,247 @@
+require "nokogiri"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def section_validate(doc)
+        doctype = doc&.at("//bibdata/ext/doctype")&.text
+        unless %w(amendment technical-corrigendum).include? doctype
+          foreword_validate(doc.root)
+          normref_validate(doc.root)
+          symbols_validate(doc.root)
+          sections_presence_validate(doc.root)
+          sections_sequence_validate(doc.root)
+        end
+        section_style(doc.root)
+        subclause_validate(doc.root)
+        super
+      end
+
+      # ISO/IEC DIR 2, 12.4
+      def foreword_validate(root)
+        f = root.at("//foreword") || return
+        s = f.at("./clause")
+        @log.add("Style", f, "foreword contains subclauses") unless s.nil?
+      end
+
+      # ISO/IEC DIR 2, 15.4
+      def normref_validate(root)
+        f = root.at("//references[@normative = 'true']") || return
+        f.at("./references | ./clause") &&
+          @log.add("Style", f, "normative references contains subclauses")
+      end
+
+      ONE_SYMBOLS_WARNING = "Only one Symbols and Abbreviated "\
+                            "Terms section in the standard".freeze
+
+      NON_DL_SYMBOLS_WARNING = "Symbols and Abbreviated Terms can "\
+                               "only contain a definition list".freeze
+
+      def symbols_validate(root)
+        f = root.xpath("//definitions")
+        f.empty? && return
+        (f.size == 1) || @log.add("Style", f.first, ONE_SYMBOLS_WARNING)
+        f.first.elements.each do |e|
+          unless %w(title dl).include? e.name
+            @log.add("Style", f.first, NON_DL_SYMBOLS_WARNING)
+            return
+          end
+        end
+      end
+
+      def seqcheck(names, msg, accepted)
+        n = names.shift
+        return [] if n.nil?
+
+        test = accepted.map { |a| n.at(a) }
+        if test.all?(&:nil?)
+          @log.add("Style", nil, msg)
+        end
+        names
+      end
+
+      def sections_presence_validate(root)
+        root.at("//sections/clause[@type = 'scope']") or
+          @log.add("Style", nil, "Scope clause missing")
+        root.at("//references[@normative = 'true']") or
+          @log.add("Style", nil, "Normative references missing")
+        root.at("//terms") or
+          @log.add("Style", nil, "Terms & definitions missing")
+      end
+
+      # spec of permissible section sequence
+      # we skip normative references, it goes to end of list
+      SEQ =
+        [
+          {
+            msg: "Initial section must be (content) Foreword",
+            val: ["./self::foreword"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::introduction", "./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Normative References must be followed by "\
+                 "Terms and Definitions",
+            val: ["./self::terms | .//terms"],
+          },
+        ].freeze
+
+      SECTIONS_XPATH =
+        "//foreword | //introduction | //sections/terms | .//annex | "\
+        "//sections/definitions | //sections/clause | "\
+        "//references[not(parent::clause)] | "\
+        "//clause[descendant::references][not(parent::clause)]".freeze
+
+      def sections_sequence_validate(root)
+        names, n = sections_sequence_validate_start(root)
+        if root&.at("//bibdata/ext/subdoctype")&.text == "vocabulary"
+          names, n = sections_sequence_validate_body_vocab(names, n)
+        else
+          names, n = sections_sequence_validate_body(names, n)
+        end
+        sections_sequence_validate_end(names, n)
+      end
+
+      def sections_sequence_validate_start(root)
+        names = root.xpath(SECTIONS_XPATH)
+        names = seqcheck(names, SEQ[0][:msg], SEQ[0][:val])
+        n = names[0]
+        names = seqcheck(names, SEQ[1][:msg], SEQ[1][:val])
+        n&.at("./self::introduction") and
+          names = seqcheck(names, SEQ[2][:msg], SEQ[2][:val])
+        names = seqcheck(names, SEQ[3][:msg], SEQ[3][:val])
+        n = names.shift
+        n = names.shift if n&.at("./self::definitions")
+        [names, n]
+      end
+
+      def sections_sequence_validate_body(names, elem)
+        if elem.nil? || elem.name != "clause"
+          @log.add("Style", elem, "Document must contain at least one clause")
+        end
+        elem&.at("./self::clause") ||
+          @log.add("Style", elem, "Document must contain clause after "\
+                                  "Terms and Definitions")
+        elem&.at("./self::clause[@type = 'scope']") &&
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+        elem = names.shift
+        while elem&.name == "clause"
+          elem&.at("./self::clause[@type = 'scope']")
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_body_vocab(names, elem)
+        while elem && %w(clause terms).include?(elem.name)
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow terms and clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_end(names, elem)
+        while elem&.name == "annex"
+          elem = names.shift
+          if elem.nil?
+            @log.add("Style", nil, "Document must include (references) "\
+                                   "Normative References")
+          end
+        end
+        elem&.at("./self::references[@normative = 'true']") ||
+          @log.add("Style", nil, "Document must include (references) "\
+                                 "Normative References")
+        elem = names&.shift
+        elem&.at("./self::references[@normative = 'false']") ||
+          @log.add("Style", elem,
+                   "Final section must be (references) Bibliography")
+        names.empty? ||
+          @log.add("Style", elem,
+                   "There are sections after the final Bibliography")
+      end
+
+      NORM_ISO_WARN = "non-ISO/IEC reference not expected as normative".freeze
+      SCOPE_WARN = "Scope contains subclauses: should be succinct".freeze
+
+      def section_style(root)
+        foreword_style(root.at("//foreword"))
+        introduction_style(root.at("//introduction"))
+        scope_style(root.at("//clause[@type = 'scope']"))
+        scope = root.at("//clause[@type = 'scope']/clause")
+        # ISO/IEC DIR 2, 14.4
+        scope.nil? || style_warning(scope, SCOPE_WARN, nil)
+        tech_report_style(root)
+      end
+
+      def tech_report_style(root)
+        root.at("//bibdata/ext/doctype")&.text == "technical-report" or return
+        root.xpath("//sections/clause[not(@type = 'scope')] | //annex")
+          .each do |s|
+          r = requirement_check(extract_text(s)) and
+            style_warning(s,
+                          "Technical Report clause may contain requirement", r)
+        end
+      end
+
+      ASSETS_TO_STYLE =
+        "//termsource | //formula | //termnote | "\
+        "//p[not(ancestor::boilerplate)] | //li[not(p)] | //dt | "\
+        "//dd[not(p)] | //td[not(p)] | //th[not(p)]".freeze
+
+      NORM_BIBITEMS =
+        "//references[@normative = 'true']/bibitem".freeze
+
+      # ISO/IEC DIR 2, 10.2
+      def norm_bibitem_style(root)
+        root.xpath(NORM_BIBITEMS).each do |b|
+          if b.at(Standoc::Converter::ISO_PUBLISHER_XPATH).nil?
+            @log.add("Style", b, "#{NORM_ISO_WARN}: #{b.text}")
+          end
+        end
+      end
+
+      def asset_style(root)
+        root.xpath("//example | //termexample").each { |e| example_style(e) }
+        root.xpath("//definition/verbal-definition").each { |e| definition_style(e) }
+        root.xpath("//note").each { |e| note_style(e) }
+        root.xpath("//fn").each { |e| footnote_style(e) }
+        root.xpath(ASSETS_TO_STYLE).each { |e| style(e, extract_text(e)) }
+        norm_bibitem_style(root)
+        super
+      end
+
+      def subclause_validate(root)
+        root.xpath("//clause/clause/clause/clause/clause/clause/clause/clause")
+          .each do |c|
+          style_warning(c, "Exceeds the maximum clause depth of 7", nil)
+        end
+      end
+
+      # ISO/IEC DIR 2, 22.3.2
+      def onlychild_clause_validate(root)
+        root.xpath(Standoc::Utils::SUBCLAUSE_XPATH).each do |c|
+          next unless c.xpath("../clause").size == 1
+
+          title = c.at("./title")
+          location = c["id"] || "#{c.text[0..60]}..."
+          location += ":#{title.text}" if c["id"] && !title.nil?
+          @log.add("Style", nil, "#{location}: subclause is only child")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_style.rb

@@ -0,0 +1,170 @@
+require "metanorma-standoc"
+require "nokogiri"
+require "tokenizer"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def extract_text(node)
+        return "" if node.nil?
+
+        node1 = Nokogiri::XML.fragment(node.to_s)
+        node1.xpath("//link | //locality | //localityStack").each(&:remove)
+        ret = ""
+        node1.traverse { |x| ret += x.text if x.text? }
+        HTMLEntities.new.decode(ret)
+      end
+
+      # ISO/IEC DIR 2, 12.2
+      def foreword_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Foreword")
+      end
+
+      # ISO/IEC DIR 2, 14.2
+      def scope_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Scope")
+      end
+
+      # ISO/IEC DIR 2, 13.2
+      def introduction_style(node)
+        return if @novalid
+
+        r = requirement_check(extract_text(node))
+        style_warning(node, "Introduction may contain requirement", r) if r
+      end
+
+      # ISO/IEC DIR 2, 16.5.6
+      def definition_style(node)
+        return if @novalid
+
+        r = requirement_check(extract_text(node))
+        style_warning(node, "Definition may contain requirement", r) if r
+      end
+
+      # ISO/IEC DIR 2, 16.5.7
+      # ISO/IEC DIR 2, 25.5
+      def example_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Example")
+        style(node, extract_text(node))
+      end
+
+      # ISO/IEC DIR 2, 24.5
+      def note_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Note")
+        style(node, extract_text(node))
+      end
+
+      # ISO/IEC DIR 2, 26.5
+      def footnote_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Footnote")
+        style(node, extract_text(node))
+      end
+
+      def style_regex(regex, warning, n, text)
+        (m = regex.match(text)) && style_warning(n, warning, m[:num])
+      end
+
+      # style check with a regex on a token
+      # and a negative match on its preceding token
+      def style_two_regex_not_prev(n, text, regex, re_prev, warning)
+        return if text.nil?
+
+        arr = Tokenizer::WhitespaceTokenizer.new.tokenize(text)
+        arr.each_index do |i|
+          m = regex.match arr[i]
+          m_prev = i.zero? ? nil : re_prev.match(arr[i - 1])
+          if !m.nil? && m_prev.nil?
+            style_warning(n, warning, m[:num])
+          end
+        end
+      end
+
+      def style(node, text)
+        return if @novalid
+
+        style_number(node, text)
+        style_percent(node, text)
+        style_abbrev(node, text)
+        style_units(node, text)
+      end
+
+      # ISO/IEC DIR 2, 9.1
+      # ISO/IEC DIR 2, Table B.1
+      def style_number(node, text)
+        style_two_regex_not_prev(
+          node, text, /^(?<num>-?[0-9]{4,}[,0-9]*)\Z/,
+          %r{\b(ISO|IEC|IEEE/|(in|January|February|March|April|May|June|August|September|October|November|December)\b)\Z},
+          "number not broken up in threes"
+        )
+        style_regex(/\b(?<num>[0-9]+\.[0-9]+)/i,
+                    "possible decimal point", node, text)
+        style_regex(/\b(?<num>billions?)\b/i,
+                    "ambiguous number", node, text)
+      end
+
+      # ISO/IEC DIR 2, 9.2.1
+      def style_percent(node, text)
+        style_regex(/\b(?<num>[0-9.,]+%)/,
+                    "no space before percent sign", node, text)
+        style_regex(/\b(?<num>[0-9.,]+ \u00b1 [0-9,.]+ %)/,
+                    "unbracketed tolerance before percent sign", node, text)
+      end
+
+      # ISO/IEC DIR 2, 8.4
+      # ISO/IEC DIR 2, 9.3
+      def style_abbrev(node, text)
+        style_regex(/(\A|\s)(?!e\.g\.|i\.e\.)
+                    (?<num>[a-z]{1,2}\.([a-z]{1,2}|\.))\b/ix,
+                    "no dots in abbreviations", node, text)
+        style_regex(/\b(?<num>ppm)\b/i,
+                    "language-specific abbreviation", node, text)
+      end
+
+      # leaving out as problematic: N J K C S T H h d B o E
+      SI_UNIT = "(m|cm|mm|km|μm|nm|g|kg|mgmol|cd|rad|sr|Hz|Hz|MHz|Pa|hPa|kJ|"\
+                "V|kV|W|MW|kW|F|μF|Ω|Wb|°C|lm|lx|Bq|Gy|Sv|kat|l|t|eV|u|Np|Bd|"\
+                "bit|kB|MB|Hart|nat|Sh|var)".freeze
+
+      # ISO/IEC DIR 2, 9.3
+      def style_units(node, text)
+        style_regex(/\b(?<num>[0-9][0-9,]*\s+[\u00b0\u2032\u2033])/,
+                    "space between number and degrees/minutes/seconds",
+                    node, text)
+        style_regex(/\b(?<num>[0-9][0-9,]*#{SI_UNIT})\b/o,
+                    "no space between number and SI unit", node, text)
+        style_non_std_units(node, text)
+      end
+
+      NONSTD_UNITS = {
+        sec: "s", mins: "min", hrs: "h", hr: "h", cc: "cm^3",
+        lit: "l", amp: "A", amps: "A", rpm: "r/min"
+      }.freeze
+
+      # ISO/IEC DIR 2, 9.3
+      def style_non_std_units(node, text)
+        NONSTD_UNITS.each do |k, v|
+          style_regex(/\b(?<num>[0-9][0-9,]*\s+#{k})\b/,
+                      "non-standard unit (should be #{v})", node, text)
+        end
+      end
+
+      def style_warning(node, msg, text = nil)
+        return if @novalid
+
+        w = msg
+        w += ": #{text}" if text
+        @log.add("Style", node, w)
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_title.rb

@@ -0,0 +1,105 @@
+require "metanorma-standoc"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def title_lang_part(doc, part, lang)
+        doc.at("//bibdata/title[@type='title-#{part}' and @language='#{lang}']")
+      end
+
+      def title_intro_validate(root)
+        title_intro_en = title_lang_part(root, "intro", "en")
+        title_intro_fr = title_lang_part(root, "intro", "fr")
+        if title_intro_en.nil? && !title_intro_fr.nil?
+          @log.add("Style", title_intro_fr, "No English Title Intro!")
+        end
+        if !title_intro_en.nil? && title_intro_fr.nil?
+          @log.add("Style", title_intro_en, "No French Title Intro!")
+        end
+      end
+
+      def title_main_validate(root)
+        title_main_en = title_lang_part(root, "main", "en")
+        title_main_fr = title_lang_part(root, "main", "fr")
+        if title_main_en.nil? && !title_main_fr.nil?
+          @log.add("Style", title_main_fr, "No English Title!")
+        end
+        if !title_main_en.nil? && title_main_fr.nil?
+          @log.add("Style", title_main_en, "No French Title!")
+        end
+      end
+
+      def title_part_validate(root)
+        title_part_en = title_lang_part(root, "part", "en")
+        title_part_fr = title_lang_part(root, "part", "fr")
+        (title_part_en.nil? && !title_part_fr.nil?) &&
+          @log.add("Style", title_part_fr, "No English Title Part!")
+        (!title_part_en.nil? && title_part_fr.nil?) &&
+          @log.add("Style", title_part_en, "No French Title Part!")
+      end
+
+      # ISO/IEC DIR 2, 11.4
+      def title_subpart_validate(root)
+        docid = root.at("//bibdata/docidentifier[@type = 'ISO']")
+        subpart = /-\d+-\d+/.match docid
+        iec = root.at("//bibdata/contributor[role/@type = 'publisher']/"\
+                      "organization[abbreviation = 'IEC' or "\
+                      "name = 'International Electrotechnical Commission']")
+        subpart && !iec and
+          @log.add("Style", docid, "Subpart defined on non-IEC document!")
+      end
+
+      # ISO/IEC DIR 2, 11.5.2
+      def title_names_type_validate(root)
+        doctypes = /International\sStandard | Technical\sSpecification |
+        Publicly\sAvailable\sSpecification | Technical\sReport | Guide /xi
+        title_main_en = title_lang_part(root, "main", "en")
+        !title_main_en.nil? && doctypes.match(title_main_en.text) and
+          @log.add("Style", title_main_en, "Main Title may name document type")
+        title_intro_en = title_lang_part(root, "intro", "en")
+        !title_intro_en.nil? && doctypes.match(title_intro_en.text) and
+          @log.add("Style", title_intro_en,
+                   "Title Intro may name document type")
+      end
+
+      # ISO/IEC DIR 2, 22.2
+      def title_first_level_validate(root)
+        root.xpath(SECTIONS_XPATH).each do |s|
+          title = s&.at("./title")&.text || s.name
+          s.xpath("./clause | ./terms | ./references").each do |ss|
+            subtitle = ss.at("./title")
+            !subtitle.nil? && !subtitle&.text&.empty? or
+              @log.add("Style", ss,
+                       "#{title}: each first-level subclause must have a title")
+          end
+        end
+      end
+
+      # ISO/IEC DIR 2, 22.2
+      def title_all_siblings(xpath, label)
+        notitle = false
+        withtitle = false
+        xpath.each do |s|
+          title_all_siblings(s.xpath("./clause | ./terms | ./references"),
+                             s&.at("./title")&.text || s["id"])
+          subtitle = s.at("./title")
+          notitle = notitle || (!subtitle || subtitle.text.empty?)
+          withtitle = withtitle || (subtitle && !subtitle.text.empty?)
+        end
+        notitle && withtitle &&
+          @log.add("Style", nil,
+                   "#{label}: all subclauses must have a title, or none")
+      end
+
+      def title_validate(root)
+        title_intro_validate(root)
+        title_main_validate(root)
+        title_part_validate(root)
+        title_subpart_validate(root)
+        title_names_type_validate(root)
+        title_first_level_validate(root)
+        title_all_siblings(root.xpath(SECTIONS_XPATH), "(top level)")
+      end
+    end
+  end
+end

data/lib/metanorma/iso/version.rb

@@ -1,5 +1,5 @@
 module Metanorma
   module ISO
-    VERSION = "1.10.5".freeze
+    VERSION = "2.0.2".freeze
   end
 end

data/lib/metanorma-iso.rb

@@ -1,5 +1,5 @@
 require "asciidoctor" unless defined? Asciidoctor::Converter
-require_relative "asciidoctor/iso/converter"
+require_relative "metanorma/iso/converter"
 require_relative "metanorma/iso/version"
 require_relative "isodoc/iso/html_convert"
 require_relative "isodoc/iso/word_convert"

data/metanorma-iso.gemspec

@@ -29,8 +29,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
-  spec.add_dependency "metanorma-standoc", "~> 1.11.0"
-  spec.add_dependency "mnconvert", "~> 2.0"
+  spec.add_dependency "metanorma-standoc", "~> 2.0.0"
+  spec.add_dependency "mnconvert", "~> 1.14"
   spec.add_dependency "ruby-jing"
   spec.add_dependency "tokenizer", "~> 0.3.0"
   spec.add_dependency "twitter_cldr"

data/spec/isodoc/ref_spec.rb

@@ -272,6 +272,7 @@ RSpec.describe IsoDoc do
             </note>
             <bibitem id="ISO3696" type="standard">
               <title format="text/plain">Water for analytical laboratory use</title>
+              <docidentifier type='metanorma-ordinal'>[1]</docidentifier>
               <docidentifier type="ISO">ISO 3696</docidentifier>
               <contributor>
                 <role type="publisher"/>
@@ -294,6 +295,7 @@ RSpec.describe IsoDoc do
             </bibitem>
             <bibitem id="ref11">
               <title>Internet Calendaring and Scheduling Core Object Specification (iCalendar)</title>
+              <docidentifier type='metanorma-ordinal'>[2]</docidentifier>
               <docidentifier type="IETF">IETF RFC 10</docidentifier>
             </bibitem>
             <bibitem id="ref12">
@@ -361,7 +363,7 @@ RSpec.describe IsoDoc do
                 <p>
                   <span class="note_label">NOTE</span>&#160; This is another annotation of document ISSN.</p>
               </div>
-              <p class="Biblio" id="ISO3696">[3]&#160; ISO 3696,
+              <p class="Biblio" id="ISO3696">[1]&#160; ISO 3696,
                 <i>Water for analytical laboratory use</i></p>
               <p class="Biblio" id="ref10">[10]&#160; <span style="font-variant:small-caps;">Standard No I.C.C 167</span>
                  .
@@ -369,7 +371,7 @@ RSpec.describe IsoDoc do
                 (see
                 <a href="http://www.icc.or.at">http://www.icc.or.at</a>
                 )</p>
-              <p class="Biblio" id="ref11">[5]&#160; IETF RFC 10,
+              <p class="Biblio" id="ref11">[2]&#160; IETF RFC 10,
                 <i>Internet Calendaring and Scheduling Core Object Specification (iCalendar)</i></p>
               <p class="Biblio" id="ref12">Citn&#160; IETF RFC 20, CitationWorks. 2019.
                 <i>How to cite a reference</i>

data/spec/isodoc/section_spec.rb

@@ -565,7 +565,7 @@ RSpec.describe IsoDoc do
              <substage language=''>60</substage>
            </status>
            <copyright>
-             <from>2021</from>
+             <from>#{Time.new.year}</from>
              <owner>
                <organization>
                  <name>International Organization for Standardization</name>

data/spec/{asciidoctor → metanorma}/amd_spec.rb

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-RSpec.describe Asciidoctor::ISO do
+RSpec.describe Metanorma::ISO do
   it "processes amendment sections" do
     expect(xmlpp(strip_guid(Asciidoctor.convert(<<~"INPUT", *OPTIONS)))).to be_equivalent_to xmlpp(<<~"OUTPUT")
       #{AMD_BLANK_HDR}