metanorma-iso 1.10.6 → 2.0.3

data/lib/metanorma/iso/validate_image.rb

@@ -0,0 +1,97 @@
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      # DRG directives 3.7; but anticipated by standoc
+      def subfigure_validate(xmldoc)
+        xmldoc.xpath("//figure//figure").each do |f|
+          { footnote: "fn", note: "note", key: "dl" }.each do |k, v|
+            f.xpath(".//#{v}").each do |n|
+              @log.add("Style", n, "#{k} is not permitted in a subfigure")
+            end
+          end
+        end
+      end
+
+      def image_name_prefix(xmldoc)
+        std = xmldoc&.at("//bibdata/ext/structuredidentifier/project-number") or
+          return
+        num = xmldoc&.at("//bibdata/docnumber")&.text or return
+        ed = xmldoc&.at("//bibdata/edition")&.text || "1"
+        prefix = num
+        std["part"] and prefix += "-#{std['part']}"
+        prefix += "_ed#{ed}"
+        amd = std["amendment"] and prefix += "amd#{amd}"
+        prefix
+      end
+
+      def image_name_suffix(xmldoc)
+        case xmldoc&.at("//bibdata/language")&.text
+        when "fr" then "_f"
+        when "de" then "_d"
+        when "ru" then "_r"
+        when "es" then "_s"
+        when "ar" then "_a"
+        # when "en" then "_e"
+        else
+          "_e"
+        end
+      end
+
+      def disjunct_error(img, cond1, cond2, msg1, msg2)
+        cond1 && !cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg1}")
+        !cond1 && cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg2}")
+      end
+
+      def image_name_parse(img, prefix)
+        m = %r[(SL)?#{prefix}fig(?<tab>Tab)?(?<annex>[A-Z])?(Text)?(?<num>\d+)
+            (?<subfig>[a-z])?(?<key>_key\d+)?(?<lang>_[a-z])?$]x
+          .match(File.basename(img["src"], ".*"))
+        m.nil? and
+          @log.add("Style", img,
+                   "image name #{img['src']} does not match DRG requirements")
+        m
+      end
+
+      def image_name_validate1(i, prefix)
+        m = image_name_parse(i, prefix) or return
+        warn i["src"]
+        disjunct_error(i, i.at("./ancestor::table"), !m[:tab].nil?,
+                       "is under a table but is not so labelled",
+                       "is labelled as under a table but is not")
+        disjunct_error(i, i.at("./ancestor::annex"), !m[:annex].nil?,
+                       "is under an annex but is not so labelled",
+                       "is labelled as under an annex but is not")
+        disjunct_error(i, i.xpath("./ancestor::figure").size > 1, !m[:subfig].nil?,
+                       "does not have a subfigure letter but is a subfigure",
+                       "has a subfigure letter but is not a subfigure")
+        lang = image_name_suffix(i.document.root)
+        (m[:lang] || "_e") == lang or
+          @log.add("Style", i,
+                   "image name #{i['src']} expected to have suffix #{lang}")
+      end
+
+      # DRG directives 3.2
+      def image_name_validate(xmldoc)
+        prefix = image_name_prefix(xmldoc) or return
+        xmldoc.xpath("//image").each do |i|
+          next if i["src"].start_with?("data:")
+
+          if /^ISO_\d+_/.match?(File.basename(i["src"]))
+          elsif /^(SL)?#{prefix}fig/.match?(File.basename(i["src"]))
+            image_name_validate1(i, prefix)
+          else
+            @log.add("Style", i,
+                     "image name #{i['src']} does not match DRG requirements: expect #{prefix}fig")
+          end
+        end
+      end
+
+      def figure_validate(xmldoc)
+        image_name_validate(xmldoc)
+        subfigure_validate(xmldoc)
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_requirements.rb

@@ -0,0 +1,111 @@
+require "metanorma-standoc"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      REQUIREMENT_RE_STR = <<~REGEXP.freeze
+        \\b
+         ( shall | (is|are)_to |
+           (is|are)_required_(not_)?to |
+           (is|are)_required_that |
+           has_to |
+           only\\b[^.,]+\\b(is|are)_permitted |
+           it_is_necessary |
+           (is|are)_not_(allowed | permitted |
+                         acceptable | permissible) |
+           (is|are)_not_to_be |
+           [.,:;]_do_not )
+        \\b
+      REGEXP
+
+      def requirement_re
+        Regexp.new(self.class::REQUIREMENT_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def requirement_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if requirement_re.match t
+        end
+        nil
+      end
+
+      RECOMMENDATION_RE_STR = <<~REGEXP.freeze
+        \\b
+            should |
+            ought_(not_)?to |
+            it_is_(not_)?recommended_that
+        \\b
+      REGEXP
+
+      def recommendation_re
+        Regexp.new(self.class::RECOMMENDATION_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def recommendation_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if recommendation_re.match t
+        end
+        nil
+      end
+
+      PERMISSION_RE_STR = <<~REGEXP.freeze
+        \\b
+             may |
+            (is|are)_(permitted | allowed | permissible ) |
+            it_is_not_required_that |
+            no\\b[^.,]+\\b(is|are)_required
+        \\b
+      REGEXP
+
+      def permission_re
+        Regexp.new(self.class::PERMISSION_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def permission_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if permission_re.match t
+        end
+        nil
+      end
+
+      POSSIBILITY_RE_STR = <<~REGEXP.freeze
+        \\b
+           can | cannot | be_able_to |
+           there_is_a_possibility_of |
+           it_is_possible_to | be_unable_to |
+           there_is_no_possibility_of |
+           it_is_not_possible_to
+        \\b
+      REGEXP
+
+      def possibility_re
+        Regexp.new(self.class::POSSIBILITY_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def possibility_check(text)
+        text.split(/\.\s+/).each { |t| return t if possibility_re.match t }
+        nil
+      end
+
+      def external_constraint(text)
+        text.split(/\.\s+/).each do |t|
+          return t if /\b(must)\b/xi.match? t
+        end
+        nil
+      end
+
+      def style_no_guidance(node, text, docpart)
+        r = requirement_check(text)
+        style_warning(node, "#{docpart} may contain requirement", r) if r
+        r = permission_check(text)
+        style_warning(node, "#{docpart} may contain permission", r) if r
+        r = recommendation_check(text)
+        style_warning(node, "#{docpart} may contain recommendation", r) if r
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_section.rb

@@ -0,0 +1,248 @@
+require "nokogiri"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def section_validate(doc)
+        doctype = doc&.at("//bibdata/ext/doctype")&.text
+        unless %w(amendment technical-corrigendum).include? doctype
+          foreword_validate(doc.root)
+          normref_validate(doc.root)
+          symbols_validate(doc.root)
+          sections_presence_validate(doc.root)
+          sections_sequence_validate(doc.root)
+        end
+        section_style(doc.root)
+        subclause_validate(doc.root)
+        super
+      end
+
+      # ISO/IEC DIR 2, 12.4
+      def foreword_validate(root)
+        f = root.at("//foreword") || return
+        s = f.at("./clause")
+        @log.add("Style", f, "foreword contains subclauses") unless s.nil?
+      end
+
+      # ISO/IEC DIR 2, 15.4
+      def normref_validate(root)
+        f = root.at("//references[@normative = 'true']") || return
+        f.at("./references | ./clause") &&
+          @log.add("Style", f, "normative references contains subclauses")
+      end
+
+      ONE_SYMBOLS_WARNING = "Only one Symbols and Abbreviated "\
+                            "Terms section in the standard".freeze
+
+      NON_DL_SYMBOLS_WARNING = "Symbols and Abbreviated Terms can "\
+                               "only contain a definition list".freeze
+
+      def symbols_validate(root)
+        f = root.xpath("//definitions")
+        f.empty? && return
+        (f.size == 1 || @vocab) or
+          @log.add("Style", f.first, ONE_SYMBOLS_WARNING)
+        f.first.elements.each do |e|
+          unless %w(title dl).include? e.name
+            @log.add("Style", f.first, NON_DL_SYMBOLS_WARNING)
+            return
+          end
+        end
+      end
+
+      def seqcheck(names, msg, accepted)
+        n = names.shift
+        return [] if n.nil?
+
+        test = accepted.map { |a| n.at(a) }
+        if test.all?(&:nil?)
+          @log.add("Style", nil, msg)
+        end
+        names
+      end
+
+      def sections_presence_validate(root)
+        root.at("//sections/clause[@type = 'scope']") or
+          @log.add("Style", nil, "Scope clause missing")
+        root.at("//references[@normative = 'true']") or
+          @log.add("Style", nil, "Normative references missing")
+        root.at("//terms") or
+          @log.add("Style", nil, "Terms & definitions missing")
+      end
+
+      # spec of permissible section sequence
+      # we skip normative references, it goes to end of list
+      SEQ =
+        [
+          {
+            msg: "Initial section must be (content) Foreword",
+            val: ["./self::foreword"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::introduction", "./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Normative References must be followed by "\
+                 "Terms and Definitions",
+            val: ["./self::terms | .//terms"],
+          },
+        ].freeze
+
+      SECTIONS_XPATH =
+        "//foreword | //introduction | //sections/terms | .//annex | "\
+        "//sections/definitions | //sections/clause | "\
+        "//references[not(parent::clause)] | "\
+        "//clause[descendant::references][not(parent::clause)]".freeze
+
+      def sections_sequence_validate(root)
+        names, n = sections_sequence_validate_start(root)
+        if @vocab
+          names, n = sections_sequence_validate_body_vocab(names, n)
+        else
+          names, n = sections_sequence_validate_body(names, n)
+        end
+        sections_sequence_validate_end(names, n)
+      end
+
+      def sections_sequence_validate_start(root)
+        names = root.xpath(SECTIONS_XPATH)
+        names = seqcheck(names, SEQ[0][:msg], SEQ[0][:val])
+        n = names[0]
+        names = seqcheck(names, SEQ[1][:msg], SEQ[1][:val])
+        n&.at("./self::introduction") and
+          names = seqcheck(names, SEQ[2][:msg], SEQ[2][:val])
+        names = seqcheck(names, SEQ[3][:msg], SEQ[3][:val])
+        n = names.shift
+        n = names.shift if n&.at("./self::definitions")
+        [names, n]
+      end
+
+      def sections_sequence_validate_body(names, elem)
+        if elem.nil? || elem.name != "clause"
+          @log.add("Style", elem, "Document must contain at least one clause")
+        end
+        elem&.at("./self::clause") ||
+          @log.add("Style", elem, "Document must contain clause after "\
+                                  "Terms and Definitions")
+        elem&.at("./self::clause[@type = 'scope']") &&
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+        elem = names.shift
+        while elem&.name == "clause"
+          elem&.at("./self::clause[@type = 'scope']")
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_body_vocab(names, elem)
+        while elem && %w(clause terms).include?(elem.name)
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow terms and clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_end(names, elem)
+        while elem&.name == "annex"
+          elem = names.shift
+          if elem.nil?
+            @log.add("Style", nil, "Document must include (references) "\
+                                   "Normative References")
+          end
+        end
+        elem&.at("./self::references[@normative = 'true']") ||
+          @log.add("Style", nil, "Document must include (references) "\
+                                 "Normative References")
+        elem = names&.shift
+        elem&.at("./self::references[@normative = 'false']") ||
+          @log.add("Style", elem,
+                   "Final section must be (references) Bibliography")
+        names.empty? ||
+          @log.add("Style", elem,
+                   "There are sections after the final Bibliography")
+      end
+
+      NORM_ISO_WARN = "non-ISO/IEC reference not expected as normative".freeze
+      SCOPE_WARN = "Scope contains subclauses: should be succinct".freeze
+
+      def section_style(root)
+        foreword_style(root.at("//foreword"))
+        introduction_style(root.at("//introduction"))
+        scope_style(root.at("//clause[@type = 'scope']"))
+        scope = root.at("//clause[@type = 'scope']/clause")
+        # ISO/IEC DIR 2, 14.4
+        scope.nil? || style_warning(scope, SCOPE_WARN, nil)
+        tech_report_style(root)
+      end
+
+      def tech_report_style(root)
+        root.at("//bibdata/ext/doctype")&.text == "technical-report" or return
+        root.xpath("//sections/clause[not(@type = 'scope')] | //annex")
+          .each do |s|
+          r = requirement_check(extract_text(s)) and
+            style_warning(s,
+                          "Technical Report clause may contain requirement", r)
+        end
+      end
+
+      ASSETS_TO_STYLE =
+        "//termsource | //formula | //termnote | "\
+        "//p[not(ancestor::boilerplate)] | //li[not(p)] | //dt | "\
+        "//dd[not(p)] | //td[not(p)] | //th[not(p)]".freeze
+
+      NORM_BIBITEMS =
+        "//references[@normative = 'true']/bibitem".freeze
+
+      # ISO/IEC DIR 2, 10.2
+      def norm_bibitem_style(root)
+        root.xpath(NORM_BIBITEMS).each do |b|
+          if b.at(Standoc::Converter::ISO_PUBLISHER_XPATH).nil?
+            @log.add("Style", b, "#{NORM_ISO_WARN}: #{b.text}")
+          end
+        end
+      end
+
+      def asset_style(root)
+        root.xpath("//example | //termexample").each { |e| example_style(e) }
+        root.xpath("//definition/verbal-definition").each { |e| definition_style(e) }
+        root.xpath("//note").each { |e| note_style(e) }
+        root.xpath("//fn").each { |e| footnote_style(e) }
+        root.xpath(ASSETS_TO_STYLE).each { |e| style(e, extract_text(e)) }
+        norm_bibitem_style(root)
+        super
+      end
+
+      def subclause_validate(root)
+        root.xpath("//clause/clause/clause/clause/clause/clause/clause/clause")
+          .each do |c|
+          style_warning(c, "Exceeds the maximum clause depth of 7", nil)
+        end
+      end
+
+      # ISO/IEC DIR 2, 22.3.2
+      def onlychild_clause_validate(root)
+        root.xpath(Standoc::Utils::SUBCLAUSE_XPATH).each do |c|
+          next unless c.xpath("../clause").size == 1
+
+          title = c.at("./title")
+          location = c["id"] || "#{c.text[0..60]}..."
+          location += ":#{title.text}" if c["id"] && !title.nil?
+          @log.add("Style", nil, "#{location}: subclause is only child")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_style.rb

@@ -0,0 +1,170 @@
+require "metanorma-standoc"
+require "nokogiri"
+require "tokenizer"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def extract_text(node)
+        return "" if node.nil?
+
+        node1 = Nokogiri::XML.fragment(node.to_s)
+        node1.xpath("//link | //locality | //localityStack").each(&:remove)
+        ret = ""
+        node1.traverse { |x| ret += x.text if x.text? }
+        HTMLEntities.new.decode(ret)
+      end
+
+      # ISO/IEC DIR 2, 12.2
+      def foreword_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Foreword")
+      end
+
+      # ISO/IEC DIR 2, 14.2
+      def scope_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Scope")
+      end
+
+      # ISO/IEC DIR 2, 13.2
+      def introduction_style(node)
+        return if @novalid
+
+        r = requirement_check(extract_text(node))
+        style_warning(node, "Introduction may contain requirement", r) if r
+      end
+
+      # ISO/IEC DIR 2, 16.5.6
+      def definition_style(node)
+        return if @novalid
+
+        r = requirement_check(extract_text(node))
+        style_warning(node, "Definition may contain requirement", r) if r
+      end
+
+      # ISO/IEC DIR 2, 16.5.7
+      # ISO/IEC DIR 2, 25.5
+      def example_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Example")
+        style(node, extract_text(node))
+      end
+
+      # ISO/IEC DIR 2, 24.5
+      def note_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Note")
+        style(node, extract_text(node))
+      end
+
+      # ISO/IEC DIR 2, 26.5
+      def footnote_style(node)
+        return if @novalid
+
+        style_no_guidance(node, extract_text(node), "Footnote")
+        style(node, extract_text(node))
+      end
+
+      def style_regex(regex, warning, n, text)
+        (m = regex.match(text)) && style_warning(n, warning, m[:num])
+      end
+
+      # style check with a regex on a token
+      # and a negative match on its preceding token
+      def style_two_regex_not_prev(n, text, regex, re_prev, warning)
+        return if text.nil?
+
+        arr = Tokenizer::WhitespaceTokenizer.new.tokenize(text)
+        arr.each_index do |i|
+          m = regex.match arr[i]
+          m_prev = i.zero? ? nil : re_prev.match(arr[i - 1])
+          if !m.nil? && m_prev.nil?
+            style_warning(n, warning, m[:num])
+          end
+        end
+      end
+
+      def style(node, text)
+        return if @novalid
+
+        style_number(node, text)
+        style_percent(node, text)
+        style_abbrev(node, text)
+        style_units(node, text)
+      end
+
+      # ISO/IEC DIR 2, 9.1
+      # ISO/IEC DIR 2, Table B.1
+      def style_number(node, text)
+        style_two_regex_not_prev(
+          node, text, /^(?<num>-?[0-9]{4,}[,0-9]*)\Z/,
+          %r{\b(ISO|IEC|IEEE/|(in|January|February|March|April|May|June|August|September|October|November|December)\b)\Z},
+          "number not broken up in threes"
+        )
+        style_regex(/\b(?<num>[0-9]+\.[0-9]+)/i,
+                    "possible decimal point", node, text)
+        style_regex(/\b(?<num>billions?)\b/i,
+                    "ambiguous number", node, text)
+      end
+
+      # ISO/IEC DIR 2, 9.2.1
+      def style_percent(node, text)
+        style_regex(/\b(?<num>[0-9.,]+%)/,
+                    "no space before percent sign", node, text)
+        style_regex(/\b(?<num>[0-9.,]+ \u00b1 [0-9,.]+ %)/,
+                    "unbracketed tolerance before percent sign", node, text)
+      end
+
+      # ISO/IEC DIR 2, 8.4
+      # ISO/IEC DIR 2, 9.3
+      def style_abbrev(node, text)
+        style_regex(/(\A|\s)(?!e\.g\.|i\.e\.)
+                    (?<num>[a-z]{1,2}\.([a-z]{1,2}|\.))\b/ix,
+                    "no dots in abbreviations", node, text)
+        style_regex(/\b(?<num>ppm)\b/i,
+                    "language-specific abbreviation", node, text)
+      end
+
+      # leaving out as problematic: N J K C S T H h d B o E
+      SI_UNIT = "(m|cm|mm|km|μm|nm|g|kg|mgmol|cd|rad|sr|Hz|Hz|MHz|Pa|hPa|kJ|"\
+                "V|kV|W|MW|kW|F|μF|Ω|Wb|°C|lm|lx|Bq|Gy|Sv|kat|l|t|eV|u|Np|Bd|"\
+                "bit|kB|MB|Hart|nat|Sh|var)".freeze
+
+      # ISO/IEC DIR 2, 9.3
+      def style_units(node, text)
+        style_regex(/\b(?<num>[0-9][0-9,]*\s+[\u00b0\u2032\u2033])/,
+                    "space between number and degrees/minutes/seconds",
+                    node, text)
+        style_regex(/\b(?<num>[0-9][0-9,]*#{SI_UNIT})\b/o,
+                    "no space between number and SI unit", node, text)
+        style_non_std_units(node, text)
+      end
+
+      NONSTD_UNITS = {
+        sec: "s", mins: "min", hrs: "h", hr: "h", cc: "cm^3",
+        lit: "l", amp: "A", amps: "A", rpm: "r/min"
+      }.freeze
+
+      # ISO/IEC DIR 2, 9.3
+      def style_non_std_units(node, text)
+        NONSTD_UNITS.each do |k, v|
+          style_regex(/\b(?<num>[0-9][0-9,]*\s+#{k})\b/,
+                      "non-standard unit (should be #{v})", node, text)
+        end
+      end
+
+      def style_warning(node, msg, text = nil)
+        return if @novalid
+
+        w = msg
+        w += ": #{text}" if text
+        @log.add("Style", node, w)
+      end
+    end
+  end
+end