metanorma-iso 1.10.6 → 2.0.0

data/lib/metanorma/iso/section.rb

@@ -0,0 +1,49 @@
+require "htmlentities"
+require "uri" if /^2\./.match?(RUBY_VERSION)
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def clause_parse(attrs, xml, node)
+        node.option? "appendix" and return appendix_parse(attrs, xml, node)
+        super
+      end
+
+      def scope_parse(attrs, xml, node)
+        attrs = attrs.merge(type: "scope") unless @amd
+        clause_parse(attrs, xml, node)
+      end
+
+      def appendix_parse(attrs, xml, node)
+        attrs[:"inline-header"] = node.option? "inline-header"
+        set_obligation(attrs, node)
+        xml.appendix **attr_code(attrs) do |xml_section|
+          xml_section.title { |name| name << node.title }
+          xml_section << node.content
+        end
+      end
+
+      def patent_notice_parse(xml, node)
+        # xml.patent_notice do |xml_section|
+        #  xml_section << node.content
+        # end
+        xml << node.content
+      end
+
+      def sectiontype(node, level = true)
+        return nil if @amd
+
+        ret = sectiontype_streamline(sectiontype1(node))
+        return ret if ret == "terms and definitions" && @vocab
+
+        super
+      end
+
+      def term_def_subclause_parse(attrs, xml, node)
+        node.role == "term" and
+          return term_def_subclause_parse1(attrs, xml, node)
+        super
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate.rb

@@ -0,0 +1,172 @@
+require "metanorma-standoc"
+require_relative "./validate_style"
+require_relative "./validate_requirements"
+require_relative "./validate_section"
+require_relative "./validate_title"
+require_relative "./validate_image"
+require "nokogiri"
+require "jing"
+require "iev"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def isosubgroup_validate(root)
+        root.xpath("//technical-committee/@type").each do |t|
+          unless %w{TC PC JTC JPC}.include? t.text
+            @log.add("Document Attributes", nil,
+                     "invalid technical committee type #{t}")
+          end
+        end
+        root.xpath("//subcommittee/@type").each do |t|
+          unless %w{SC JSC}.include? t.text
+            @log.add("Document Attributes", nil,
+                     "invalid subcommittee type #{t}")
+          end
+        end
+      end
+
+      # ISO/IEC DIR 2, 15.5.3
+      # does not deal with preceding text marked up
+      def see_xrefs_validate(root)
+        root.xpath("//xref").each do |t|
+          preceding = t.at("./preceding-sibling::text()[last()]")
+          next unless !preceding.nil? &&
+            /\b(see| refer to)\s*\Z/mi.match(preceding)
+
+          (target = root.at("//*[@id = '#{t['target']}']")) || next
+          if target&.at("./ancestor-or-self::*[@obligation = 'normative']")
+            @log.add("Style", t,
+                     "'see #{t['target']}' is pointing to a normative section")
+          end
+        end
+      end
+
+      # ISO/IEC DIR 2, 15.5.3
+      def see_erefs_validate(root)
+        root.xpath("//eref").each do |t|
+          prec = t.at("./preceding-sibling::text()[last()]")
+          next unless !prec.nil? && /\b(see|refer to)\s*\Z/mi.match(prec)
+
+          unless target = root.at("//*[@id = '#{t['bibitemid']}']")
+            @log.add("Bibliography", t,
+                     "'#{t} is not pointing to a real reference")
+            next
+          end
+          target.at("./ancestor::references[@normative = 'true']") and
+            @log.add("Style", t,
+                     "'see #{t}' is pointing to a normative reference")
+        end
+      end
+
+      # ISO/IEC DIR 2, 10.4
+      def locality_erefs_validate(root)
+        root.xpath("//eref[descendant::locality]").each do |t|
+          if /^(ISO|IEC)/.match?(t["citeas"]) &&
+              !/: ?(\d+{4}|–)$/.match?(t["citeas"])
+            @log.add("Style", t,
+                     "undated reference #{t['citeas']} should not contain "\
+                     "specific elements")
+          end
+        end
+      end
+
+      def termdef_warn(text, regex, elem, term, msg)
+        regex.match(text) && @log.add("Style", elem, "#{term}: #{msg}")
+      end
+
+      # ISO/IEC DIR 2, 16.5.6
+      def termdef_style(xmldoc)
+        xmldoc.xpath("//term").each do |t|
+          para = t.at("./definition/verbal-definition") || return
+          term = t.at("./preferred//name").text
+          termdef_warn(para.text, /\A(the|a)\b/i, t, term,
+                       "term definition starts with article")
+          termdef_warn(para.text, /\.\Z/i, t, term,
+                       "term definition ends with period")
+        end
+      end
+
+      def doctype_validate(xmldoc)
+        doctype = xmldoc&.at("//bibdata/ext/doctype")&.text
+        %w(international-standard technical-specification technical-report
+           publicly-available-specification international-workshop-agreement
+           guide amendment technical-corrigendum).include? doctype or
+          @log.add("Document Attributes", nil,
+                   "#{doctype} is not a recognised document type")
+      end
+
+      def script_validate(xmldoc)
+        script = xmldoc&.at("//bibdata/script")&.text
+        script == "Latn" or
+          @log.add("Document Attributes", nil,
+                   "#{script} is not a recognised script")
+      end
+
+      def stage_validate(xmldoc)
+        stage = xmldoc&.at("//bibdata/status/stage")&.text
+        %w(00 10 20 30 40 50 60 90 95).include? stage or
+          @log.add("Document Attributes", nil,
+                   "#{stage} is not a recognised stage")
+      end
+
+      def substage_validate(xmldoc)
+        substage = xmldoc&.at("//bibdata/status/substage")&.text or return
+        %w(00 20 60 90 92 93 98 99).include? substage or
+          @log.add("Document Attributes", nil,
+                   "#{substage} is not a recognised substage")
+      end
+
+      def iteration_validate(xmldoc)
+        iteration = xmldoc&.at("//bibdata/status/iteration")&.text or return
+        /^\d+/.match(iteration) or
+          @log.add("Document Attributes", nil,
+                   "#{iteration} is not a recognised iteration")
+      end
+
+      def bibdata_validate(doc)
+        doctype_validate(doc)
+        script_validate(doc)
+        stage_validate(doc)
+        substage_validate(doc)
+        iteration_validate(doc)
+      end
+
+      def content_validate(doc)
+        super
+        title_validate(doc.root)
+        isosubgroup_validate(doc.root)
+        onlychild_clause_validate(doc.root)
+        termdef_style(doc.root)
+        see_xrefs_validate(doc.root)
+        see_erefs_validate(doc.root)
+        locality_erefs_validate(doc.root)
+        bibdata_validate(doc.root)
+        bibitem_validate(doc.root)
+        figure_validate(doc.root)
+      end
+
+      def bibitem_validate(xmldoc)
+        xmldoc.xpath("//bibitem[date/on = '–']").each do |b|
+          b.at("./note[@type = 'Unpublished-Status']") or
+            @log.add("Style", b,
+                     "Reference #{b&.at('./@id')&.text} does not have an "\
+                     "associated footnote indicating unpublished status")
+        end
+      end
+
+      def validate(doc)
+        content_validate(doc)
+        doctype = doc&.at("//bibdata/ext/doctype")&.text
+        schema = case doctype
+                 when "amendment", "technical-corrigendum" # @amd
+                   "isostandard-amd.rng"
+                 else
+                   "isostandard.rng"
+                 end
+        schema_validate(formattedstr_strip(doc.dup),
+                        File.join(File.dirname(__FILE__), schema))
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_image.rb

@@ -0,0 +1,97 @@
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      # DRG directives 3.7; but anticipated by standoc
+      def subfigure_validate(xmldoc)
+        xmldoc.xpath("//figure//figure").each do |f|
+          { footnote: "fn", note: "note", key: "dl" }.each do |k, v|
+            f.xpath(".//#{v}").each do |n|
+              @log.add("Style", n, "#{k} is not permitted in a subfigure")
+            end
+          end
+        end
+      end
+
+      def image_name_prefix(xmldoc)
+        std = xmldoc&.at("//bibdata/ext/structuredidentifier/project-number") or
+          return
+        num = xmldoc&.at("//bibdata/docnumber")&.text or return
+        ed = xmldoc&.at("//bibdata/edition")&.text || "1"
+        prefix = num
+        std["part"] and prefix += "-#{std['part']}"
+        prefix += "_ed#{ed}"
+        amd = std["amendment"] and prefix += "amd#{amd}"
+        prefix
+      end
+
+      def image_name_suffix(xmldoc)
+        case xmldoc&.at("//bibdata/language")&.text
+        when "fr" then "_f"
+        when "de" then "_d"
+        when "ru" then "_r"
+        when "es" then "_s"
+        when "ar" then "_a"
+        # when "en" then "_e"
+        else
+          "_e"
+        end
+      end
+
+      def disjunct_error(img, cond1, cond2, msg1, msg2)
+        cond1 && !cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg1}")
+        !cond1 && cond2 and
+          @log.add("Style", img, "image name #{img['src']} #{msg2}")
+      end
+
+      def image_name_parse(img, prefix)
+        m = %r[(SL)?#{prefix}fig(?<tab>Tab)?(?<annex>[A-Z])?(Text)?(?<num>\d+)
+            (?<subfig>[a-z])?(?<key>_key\d+)?(?<lang>_[a-z])?$]x
+          .match(File.basename(img["src"], ".*"))
+        m.nil? and
+          @log.add("Style", img,
+                   "image name #{img['src']} does not match DRG requirements")
+        m
+      end
+
+      def image_name_validate1(i, prefix)
+        m = image_name_parse(i, prefix) or return
+        warn i["src"]
+        disjunct_error(i, i.at("./ancestor::table"), !m[:tab].nil?,
+                       "is under a table but is not so labelled",
+                       "is labelled as under a table but is not")
+        disjunct_error(i, i.at("./ancestor::annex"), !m[:annex].nil?,
+                       "is under an annex but is not so labelled",
+                       "is labelled as under an annex but is not")
+        disjunct_error(i, i.xpath("./ancestor::figure").size > 1, !m[:subfig].nil?,
+                       "does not have a subfigure letter but is a subfigure",
+                       "has a subfigure letter but is not a subfigure")
+        lang = image_name_suffix(i.document.root)
+        (m[:lang] || "_e") == lang or
+          @log.add("Style", i,
+                   "image name #{i['src']} expected to have suffix #{lang}")
+      end
+
+      # DRG directives 3.2
+      def image_name_validate(xmldoc)
+        prefix = image_name_prefix(xmldoc) or return
+        xmldoc.xpath("//image").each do |i|
+          next if i["src"].start_with?("data:")
+
+          if /^ISO_\d+_/.match?(File.basename(i["src"]))
+          elsif /^(SL)?#{prefix}fig/.match?(File.basename(i["src"]))
+            image_name_validate1(i, prefix)
+          else
+            @log.add("Style", i,
+                     "image name #{i['src']} does not match DRG requirements: expect #{prefix}fig")
+          end
+        end
+      end
+
+      def figure_validate(xmldoc)
+        image_name_validate(xmldoc)
+        subfigure_validate(xmldoc)
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_requirements.rb

@@ -0,0 +1,111 @@
+require "metanorma-standoc"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      REQUIREMENT_RE_STR = <<~REGEXP.freeze
+        \\b
+         ( shall | (is|are)_to |
+           (is|are)_required_(not_)?to |
+           (is|are)_required_that |
+           has_to |
+           only\\b[^.,]+\\b(is|are)_permitted |
+           it_is_necessary |
+           (is|are)_not_(allowed | permitted |
+                         acceptable | permissible) |
+           (is|are)_not_to_be |
+           [.,:;]_do_not )
+        \\b
+      REGEXP
+
+      def requirement_re
+        Regexp.new(self.class::REQUIREMENT_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def requirement_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if requirement_re.match t
+        end
+        nil
+      end
+
+      RECOMMENDATION_RE_STR = <<~REGEXP.freeze
+        \\b
+            should |
+            ought_(not_)?to |
+            it_is_(not_)?recommended_that
+        \\b
+      REGEXP
+
+      def recommendation_re
+        Regexp.new(self.class::RECOMMENDATION_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def recommendation_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if recommendation_re.match t
+        end
+        nil
+      end
+
+      PERMISSION_RE_STR = <<~REGEXP.freeze
+        \\b
+             may |
+            (is|are)_(permitted | allowed | permissible ) |
+            it_is_not_required_that |
+            no\\b[^.,]+\\b(is|are)_required
+        \\b
+      REGEXP
+
+      def permission_re
+        Regexp.new(self.class::PERMISSION_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def permission_check(text)
+        text.split(/\.\s+/).each do |t|
+          return t if permission_re.match t
+        end
+        nil
+      end
+
+      POSSIBILITY_RE_STR = <<~REGEXP.freeze
+        \\b
+           can | cannot | be_able_to |
+           there_is_a_possibility_of |
+           it_is_possible_to | be_unable_to |
+           there_is_no_possibility_of |
+           it_is_not_possible_to
+        \\b
+      REGEXP
+
+      def possibility_re
+        Regexp.new(self.class::POSSIBILITY_RE_STR.gsub(/\s/, "")
+          .gsub(/_/, "\\s"), Regexp::IGNORECASE)
+      end
+
+      def possibility_check(text)
+        text.split(/\.\s+/).each { |t| return t if possibility_re.match t }
+        nil
+      end
+
+      def external_constraint(text)
+        text.split(/\.\s+/).each do |t|
+          return t if /\b(must)\b/xi.match? t
+        end
+        nil
+      end
+
+      def style_no_guidance(node, text, docpart)
+        r = requirement_check(text)
+        style_warning(node, "#{docpart} may contain requirement", r) if r
+        r = permission_check(text)
+        style_warning(node, "#{docpart} may contain permission", r) if r
+        r = recommendation_check(text)
+        style_warning(node, "#{docpart} may contain recommendation", r) if r
+      end
+    end
+  end
+end

data/lib/metanorma/iso/validate_section.rb

@@ -0,0 +1,247 @@
+require "nokogiri"
+
+module Metanorma
+  module ISO
+    class Converter < Standoc::Converter
+      def section_validate(doc)
+        doctype = doc&.at("//bibdata/ext/doctype")&.text
+        unless %w(amendment technical-corrigendum).include? doctype
+          foreword_validate(doc.root)
+          normref_validate(doc.root)
+          symbols_validate(doc.root)
+          sections_presence_validate(doc.root)
+          sections_sequence_validate(doc.root)
+        end
+        section_style(doc.root)
+        subclause_validate(doc.root)
+        super
+      end
+
+      # ISO/IEC DIR 2, 12.4
+      def foreword_validate(root)
+        f = root.at("//foreword") || return
+        s = f.at("./clause")
+        @log.add("Style", f, "foreword contains subclauses") unless s.nil?
+      end
+
+      # ISO/IEC DIR 2, 15.4
+      def normref_validate(root)
+        f = root.at("//references[@normative = 'true']") || return
+        f.at("./references | ./clause") &&
+          @log.add("Style", f, "normative references contains subclauses")
+      end
+
+      ONE_SYMBOLS_WARNING = "Only one Symbols and Abbreviated "\
+                            "Terms section in the standard".freeze
+
+      NON_DL_SYMBOLS_WARNING = "Symbols and Abbreviated Terms can "\
+                               "only contain a definition list".freeze
+
+      def symbols_validate(root)
+        f = root.xpath("//definitions")
+        f.empty? && return
+        (f.size == 1) || @log.add("Style", f.first, ONE_SYMBOLS_WARNING)
+        f.first.elements.each do |e|
+          unless %w(title dl).include? e.name
+            @log.add("Style", f.first, NON_DL_SYMBOLS_WARNING)
+            return
+          end
+        end
+      end
+
+      def seqcheck(names, msg, accepted)
+        n = names.shift
+        return [] if n.nil?
+
+        test = accepted.map { |a| n.at(a) }
+        if test.all?(&:nil?)
+          @log.add("Style", nil, msg)
+        end
+        names
+      end
+
+      def sections_presence_validate(root)
+        root.at("//sections/clause[@type = 'scope']") or
+          @log.add("Style", nil, "Scope clause missing")
+        root.at("//references[@normative = 'true']") or
+          @log.add("Style", nil, "Normative references missing")
+        root.at("//terms") or
+          @log.add("Style", nil, "Terms & definitions missing")
+      end
+
+      # spec of permissible section sequence
+      # we skip normative references, it goes to end of list
+      SEQ =
+        [
+          {
+            msg: "Initial section must be (content) Foreword",
+            val: ["./self::foreword"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::introduction", "./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Prefatory material must be followed by (clause) Scope",
+            val: ["./self::clause[@type = 'scope']"],
+          },
+          {
+            msg: "Normative References must be followed by "\
+                 "Terms and Definitions",
+            val: ["./self::terms | .//terms"],
+          },
+        ].freeze
+
+      SECTIONS_XPATH =
+        "//foreword | //introduction | //sections/terms | .//annex | "\
+        "//sections/definitions | //sections/clause | "\
+        "//references[not(parent::clause)] | "\
+        "//clause[descendant::references][not(parent::clause)]".freeze
+
+      def sections_sequence_validate(root)
+        names, n = sections_sequence_validate_start(root)
+        if root&.at("//bibdata/ext/subdoctype")&.text == "vocabulary"
+          names, n = sections_sequence_validate_body_vocab(names, n)
+        else
+          names, n = sections_sequence_validate_body(names, n)
+        end
+        sections_sequence_validate_end(names, n)
+      end
+
+      def sections_sequence_validate_start(root)
+        names = root.xpath(SECTIONS_XPATH)
+        names = seqcheck(names, SEQ[0][:msg], SEQ[0][:val])
+        n = names[0]
+        names = seqcheck(names, SEQ[1][:msg], SEQ[1][:val])
+        n&.at("./self::introduction") and
+          names = seqcheck(names, SEQ[2][:msg], SEQ[2][:val])
+        names = seqcheck(names, SEQ[3][:msg], SEQ[3][:val])
+        n = names.shift
+        n = names.shift if n&.at("./self::definitions")
+        [names, n]
+      end
+
+      def sections_sequence_validate_body(names, elem)
+        if elem.nil? || elem.name != "clause"
+          @log.add("Style", elem, "Document must contain at least one clause")
+        end
+        elem&.at("./self::clause") ||
+          @log.add("Style", elem, "Document must contain clause after "\
+                                  "Terms and Definitions")
+        elem&.at("./self::clause[@type = 'scope']") &&
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+        elem = names.shift
+        while elem&.name == "clause"
+          elem&.at("./self::clause[@type = 'scope']")
+          @log.add("Style", elem,
+                   "Scope must occur before Terms and Definitions")
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_body_vocab(names, elem)
+        while elem && %w(clause terms).include?(elem.name)
+          elem = names.shift
+        end
+        %w(annex references).include? elem&.name or
+          @log.add("Style", elem,
+                   "Only annexes and references can follow terms and clauses")
+        [names, elem]
+      end
+
+      def sections_sequence_validate_end(names, elem)
+        while elem&.name == "annex"
+          elem = names.shift
+          if elem.nil?
+            @log.add("Style", nil, "Document must include (references) "\
+                                   "Normative References")
+          end
+        end
+        elem&.at("./self::references[@normative = 'true']") ||
+          @log.add("Style", nil, "Document must include (references) "\
+                                 "Normative References")
+        elem = names&.shift
+        elem&.at("./self::references[@normative = 'false']") ||
+          @log.add("Style", elem,
+                   "Final section must be (references) Bibliography")
+        names.empty? ||
+          @log.add("Style", elem,
+                   "There are sections after the final Bibliography")
+      end
+
+      NORM_ISO_WARN = "non-ISO/IEC reference not expected as normative".freeze
+      SCOPE_WARN = "Scope contains subclauses: should be succinct".freeze
+
+      def section_style(root)
+        foreword_style(root.at("//foreword"))
+        introduction_style(root.at("//introduction"))
+        scope_style(root.at("//clause[@type = 'scope']"))
+        scope = root.at("//clause[@type = 'scope']/clause")
+        # ISO/IEC DIR 2, 14.4
+        scope.nil? || style_warning(scope, SCOPE_WARN, nil)
+        tech_report_style(root)
+      end
+
+      def tech_report_style(root)
+        root.at("//bibdata/ext/doctype")&.text == "technical-report" or return
+        root.xpath("//sections/clause[not(@type = 'scope')] | //annex")
+          .each do |s|
+          r = requirement_check(extract_text(s)) and
+            style_warning(s,
+                          "Technical Report clause may contain requirement", r)
+        end
+      end
+
+      ASSETS_TO_STYLE =
+        "//termsource | //formula | //termnote | "\
+        "//p[not(ancestor::boilerplate)] | //li[not(p)] | //dt | "\
+        "//dd[not(p)] | //td[not(p)] | //th[not(p)]".freeze
+
+      NORM_BIBITEMS =
+        "//references[@normative = 'true']/bibitem".freeze
+
+      # ISO/IEC DIR 2, 10.2
+      def norm_bibitem_style(root)
+        root.xpath(NORM_BIBITEMS).each do |b|
+          if b.at(Standoc::Converter::ISO_PUBLISHER_XPATH).nil?
+            @log.add("Style", b, "#{NORM_ISO_WARN}: #{b.text}")
+          end
+        end
+      end
+
+      def asset_style(root)
+        root.xpath("//example | //termexample").each { |e| example_style(e) }
+        root.xpath("//definition/verbal-definition").each { |e| definition_style(e) }
+        root.xpath("//note").each { |e| note_style(e) }
+        root.xpath("//fn").each { |e| footnote_style(e) }
+        root.xpath(ASSETS_TO_STYLE).each { |e| style(e, extract_text(e)) }
+        norm_bibitem_style(root)
+        super
+      end
+
+      def subclause_validate(root)
+        root.xpath("//clause/clause/clause/clause/clause/clause/clause/clause")
+          .each do |c|
+          style_warning(c, "Exceeds the maximum clause depth of 7", nil)
+        end
+      end
+
+      # ISO/IEC DIR 2, 22.3.2
+      def onlychild_clause_validate(root)
+        root.xpath(Standoc::Utils::SUBCLAUSE_XPATH).each do |c|
+          next unless c.xpath("../clause").size == 1
+
+          title = c.at("./title")
+          location = c["id"] || "#{c.text[0..60]}..."
+          location += ":#{title.text}" if c["id"] && !title.nil?
+          @log.add("Style", nil, "#{location}: subclause is only child")
+        end
+      end
+    end
+  end
+end