metanorma-standoc 1.11.1 → 2.0.0

data/lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true.
+require "metanorma/standoc/utils"
+
+
+module Metanorma
+  module Standoc
+    # Intelligent term lookup xml modifier
+    class TermLookupCleanup
+      AUTOMATIC_GENERATED_ID_REGEXP = /\A_/.freeze
+      EXISTING_TERM_REGEXP = /\Aterm-/.freeze
+      EXISTING_SYMBOL_REGEXP = /\Asymbol-/.freeze
+
+      attr_reader :xmldoc, :termlookup, :log
+
+      def initialize(xmldoc, log)
+        @xmldoc = xmldoc
+        @log = log
+        @termlookup = { term: {}, symbol: {}, secondary2primary: {} }
+        @idhash = {}
+      end
+
+      def call
+        @idhash = populate_idhash
+        @termlookup = replace_automatic_generated_ids_terms
+        set_termxref_tags_target
+        concept_cleanup
+        related_cleanup
+      end
+
+      private
+
+      def concept_cleanup
+        xmldoc.xpath("//concept").each do |n|
+          n.delete("type")
+          refterm = n.at("./refterm") or next
+          p = @termlookup[:secondary2primary][refterm.text] and
+            refterm.children = p
+        end
+      end
+
+      def related_cleanup
+        xmldoc.xpath("//related").each do |n|
+          refterm = n.at("./refterm") or next
+          p = @termlookup[:secondary2primary][refterm.text] and
+            refterm.children = p
+          refterm.replace("<preferred><expression><name>#{refterm.children.to_xml}"\
+                          "</name></expression></preferred>")
+        end
+      end
+
+      def populate_idhash
+        xmldoc.xpath("//*[@id]").each_with_object({}) do |n, mem|
+          next unless /^(term|symbol)-/.match?(n["id"])
+
+          mem[n["id"]] = true
+        end
+      end
+
+      def set_termxref_tags_target
+        xmldoc.xpath("//termxref").each do |node|
+          target = normalize_ref_id(node.text)
+          if termlookup[:term][target].nil? && termlookup[:symbol][target].nil?
+            remove_missing_ref(node, target)
+            next
+          end
+          x = node.at("../xrefrender") and modify_ref_node(x, target)
+          node.name = "refterm"
+        end
+      end
+
+      def remove_missing_ref(node, target)
+        if node.at("../concept[@type = 'symbol']")
+          remove_missing_ref_symbol(node, target)
+        else
+          remove_missing_ref_term(node, target)
+        end
+      end
+
+      def remove_missing_ref_term(node, target)
+        log.add("AsciiDoc Input", node,
+                %(Error: Term reference in `term[#{target}]` missing: \
+                "#{target}" is not defined in document))
+        node.name = "strong"
+        node&.at("../xrefrender")&.remove
+        display = node&.at("../renderterm")&.remove&.children
+        display = [] if display.nil? || display&.to_xml == node.text
+        d = display.empty? ? "" : ", display <tt>#{display.to_xml}</tt>"
+        node.children = "term <tt>#{node.text}</tt>#{d} "\
+                        "not resolved via ID <tt>#{target}</tt>"
+      end
+
+      def remove_missing_ref_symbol(node, target)
+        log.add("AsciiDoc Input", node,
+                %(Error: Symbol reference in `symbol[#{target}]` missing: \
+                "#{target}" is not defined in document))
+        node.name = "strong"
+        node&.at("../xrefrender")&.remove
+        display = node&.at("../renderterm")&.remove&.children
+        display = [] if display.nil? || display&.to_xml == node.text
+        d = display.empty? ? "" : ", display <tt>#{display.to_xml}</tt>"
+        node.children = "symbol <tt>#{node.text}</tt>#{d} "\
+                        "not resolved via ID <tt>#{target}</tt>"
+      end
+
+      def modify_ref_node(node, target)
+        node.name = "xref"
+        s = termlookup[:symbol][target]
+        t = termlookup[:term][target]
+        type = node.parent["type"]
+        if type == "term" || ((!type || node.parent.name == "related") && t)
+          node["target"] = t
+        elsif type == "symbol" ||
+          ((!type || node.parent.name == "related") && s)
+          node["target"] = s
+        end
+      end
+
+      def replace_automatic_generated_ids_terms
+        r = xmldoc.xpath("//term").each.with_object({}) do |n, res|
+          normalize_id_and_memorize(n, res, "./preferred//name",
+                                    "term")
+        end
+        s = xmldoc.xpath("//definitions//dt").each.with_object({}) do |n, res|
+          normalize_id_and_memorize(n, res, ".", "symbol")
+        end
+        { term: r, symbol: s, secondary2primary: pref_secondary2primary }
+      end
+
+      def pref_secondary2primary
+        xmldoc.xpath("//term").each.with_object({}) do |n, res|
+          n.xpath("./preferred//name").each_with_index do |p, i|
+            i.zero? and term = p.text
+            i.positive? and res[p.text] = term
+          end
+        end
+      end
+
+      def normalize_id_and_memorize(node, res_table, text_selector, prefix)
+        normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
+        memorize_other_pref_terms(node, res_table, text_selector)
+      end
+
+      def normalize_id_and_memorize_init(node, res_table, text_selector, prefix)
+        term_text = normalize_ref_id(node.at(text_selector).text)
+        unless AUTOMATIC_GENERATED_ID_REGEXP.match(node["id"]).nil? &&
+            !node["id"].nil?
+          id = unique_text_id(term_text, prefix)
+          node["id"] = id
+          @idhash[id] = true
+        end
+        res_table[term_text] = node["id"]
+      end
+
+      def memorize_other_pref_terms(node, res_table, text_selector)
+        node.xpath(text_selector).each_with_index do |p, i|
+          next unless i.positive?
+
+          res_table[normalize_ref_id(p.text)] = node["id"]
+        end
+      end
+
+      def normalize_ref_id(text)
+        Metanorma::Utils::to_ncname(text.downcase.gsub(/[[:space:]]/, "-"))
+      end
+
+      def unique_text_id(text, prefix)
+        unless @idhash["#{prefix}-#{text}"]
+          return "#{prefix}-#{text}"
+        end
+
+        (1..Float::INFINITY).lazy.each do |index|
+          unless @idhash["#{prefix}-#{text}-#{index}"]
+            break("#{prefix}-#{text}-#{index}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/terms.rb

@@ -0,0 +1,160 @@
+module Metanorma
+  module Standoc
+    module Section
+      def in_terms?
+        @term_def
+      end
+
+      def nonterm_symbols_parse(attrs, xml, node)
+        defs = @definitions
+        @definitions = false
+        clause_parse(attrs, xml, node)
+        @definitions = defs
+      end
+
+      def symbols_attrs(node, attr)
+        case sectiontype1(node)
+        when "symbols" then attr.merge(type: "symbols")
+        when "abbreviated terms", "abbreviations"
+          attr.merge(type: "abbreviated_terms")
+        else
+          attr
+        end
+      end
+
+      def symbols_parse(attr, xml, node)
+        node.role == "nonterm" and return nonterm_symbols_parse(attr, xml, node)
+        xml.definitions **attr_code(attr) do |xml_section|
+          xml_section.title { |t| t << node.title }
+          defs = @definitions
+          termdefs = @term_def
+          @definitions = true
+          @term_def = false
+          xml_section << node.content
+          @definitions = defs
+          @term_def = termdefs
+        end
+      end
+
+      def nonterm_term_def_subclause_parse(attrs, xml, node)
+        defs = @term_def
+        @term_def = false
+        clause_parse(attrs, xml, node)
+        @term_def = defs
+      end
+
+      def terms_boilerplate_parse(attrs, xml, node)
+        defs = @term_def
+        @term_def = false
+        clause_parse(attrs.merge(type: "boilerplate"), xml, node)
+        @term_def = defs
+      end
+
+      # subclause contains subclauses
+      def term_def_subclause_parse(attrs, xml, node)
+        node.role == "nonterm" and
+          return nonterm_term_def_subclause_parse(attrs, xml, node)
+        node.role == "boilerplate" and
+          return terms_boilerplate_parse(attrs, xml, node)
+        st = sectiontype(node, false)
+        return symbols_parse(attrs, xml, node) if @definitions
+
+        sub = node.find_by(context: :section) { |s| s.level == node.level + 1 }
+        sub.empty? || (return term_def_parse(attrs, xml, node, false))
+        st == "symbols and abbreviated terms" and
+          return symbols_parse(attrs, xml, node)
+        st == "terms and definitions" and return clause_parse(attrs, xml, node)
+        term_def_subclause_parse1(attrs, xml, node)
+      end
+
+      def term_def_subclause_parse1(attrs, xml, node)
+        xml.term **attr_code(attrs) do |xml_section|
+          term_designation(xml_section, node, "preferred", node.title)
+          xml_section << node.content
+        end
+      end
+
+      def term_def_parse(attrs, xml, node, _toplevel)
+        xml.terms **attr_code(attrs) do |section|
+          section.title { |t| t << node.title }
+          (s = node.attr("source")) && s.split(",").each do |s1|
+            section.termdocsource(nil, **attr_code(bibitemid: s1))
+          end
+          section << node.content
+        end
+      end
+
+      def term_designation(xml, _node, tag, text)
+        xml.send tag do |p|
+          p.expression do |e|
+            e.name { |name| name << text }
+          end
+        end
+      end
+
+      def term_source_attrs(_node, seen_xref)
+        { case: seen_xref.children[0]["case"],
+          droploc: seen_xref.children[0]["droploc"],
+          bibitemid: seen_xref.children[0]["target"],
+          format: seen_xref.children[0]["format"], type: "inline" }
+      end
+
+      def add_term_source(node, xml_t, seen_xref, match)
+        if seen_xref.children[0].name == "concept"
+          xml_t.origin { |o| o << seen_xref.children[0].to_xml }
+        else
+          attrs = term_source_attrs(node, seen_xref)
+          attrs.delete(:text)
+          xml_t.origin **attr_code(attrs) do |o|
+            o << seen_xref.children[0].children.to_xml
+          end
+        end
+        add_term_source_mod(xml_t, match)
+      end
+
+      def add_term_source_mod(xml_t, match)
+        match[:text] && xml_t.modification do |mod|
+          mod.p { |p| p << match[:text].sub(/^\s+/, "") }
+        end
+      end
+
+      TERM_REFERENCE_RE_STR = <<~REGEXP.freeze
+        ^(?<xref><(xref|concept)[^>]+>(.*?</(xref|concept)>)?)
+               (,\s(?<text>.*))?
+        $
+      REGEXP
+      TERM_REFERENCE_RE =
+        Regexp.new(TERM_REFERENCE_RE_STR.gsub(/\s/, "").gsub(/_/, "\\s"),
+                   Regexp::IGNORECASE | Regexp::MULTILINE)
+
+      def extract_termsource_refs(text, node)
+        matched = TERM_REFERENCE_RE.match text
+        matched.nil? and @log.add("AsciiDoc Input", node,
+                                  "term reference not in expected format:"\
+                                  "#{text}")
+        matched
+      end
+
+      def termsource(node)
+        matched = extract_termsource_refs(node.content, node) || return
+        noko do |xml|
+          status = node.attr("status") ||
+            (matched[:text] ? "modified" : "identical")
+          attrs = { status: status, type: node.attr("type") || "authoritative" }
+          xml.termsource **attrs do |xml_t|
+            seen_xref = Nokogiri::XML.fragment(matched[:xref])
+            add_term_source(node, xml_t, seen_xref, matched)
+          end
+        end.join("\n")
+      end
+
+      def termdefinition(node)
+        noko do |xml|
+          xml.definition do |d|
+            d << node.content
+          end
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/utils.rb

@@ -0,0 +1,101 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "uuidtools"
+
+module Metanorma
+  module Standoc
+    module Utils
+      def convert(node, transform = nil, opts = {})
+        transform ||= node.node_name
+        opts.empty? ? (send transform, node) : (send transform, node, opts)
+      end
+
+      def document_ns_attributes(_doc)
+        nil
+      end
+
+      NOKOHEAD = <<~HERE.freeze
+        <!DOCTYPE html SYSTEM
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+        <head> <title></title> <meta charset="UTF-8" /> </head>
+        <body> </body> </html>
+      HERE
+
+      # block for processing XML document fragments as XHTML,
+      # to allow for HTMLentities
+      # Unescape special chars used in Asciidoctor substitution processing
+      def noko(&block)
+        doc = ::Nokogiri::XML.parse(NOKOHEAD)
+        fragment = doc.fragment("")
+        ::Nokogiri::XML::Builder.with fragment, &block
+        fragment.to_xml(encoding: "US-ASCII", indent: 0).lines.map do |l|
+          l.gsub(/>\n$/, ">").gsub(/\s*\n$/m, " ").gsub("&#150;", "\u0096")
+            .gsub("&#151;", "\u0097").gsub("&#x96;", "\u0096")
+            .gsub("&#x97;", "\u0097")
+        end
+      end
+
+      def attr_code(attributes)
+        attributes.compact.transform_values do |v|
+          v.is_a?(String) ? HTMLEntities.new.decode(v) : v
+        end
+      end
+
+      # if the contents of node are blocks, output them to out;
+      # else, wrap them in <p>
+      def wrap_in_para(node, out)
+        if node.blocks? then out << node.content
+        else
+          out.p { |p| p << node.content }
+        end
+      end
+
+      SUBCLAUSE_XPATH = "//clause[not(parent::sections)]"\
+                        "[not(ancestor::boilerplate)]".freeze
+
+      def isodoc(lang, script, i18nyaml = nil)
+        conv = html_converter(EmptyAttr.new)
+        i18n = conv.i18n_init(lang, script, i18nyaml)
+        conv.metadata_init(lang, script, i18n)
+        conv
+      end
+
+      def dl_to_attrs(elem, dlist, name)
+        e = dlist.at("./dt[text()='#{name}']") or return
+        val = e.at("./following::dd/p") || e.at("./following::dd") or return
+        elem[name] = val.text
+      end
+
+      def dl_to_elems(ins, elem, dlist, name)
+        a = elem.at("./#{name}[last()]")
+        ins = a if a
+        dlist.xpath("./dt[text()='#{name}']").each do |e|
+          v = e.at("./following::dd")
+          e = v.elements and e.size == 1 && e.first.name == "p" and v = e.first
+          v.name = name
+          ins.next = v
+          ins = ins.next
+        end
+        ins
+      end
+
+      def term_expr(elem)
+        "<expression><name>#{elem}</name></expression>"
+      end
+
+      class EmptyAttr
+        def attr(_any_attribute)
+          nil
+        end
+
+        def attributes
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/validate.rb

@@ -0,0 +1,158 @@
+require "metanorma/standoc/utils"
+require_relative "./validate_section"
+require "nokogiri"
+require "jing"
+require "iev"
+
+module Metanorma
+  module Standoc
+    module Validate
+      SOURCELOCALITY = "./origin//locality[@type = 'clause']/"\
+                       "referenceFrom".freeze
+
+      def init_iev
+        return nil if @no_isobib
+        return @iev if @iev
+
+        @iev = Iev::Db.new(@iev_globalname, @iev_localname) unless @no_isobib
+        @iev
+      end
+
+      def iev_validate(xmldoc)
+        @iev = init_iev or return
+        xmldoc.xpath("//term").each do |t|
+          t.xpath(".//termsource").each do |src|
+            (/^IEC 60050-/.match(src&.at("./origin/@citeas")&.text) &&
+          loc = src.xpath(SOURCELOCALITY)&.text) or next
+            iev_validate1(t, loc, xmldoc)
+          end
+        end
+      end
+
+      def iev_validate1(term, loc, xmldoc)
+        iev = @iev.fetch(loc,
+                         xmldoc&.at("//language")&.text || "en") or return
+        pref = term.xpath("./preferred//name").inject([]) do |m, x|
+          m << x&.text&.downcase
+        end
+        pref.include?(iev.downcase) or
+          @log.add("Bibliography", term, %(Term "#{pref[0]}" does not match ) +
+                   %(IEV #{loc} "#{iev}"))
+      end
+
+      def content_validate(doc)
+        @fatalerror = []
+        xref_validate(doc)
+        section_validate(doc)
+        norm_ref_validate(doc)
+        repeat_id_validate(doc.root)
+        iev_validate(doc.root)
+        concept_validate(doc, "concept", "refterm")
+        concept_validate(doc, "related", "preferred//name")
+        @fatalerror.empty? or clean_abort(@fatalerror.join("\n"), doc.to_xml)
+      end
+
+      def norm_ref_validate(doc)
+        found = false
+        doc.xpath("//references[@normative = 'true']/bibitem").each do |b|
+          next unless docid = b.at("./docidentifier[@type = 'metanorma']")
+          next unless /^\[\d+\]$/.match?(docid.text)
+
+          @log.add("Bibliography", b,
+                   "Numeric reference in normative references")
+          found = true
+        end
+        found and @fatalerror << "Numeric reference in normative references"
+      end
+
+      def concept_validate(doc, tag, refterm)
+        found = false
+        doc.xpath("//#{tag}/xref").each do |x|
+          next if doc.at("//term[@id = '#{x['target']}']")
+          next if doc.at("//definitions//dt[@id = '#{x['target']}']")
+
+          ref = x&.at("../#{refterm}")&.text
+          @log.add("Anchors", x,
+                   "#{tag.capitalize} #{ref} is pointing to "\
+                   "#{x['target']}, which is not a term or symbol")
+          found = true
+        end
+        found and
+          @fatalerror << "#{tag.capitalize} not cross-referencing term or symbol"
+      end
+
+      def repeat_id_validate1(ids, elem)
+        if ids[elem["id"]]
+          @log.add("Anchors", elem, "Anchor #{elem['id']} has already been "\
+                                    "used at line #{ids[elem['id']]}")
+          @fatalerror << "Multiple instances of same ID: #{elem['id']}"
+        else
+          ids[elem["id"]] = elem.line
+        end
+        ids
+      end
+
+      def repeat_id_validate(doc)
+        ids = {}
+        doc.xpath("//*[@id]").each do |x|
+          ids = repeat_id_validate1(ids, x)
+        end
+      end
+
+      def schema_validate(doc, schema)
+        Tempfile.open(["tmp", ".xml"], encoding: "UTF-8") do |f|
+          schema_validate1(f, doc, schema)
+        rescue Jing::Error => e
+          clean_abort("Jing failed with error: #{e}", doc.to_xml)
+        ensure
+          f.close!
+        end
+      end
+
+      def schema_validate1(file, doc, schema)
+        file.write(doc.to_xml)
+        file.close
+        errors = Jing.new(schema, encoding: "UTF-8").validate(file.path)
+        warn "Syntax Valid!" if errors.none?
+        errors.each do |e|
+          @log.add("Metanorma XML Syntax",
+                   "XML Line #{'%06d' % e[:line]}:#{e[:column]}", e[:message])
+        end
+      end
+
+      # RelaxNG cannot cope well with wildcard attributes. So we strip
+      # any attributes from FormattedString instances (which can contain
+      # xs:any markup, and are signalled with @format) before validation.
+      def formattedstr_strip(doc)
+        doc.xpath("//*[@format] | //stem | //bibdata//description | "\
+                  "//formattedref | //bibdata//note | //bibdata/abstract | "\
+                  "//bibitem/abstract | //bibitem/note | //misc-container")
+          .each do |n|
+          n.elements.each do |e|
+            e.traverse do |e1|
+              e1.element? and e1.each { |k, _v| e1.delete(k) }
+            end
+          end
+        end
+        doc
+      end
+
+      # manually check for xref/@target, xref/@to integrity
+      def xref_validate(doc)
+        ids = doc.xpath("//*/@id").each_with_object({}) { |x, m| m[x.text] = 1 }
+        doc.xpath("//xref/@target | //xref/@to").each do |x|
+          next if ids[x.text]
+
+          @log.add("Anchors", x.parent,
+                   "Crossreference target #{x.text} is undefined")
+        end
+      end
+
+      def validate(doc)
+        content_validate(doc)
+        schema_validate(formattedstr_strip(doc.dup),
+                        File.join(File.dirname(__FILE__), "isodoc.rng"))
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/validate_section.rb

@@ -0,0 +1,55 @@
+require "nokogiri"
+
+module Metanorma
+  module Standoc
+    module Validate
+      def section_validate(doc)
+        sourcecode_style(doc.root)
+        hanging_para_style(doc.root)
+        asset_style(doc.root)
+      end
+
+      def sourcecode_style(root)
+        root.xpath("//sourcecode").each do |x|
+          callouts = x.elements.select { |e| e.name == "callout" }
+          annotations = x.elements.select { |e| e.name == "annotation" }
+          if callouts.size != annotations.size
+            @log.add("AsciiDoc Input", x,
+                     "mismatch of callouts and annotations")
+          end
+        end
+      end
+
+      def style_warning(node, msg, text = nil)
+        w = msg
+        w += ": #{text}" if text
+        @log.add("Metanorma XML Style Warning", node, w)
+      end
+
+      def asset_title_style(root)
+        root.xpath("//figure[image][not(name)]").each do |node|
+          style_warning(node, "Figure should have title", nil)
+        end
+        root.xpath("//table[not(name)]").each do |node|
+          style_warning(node, "Table should have title", nil)
+        end
+      end
+
+      def asset_style(root)
+        asset_title_style(root)
+      end
+
+      def hanging_para_style(root)
+        root.xpath("//clause | //annex | //foreword | //introduction | "\
+                   "//acknowledgements").each do |c|
+          next unless c.at("./clause")
+          next if c.elements.reject do |n|
+                    %w(clause title).include? n.name
+                  end.empty?
+
+          style_warning(c, "Hanging paragraph in clause")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "1.11.1".freeze
+    VERSION = "2.0.0".freeze
   end
 end

data/lib/metanorma-standoc.rb

@@ -1,6 +1,6 @@
 require "asciidoctor" unless defined? Asciidoctor::Converter
 require_relative "isodoc/pdf_convert"
-require_relative "asciidoctor/standoc/converter"
+require_relative "metanorma/standoc/converter"
 require_relative "metanorma/standoc/version"
 require "asciidoctor/extensions"