metanorma-standoc 1.11.4 → 2.0.0

data/lib/metanorma/standoc/terms.rb

@@ -0,0 +1,160 @@
+module Metanorma
+  module Standoc
+    module Section
+      def in_terms?
+        @term_def
+      end
+
+      def nonterm_symbols_parse(attrs, xml, node)
+        defs = @definitions
+        @definitions = false
+        clause_parse(attrs, xml, node)
+        @definitions = defs
+      end
+
+      def symbols_attrs(node, attr)
+        case sectiontype1(node)
+        when "symbols" then attr.merge(type: "symbols")
+        when "abbreviated terms", "abbreviations"
+          attr.merge(type: "abbreviated_terms")
+        else
+          attr
+        end
+      end
+
+      def symbols_parse(attr, xml, node)
+        node.role == "nonterm" and return nonterm_symbols_parse(attr, xml, node)
+        xml.definitions **attr_code(attr) do |xml_section|
+          xml_section.title { |t| t << node.title }
+          defs = @definitions
+          termdefs = @term_def
+          @definitions = true
+          @term_def = false
+          xml_section << node.content
+          @definitions = defs
+          @term_def = termdefs
+        end
+      end
+
+      def nonterm_term_def_subclause_parse(attrs, xml, node)
+        defs = @term_def
+        @term_def = false
+        clause_parse(attrs, xml, node)
+        @term_def = defs
+      end
+
+      def terms_boilerplate_parse(attrs, xml, node)
+        defs = @term_def
+        @term_def = false
+        clause_parse(attrs.merge(type: "boilerplate"), xml, node)
+        @term_def = defs
+      end
+
+      # subclause contains subclauses
+      def term_def_subclause_parse(attrs, xml, node)
+        node.role == "nonterm" and
+          return nonterm_term_def_subclause_parse(attrs, xml, node)
+        node.role == "boilerplate" and
+          return terms_boilerplate_parse(attrs, xml, node)
+        st = sectiontype(node, false)
+        return symbols_parse(attrs, xml, node) if @definitions
+
+        sub = node.find_by(context: :section) { |s| s.level == node.level + 1 }
+        sub.empty? || (return term_def_parse(attrs, xml, node, false))
+        st == "symbols and abbreviated terms" and
+          return symbols_parse(attrs, xml, node)
+        st == "terms and definitions" and return clause_parse(attrs, xml, node)
+        term_def_subclause_parse1(attrs, xml, node)
+      end
+
+      def term_def_subclause_parse1(attrs, xml, node)
+        xml.term **attr_code(attrs) do |xml_section|
+          term_designation(xml_section, node, "preferred", node.title)
+          xml_section << node.content
+        end
+      end
+
+      def term_def_parse(attrs, xml, node, _toplevel)
+        xml.terms **attr_code(attrs) do |section|
+          section.title { |t| t << node.title }
+          (s = node.attr("source")) && s.split(",").each do |s1|
+            section.termdocsource(nil, **attr_code(bibitemid: s1))
+          end
+          section << node.content
+        end
+      end
+
+      def term_designation(xml, _node, tag, text)
+        xml.send tag do |p|
+          p.expression do |e|
+            e.name { |name| name << text }
+          end
+        end
+      end
+
+      def term_source_attrs(_node, seen_xref)
+        { case: seen_xref.children[0]["case"],
+          droploc: seen_xref.children[0]["droploc"],
+          bibitemid: seen_xref.children[0]["target"],
+          format: seen_xref.children[0]["format"], type: "inline" }
+      end
+
+      def add_term_source(node, xml_t, seen_xref, match)
+        if seen_xref.children[0].name == "concept"
+          xml_t.origin { |o| o << seen_xref.children[0].to_xml }
+        else
+          attrs = term_source_attrs(node, seen_xref)
+          attrs.delete(:text)
+          xml_t.origin **attr_code(attrs) do |o|
+            o << seen_xref.children[0].children.to_xml
+          end
+        end
+        add_term_source_mod(xml_t, match)
+      end
+
+      def add_term_source_mod(xml_t, match)
+        match[:text] && xml_t.modification do |mod|
+          mod.p { |p| p << match[:text].sub(/^\s+/, "") }
+        end
+      end
+
+      TERM_REFERENCE_RE_STR = <<~REGEXP.freeze
+        ^(?<xref><(xref|concept)[^>]+>(.*?</(xref|concept)>)?)
+               (,\s(?<text>.*))?
+        $
+      REGEXP
+      TERM_REFERENCE_RE =
+        Regexp.new(TERM_REFERENCE_RE_STR.gsub(/\s/, "").gsub(/_/, "\\s"),
+                   Regexp::IGNORECASE | Regexp::MULTILINE)
+
+      def extract_termsource_refs(text, node)
+        matched = TERM_REFERENCE_RE.match text
+        matched.nil? and @log.add("AsciiDoc Input", node,
+                                  "term reference not in expected format:"\
+                                  "#{text}")
+        matched
+      end
+
+      def termsource(node)
+        matched = extract_termsource_refs(node.content, node) || return
+        noko do |xml|
+          status = node.attr("status") ||
+            (matched[:text] ? "modified" : "identical")
+          attrs = { status: status, type: node.attr("type") || "authoritative" }
+          xml.termsource **attrs do |xml_t|
+            seen_xref = Nokogiri::XML.fragment(matched[:xref])
+            add_term_source(node, xml_t, seen_xref, matched)
+          end
+        end.join("\n")
+      end
+
+      def termdefinition(node)
+        noko do |xml|
+          xml.definition do |d|
+            d << node.content
+          end
+        end.join("\n")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/utils.rb

@@ -0,0 +1,101 @@
+require "date"
+require "nokogiri"
+require "htmlentities"
+require "json"
+require "pathname"
+require "uuidtools"
+
+module Metanorma
+  module Standoc
+    module Utils
+      def convert(node, transform = nil, opts = {})
+        transform ||= node.node_name
+        opts.empty? ? (send transform, node) : (send transform, node, opts)
+      end
+
+      def document_ns_attributes(_doc)
+        nil
+      end
+
+      NOKOHEAD = <<~HERE.freeze
+        <!DOCTYPE html SYSTEM
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+        <html xmlns="http://www.w3.org/1999/xhtml">
+        <head> <title></title> <meta charset="UTF-8" /> </head>
+        <body> </body> </html>
+      HERE
+
+      # block for processing XML document fragments as XHTML,
+      # to allow for HTMLentities
+      # Unescape special chars used in Asciidoctor substitution processing
+      def noko(&block)
+        doc = ::Nokogiri::XML.parse(NOKOHEAD)
+        fragment = doc.fragment("")
+        ::Nokogiri::XML::Builder.with fragment, &block
+        fragment.to_xml(encoding: "US-ASCII", indent: 0).lines.map do |l|
+          l.gsub(/>\n$/, ">").gsub(/\s*\n$/m, " ").gsub("&#150;", "\u0096")
+            .gsub("&#151;", "\u0097").gsub("&#x96;", "\u0096")
+            .gsub("&#x97;", "\u0097")
+        end
+      end
+
+      def attr_code(attributes)
+        attributes.compact.transform_values do |v|
+          v.is_a?(String) ? HTMLEntities.new.decode(v) : v
+        end
+      end
+
+      # if the contents of node are blocks, output them to out;
+      # else, wrap them in <p>
+      def wrap_in_para(node, out)
+        if node.blocks? then out << node.content
+        else
+          out.p { |p| p << node.content }
+        end
+      end
+
+      SUBCLAUSE_XPATH = "//clause[not(parent::sections)]"\
+                        "[not(ancestor::boilerplate)]".freeze
+
+      def isodoc(lang, script, i18nyaml = nil)
+        conv = html_converter(EmptyAttr.new)
+        i18n = conv.i18n_init(lang, script, i18nyaml)
+        conv.metadata_init(lang, script, i18n)
+        conv
+      end
+
+      def dl_to_attrs(elem, dlist, name)
+        e = dlist.at("./dt[text()='#{name}']") or return
+        val = e.at("./following::dd/p") || e.at("./following::dd") or return
+        elem[name] = val.text
+      end
+
+      def dl_to_elems(ins, elem, dlist, name)
+        a = elem.at("./#{name}[last()]")
+        ins = a if a
+        dlist.xpath("./dt[text()='#{name}']").each do |e|
+          v = e.at("./following::dd")
+          e = v.elements and e.size == 1 && e.first.name == "p" and v = e.first
+          v.name = name
+          ins.next = v
+          ins = ins.next
+        end
+        ins
+      end
+
+      def term_expr(elem)
+        "<expression><name>#{elem}</name></expression>"
+      end
+
+      class EmptyAttr
+        def attr(_any_attribute)
+          nil
+        end
+
+        def attributes
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/validate.rb

@@ -0,0 +1,158 @@
+require "metanorma/standoc/utils"
+require_relative "./validate_section"
+require "nokogiri"
+require "jing"
+require "iev"
+
+module Metanorma
+  module Standoc
+    module Validate
+      SOURCELOCALITY = "./origin//locality[@type = 'clause']/"\
+                       "referenceFrom".freeze
+
+      def init_iev
+        return nil if @no_isobib
+        return @iev if @iev
+
+        @iev = Iev::Db.new(@iev_globalname, @iev_localname) unless @no_isobib
+        @iev
+      end
+
+      def iev_validate(xmldoc)
+        @iev = init_iev or return
+        xmldoc.xpath("//term").each do |t|
+          t.xpath(".//termsource").each do |src|
+            (/^IEC 60050-/.match(src&.at("./origin/@citeas")&.text) &&
+          loc = src.xpath(SOURCELOCALITY)&.text) or next
+            iev_validate1(t, loc, xmldoc)
+          end
+        end
+      end
+
+      def iev_validate1(term, loc, xmldoc)
+        iev = @iev.fetch(loc,
+                         xmldoc&.at("//language")&.text || "en") or return
+        pref = term.xpath("./preferred//name").inject([]) do |m, x|
+          m << x&.text&.downcase
+        end
+        pref.include?(iev.downcase) or
+          @log.add("Bibliography", term, %(Term "#{pref[0]}" does not match ) +
+                   %(IEV #{loc} "#{iev}"))
+      end
+
+      def content_validate(doc)
+        @fatalerror = []
+        xref_validate(doc)
+        section_validate(doc)
+        norm_ref_validate(doc)
+        repeat_id_validate(doc.root)
+        iev_validate(doc.root)
+        concept_validate(doc, "concept", "refterm")
+        concept_validate(doc, "related", "preferred//name")
+        @fatalerror.empty? or clean_abort(@fatalerror.join("\n"), doc.to_xml)
+      end
+
+      def norm_ref_validate(doc)
+        found = false
+        doc.xpath("//references[@normative = 'true']/bibitem").each do |b|
+          next unless docid = b.at("./docidentifier[@type = 'metanorma']")
+          next unless /^\[\d+\]$/.match?(docid.text)
+
+          @log.add("Bibliography", b,
+                   "Numeric reference in normative references")
+          found = true
+        end
+        found and @fatalerror << "Numeric reference in normative references"
+      end
+
+      def concept_validate(doc, tag, refterm)
+        found = false
+        doc.xpath("//#{tag}/xref").each do |x|
+          next if doc.at("//term[@id = '#{x['target']}']")
+          next if doc.at("//definitions//dt[@id = '#{x['target']}']")
+
+          ref = x&.at("../#{refterm}")&.text
+          @log.add("Anchors", x,
+                   "#{tag.capitalize} #{ref} is pointing to "\
+                   "#{x['target']}, which is not a term or symbol")
+          found = true
+        end
+        found and
+          @fatalerror << "#{tag.capitalize} not cross-referencing term or symbol"
+      end
+
+      def repeat_id_validate1(ids, elem)
+        if ids[elem["id"]]
+          @log.add("Anchors", elem, "Anchor #{elem['id']} has already been "\
+                                    "used at line #{ids[elem['id']]}")
+          @fatalerror << "Multiple instances of same ID: #{elem['id']}"
+        else
+          ids[elem["id"]] = elem.line
+        end
+        ids
+      end
+
+      def repeat_id_validate(doc)
+        ids = {}
+        doc.xpath("//*[@id]").each do |x|
+          ids = repeat_id_validate1(ids, x)
+        end
+      end
+
+      def schema_validate(doc, schema)
+        Tempfile.open(["tmp", ".xml"], encoding: "UTF-8") do |f|
+          schema_validate1(f, doc, schema)
+        rescue Jing::Error => e
+          clean_abort("Jing failed with error: #{e}", doc.to_xml)
+        ensure
+          f.close!
+        end
+      end
+
+      def schema_validate1(file, doc, schema)
+        file.write(doc.to_xml)
+        file.close
+        errors = Jing.new(schema, encoding: "UTF-8").validate(file.path)
+        warn "Syntax Valid!" if errors.none?
+        errors.each do |e|
+          @log.add("Metanorma XML Syntax",
+                   "XML Line #{'%06d' % e[:line]}:#{e[:column]}", e[:message])
+        end
+      end
+
+      # RelaxNG cannot cope well with wildcard attributes. So we strip
+      # any attributes from FormattedString instances (which can contain
+      # xs:any markup, and are signalled with @format) before validation.
+      def formattedstr_strip(doc)
+        doc.xpath("//*[@format] | //stem | //bibdata//description | "\
+                  "//formattedref | //bibdata//note | //bibdata/abstract | "\
+                  "//bibitem/abstract | //bibitem/note | //misc-container")
+          .each do |n|
+          n.elements.each do |e|
+            e.traverse do |e1|
+              e1.element? and e1.each { |k, _v| e1.delete(k) }
+            end
+          end
+        end
+        doc
+      end
+
+      # manually check for xref/@target, xref/@to integrity
+      def xref_validate(doc)
+        ids = doc.xpath("//*/@id").each_with_object({}) { |x, m| m[x.text] = 1 }
+        doc.xpath("//xref/@target | //xref/@to").each do |x|
+          next if ids[x.text]
+
+          @log.add("Anchors", x.parent,
+                   "Crossreference target #{x.text} is undefined")
+        end
+      end
+
+      def validate(doc)
+        content_validate(doc)
+        schema_validate(formattedstr_strip(doc.dup),
+                        File.join(File.dirname(__FILE__), "isodoc.rng"))
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/validate_section.rb

@@ -0,0 +1,55 @@
+require "nokogiri"
+
+module Metanorma
+  module Standoc
+    module Validate
+      def section_validate(doc)
+        sourcecode_style(doc.root)
+        hanging_para_style(doc.root)
+        asset_style(doc.root)
+      end
+
+      def sourcecode_style(root)
+        root.xpath("//sourcecode").each do |x|
+          callouts = x.elements.select { |e| e.name == "callout" }
+          annotations = x.elements.select { |e| e.name == "annotation" }
+          if callouts.size != annotations.size
+            @log.add("AsciiDoc Input", x,
+                     "mismatch of callouts and annotations")
+          end
+        end
+      end
+
+      def style_warning(node, msg, text = nil)
+        w = msg
+        w += ": #{text}" if text
+        @log.add("Metanorma XML Style Warning", node, w)
+      end
+
+      def asset_title_style(root)
+        root.xpath("//figure[image][not(name)]").each do |node|
+          style_warning(node, "Figure should have title", nil)
+        end
+        root.xpath("//table[not(name)]").each do |node|
+          style_warning(node, "Table should have title", nil)
+        end
+      end
+
+      def asset_style(root)
+        asset_title_style(root)
+      end
+
+      def hanging_para_style(root)
+        root.xpath("//clause | //annex | //foreword | //introduction | "\
+                   "//acknowledgements").each do |c|
+          next unless c.at("./clause")
+          next if c.elements.reject do |n|
+                    %w(clause title).include? n.name
+                  end.empty?
+
+          style_warning(c, "Hanging paragraph in clause")
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/version.rb

@@ -19,6 +19,6 @@ module Metanorma
   end
 
   module Standoc
-    VERSION = "1.11.4".freeze
+    VERSION = "2.0.0".freeze
   end
 end

data/lib/metanorma-standoc.rb

@@ -1,6 +1,6 @@
 require "asciidoctor" unless defined? Asciidoctor::Converter
 require_relative "isodoc/pdf_convert"
-require_relative "asciidoctor/standoc/converter"
+require_relative "metanorma/standoc/converter"
 require_relative "metanorma/standoc/version"
 require "asciidoctor/extensions"
 

data/metanorma-standoc.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "asciidoctor", "~> 2.0.0"
   spec.add_dependency "iev", "~> 0.3.0"
-  spec.add_dependency "isodoc", "~> 1.8.0"
+  spec.add_dependency "isodoc", "~> 2.0.0"
   spec.add_dependency "metanorma-plugin-datastruct"
   spec.add_dependency "metanorma-plugin-lutaml"
   spec.add_dependency "ruby-jing"

data/spec/{asciidoctor → metanorma}/base_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "fileutils"
 
-RSpec.describe Asciidoctor::Standoc do
+RSpec.describe Metanorma::Standoc do
   it "has a version number" do
     expect(Metanorma::Standoc::VERSION).not_to be nil
   end
@@ -26,7 +26,7 @@ RSpec.describe Asciidoctor::Standoc do
                <stage>published</stage>
              </status>
              <copyright>
-               <from>2021</from>
+               <from>#{Time.now.year}</from>
              </copyright>
              <ext>
                <doctype>article</doctype>
@@ -222,6 +222,9 @@ RSpec.describe Asciidoctor::Standoc do
       :isbn: ISBN-13
       :isbn10: ISBN-10
       :classification: a:b, c
+      :toclevels: 2
+      :doctoclevels: 3
+      :htmltoclevels: 4
     INPUT
     output = <<~OUTPUT
           <?xml version="1.0" encoding="UTF-8"?>
@@ -481,6 +484,20 @@ RSpec.describe Asciidoctor::Standoc do
         </ics>
         </ext>
       </bibdata>
+       <misc-container>
+   <presentation-metadata>
+     <name>TOC Heading Levels</name>
+     <value>2</value>
+   </presentation-metadata>
+   <presentation-metadata>
+     <name>TOC Heading Levels</name>
+     <value>2</value>
+   </presentation-metadata>
+   <presentation-metadata>
+     <name>TOC Heading Levels</name>
+     <value>2</value>
+   </presentation-metadata>
+ </misc-container>
       <sections/>
       </standard-document>
     OUTPUT
@@ -742,7 +759,7 @@ RSpec.describe Asciidoctor::Standoc do
             <stage>published</stage>
           </status>
           <copyright>
-            <from>2021</from>
+            <from>#{Time.now.year}</from>
           </copyright>
           <ext>
             <doctype>article</doctype>
@@ -870,7 +887,7 @@ RSpec.describe Asciidoctor::Standoc do
           <stage>published</stage>
         </status>
         <copyright>
-          <from>2021</from>
+          <from>#{Time.now.year}</from>
         </copyright>
         <relation type='isCitedIn'>
           <description>normatively cited in</description>
@@ -1000,21 +1017,21 @@ QU1FOiB0ZXN0Cgo=
 
   it "process mn2pdf attributes" do
     node = Nokogiri::XML("<fake/>").at("fake")
-    node[Asciidoctor::Standoc::Base::FONTS_MANIFEST] =
+    node[Metanorma::Standoc::Base::FONTS_MANIFEST] =
       "passed/as/font/manifest/to/mn2pdf.jar"
 
-    options = Asciidoctor::Standoc::Converter
+    options = Metanorma::Standoc::Converter
       .new(:standoc, header_footer: true)
       .doc_extract_attributes(node)
 
     expect(options.dig(:mn2pdf, :font_manifest))
-      .to eq(node[Asciidoctor::Standoc::Base::FONTS_MANIFEST])
+      .to eq(node[Metanorma::Standoc::Base::FONTS_MANIFEST])
   end
 
   private
 
   def mock_org_abbrevs
-    allow_any_instance_of(::Asciidoctor::Standoc::Front)
+    allow_any_instance_of(::Metanorma::Standoc::Front)
       .to receive(:org_abbrev).and_return(
         { "International Standards Organization" => "ISO",
           "International Electrotechnical Commission" => "IEC" },
@@ -1022,14 +1039,14 @@ QU1FOiB0ZXN0Cgo=
   end
 
   def mock_default_publisher
-    allow_any_instance_of(::Asciidoctor::Standoc::Front)
+    allow_any_instance_of(::Metanorma::Standoc::Front)
       .to receive(:default_publisher).and_return(
         "International Standards Organization",
       )
   end
 
   def mock_relaton_relation_descriptions
-    allow_any_instance_of(::Asciidoctor::Standoc::Front)
+    allow_any_instance_of(::Metanorma::Standoc::Front)
       .to receive(:relaton_relation_descriptions).and_return(
         "normatively-cited-in" => "isCitedIn",
       )

data/spec/{asciidoctor → metanorma}/blank_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "fileutils"
 
-RSpec.describe Asciidoctor::Standoc do
+RSpec.describe Metanorma::Standoc do
   it "processes a blank document" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}

data/spec/{asciidoctor → metanorma}/blocks_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "open3"
 
-RSpec.describe Asciidoctor::Standoc do
+RSpec.describe Metanorma::Standoc do
   it "processes format-specific pass blocks" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}

data/spec/{asciidoctor → metanorma}/cleanup_blocks_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 require "fileutils"
 
-RSpec.describe Asciidoctor::Standoc do
+RSpec.describe Metanorma::Standoc do
   it "processes svgmap" do
     FileUtils.cp "spec/fixtures/action_schemaexpg1.svg",
                  "action_schemaexpg1.svg"

data/spec/{asciidoctor → metanorma}/cleanup_sections_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 require "relaton_iec"
 require "fileutils"
 
-RSpec.describe Asciidoctor::Standoc do
+RSpec.describe Metanorma::Standoc do
   it "appends any initial user-supplied text to boilerplate in terms and definitions" do
     input = <<~INPUT
       #{ASCIIDOC_BLANK_HDR}