metanorma-standoc 3.0.4 → 3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86e5f93e957beab663c8b30d6ea82cc211b158a6bf8b9f0e49a9843c21e910d2
-  data.tar.gz: 3b98332cd095ac90319145032f637b3fc1388ddc0f5ccbb6b1599d0b331cdd35
+  metadata.gz: 982e6533abd03671b4c76ba32204915843cdc0c644b06ec0e8a34798a5d50d1d
+  data.tar.gz: c12cb3a77574fef2d8aa219dae15470ed14ee50e3d7ca55c527f54dc73242a4e
 SHA512:
-  metadata.gz: 2e8aa9bd17082e4e9a06f26635282cb4eb3088ecf636c7144299b57d547cfb2ef2913f286bf67aa40557670b46abd0625d9545f9a8812b77cb597c2e00ef3af4
-  data.tar.gz: ec6148203ca15276fe22bcad392a30e1adb581791f6c4577f20fffaa401998f46384b0f374625dc34c40cc02f53695777eb974b1f678c942598d4efdafaf7885
+  metadata.gz: eec8efe148aeef9f65007f135aed6ad0a91c7c8c0a94749d75c1946d53e0877a3c8840c32679e20d01048fee434d35949453dcfd7335417f049f225eab31afdf
+  data.tar.gz: 8e2dc0a69568e2f460dcf39fa081756ed0d5610b00c02516c02ffd14bb84682a03dc07a29dc261651f65576175c36561d53a80cea8889c9519b8053c3051c612

data/lib/metanorma/standoc/base.rb

@@ -11,30 +11,28 @@ require_relative "localbib"
 require_relative "init"
 require "mn-requirements"
 
-require 'set'
-
 module Asciidoctor
   module Compliance
-    ADMONITION_STYLES.add('EDITOR') unless ADMONITION_STYLES.include?('EDITOR')
+    ADMONITION_STYLES.add("EDITOR") unless ADMONITION_STYLES.include?("EDITOR")
   end
 end
 
 module Metanorma
   module Standoc
     module Base
-      #XML_ROOT_TAG = "standard-document".freeze
-      #XML_NAMESPACE = "https://www.metanorma.org/ns/standoc".freeze
+      # XML_ROOT_TAG = "standard-document".freeze
+      # XML_NAMESPACE = "https://www.metanorma.org/ns/standoc".freeze
       FONTS_MANIFEST = "fonts-manifest".freeze
 
       attr_accessor :log
 
       def xml_root_tag
-        #self.class::XML_ROOT_TAG
+        # self.class::XML_ROOT_TAG
         "metanorma"
       end
 
       def xml_namespace
-        #self.class::XML_NAMESPACE
+        # self.class::XML_NAMESPACE
         "https://www.metanorma.org/ns/standoc"
       end
 

data/lib/metanorma/standoc/basicdoc.rng

@@ -1701,16 +1701,22 @@ which can be bookmarks as well as block or section references</a:documentation>
     <a:documentation>Inline reference to a paragraph or paragraphs, appearing as a footnote.
 The target of a footnote is the location it is embedded in within the text</a:documentation>
     <element name="fn">
-      <attribute name="reference">
-        <a:documentation>The number of the footnote, used to identify it visually</a:documentation>
-      </attribute>
-      <oneOrMore>
-        <ref name="paragraph">
-          <a:documentation>The content of the footnote</a:documentation>
-        </ref>
-      </oneOrMore>
+      <ref name="FnAttributes"/>
+      <ref name="FnBody"/>
     </element>
   </define>
+  <define name="FnBody">
+    <oneOrMore>
+      <ref name="paragraph">
+        <a:documentation>The content of the footnote</a:documentation>
+      </ref>
+    </oneOrMore>
+  </define>
+  <define name="FnAttributes">
+    <attribute name="reference">
+      <a:documentation>The number of the footnote, used to identify it visually</a:documentation>
+    </attribute>
+  </define>
   <define name="callout">
     <a:documentation>Inline reference to a paragraph or paragraphs, appearing as annotation of source code</a:documentation>
     <element name="callout">

data/lib/metanorma/standoc/biblio-standoc.rng

@@ -1,13 +1,14 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <grammar xmlns:a="http://relaxng.org/ns/compatibility/annotations/1.0" xmlns="http://relaxng.org/ns/structure/1.0">
-  <!--
-    Add-ons to biblio.rnc for standoc model: defines the extension point BibDataExtensionType
-    of relaton
-    
-    Specialisations as for biblio.rnc. Extension point can be redefined completely for a flavour of standoc
-    (SDO); but other elements in Bibdata can only be extended (more specialised vocabularies for Bibdata)
-  -->
   <include href="biblio.rng">
+    <!-- ALERT: we cannot have comments on root element, as they intervene with https://github.com/metanorma/metanorma/issues/437 fix -->
+    <!--
+      Add-ons to biblio.rnc for standoc model: defines the extension point BibDataExtensionType
+      of relaton
+      
+      Specialisations as for biblio.rnc. Extension point can be redefined completely for a flavour of standoc
+      (SDO); but other elements in Bibdata can only be extended (more specialised vocabularies for Bibdata)
+    -->
     <define name="BibData">
       <a:documentation>The bibliographic description of a standardisation document</a:documentation>
       <ref name="StandardBibliographicItem"/>
@@ -91,6 +92,9 @@ a standards definition organization</a:documentation>
         <a:documentation>Representation of the identifier for the standardisation document, giving its individual semantic components</a:documentation>
       </ref>
     </zeroOrMore>
+    <ref name="DocumentImages">
+      <a:documentation>Coverpage and other images to be rendered with document</a:documentation>
+    </ref>
   </define>
   <define name="doctype">
     <a:documentation>Classification of the standardisation document</a:documentation>
@@ -268,6 +272,32 @@ and not those document components</a:documentation>
       </optional>
     </element>
   </define>
+  <define name="DocumentImages">
+    <zeroOrMore>
+      <element name="coverpage-image">
+        <a:documentation>Images to be displayed on the coverpage of the document</a:documentation>
+        <ref name="image-no-id"/>
+      </element>
+    </zeroOrMore>
+    <zeroOrMore>
+      <element name="innercoverpage-image">
+        <a:documentation>Images to be displayed on the inner coverpage of the document</a:documentation>
+        <ref name="image-no-id"/>
+      </element>
+    </zeroOrMore>
+    <zeroOrMore>
+      <element name="tocside-image">
+        <a:documentation>Images to be displayed on the Table of Contents page of the document</a:documentation>
+        <ref name="image-no-id"/>
+      </element>
+    </zeroOrMore>
+    <zeroOrMore>
+      <element name="backpage-image">
+        <a:documentation>Images to be displayed on the backpage of the document</a:documentation>
+        <ref name="image-no-id"/>
+      </element>
+    </zeroOrMore>
+  </define>
   <define name="StandardBibliographicItem">
     <ref name="BibliographicItem"/>
     <zeroOrMore>

data/lib/metanorma/standoc/biblio.rng

@@ -1,23 +1,25 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  instantiations of this grammar may replace leaf strings
-  with more elaborated types; e.g. title (text) replaced with
-  title-main, title-intro, title-part; type replaced with
-  enum.
-  
-  some renaming at leaf nodes is permissible
-  
-  obligations can change both from optional to mandatory,
-  and from mandatory to optional; optional elements may
-  be omitted; freely positioned alternatives may be replaced
-  with strict ordering
-  
-  DO NOT introduce a namespace here. We do not want a distinct namespace
-  for these elements, and a distinct namespace for any grammar inheriting
-  these elements; we just want one namespace for any child grammars
-  of this.
--->
 <grammar xmlns:a="http://relaxng.org/ns/compatibility/annotations/1.0" xmlns="http://relaxng.org/ns/structure/1.0" datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
+  <!--
+    ALERT: we cannot have comments on root element, as they intervene with https://github.com/metanorma/metanorma/issues/437 fix
+    
+    Instantiations of this grammar may replace leaf strings
+    with more elaborated types; e.g. title (text) replaced with
+    title-main, title-intro, title-part; type replaced with
+    enum.
+    
+    Some renaming at leaf nodes is permissible
+    
+    Obligations can change both from optional to mandatory,
+    and from mandatory to optional; optional elements may
+    be omitted; freely positioned alternatives may be replaced
+    with strict ordering
+    
+    DO NOT introduce a namespace here. We do not want a distinct namespace
+    for these elements, and a distinct namespace for any grammar inheriting
+    these elements; we just want one namespace for any child grammars
+    of this.
+  -->
   <!--
     https://www.myintervals.com/blog/2009/05/20/iso-8601-date-validation-that-doesnt-suck/
     iso8601date = xsd:string { pattern = "([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?" }
@@ -1241,6 +1243,11 @@ Refer to `BibliographicItem` for definitions</a:documentation>
   </define>
   <define name="formattedref">
     <element name="formattedref">
+      <optional>
+        <attribute name="format">
+          <a:documentation>format of formatted reference; Metanorma assumes references are formatted as Metanorma XML</a:documentation>
+        </attribute>
+      </optional>
       <oneOrMore>
         <ref name="TextElement"/>
       </oneOrMore>
@@ -1812,6 +1819,11 @@ May be used to differentiate rendering of notes in bibliographies</a:documentati
     <a:documentation>Abstract of bibliographic item</a:documentation>
     <element name="abstract">
       <ref name="LocalizedStringAttributes"/>
+      <optional>
+        <attribute name="format">
+          <a:documentation>What format the formatted abstract is in. In Metanorma, assumed to be Metanorma XML</a:documentation>
+        </attribute>
+      </optional>
       <choice>
         <oneOrMore>
           <ref name="BasicBlockNoId">

data/lib/metanorma/standoc/blocks.rb

@@ -211,13 +211,34 @@ module Metanorma
       end
 
       def pass(node)
+        format = node.attr("format") || "metanorma"
         noko do |xml|
-          xml.passthrough **attr_code(formats:
-                                      node.attr("format") || "metanorma") do |p|
-            p << @c.encode(node.content, :basic, :hexadecimal)
+          xml.passthrough **attr_code(formats: format) do |p|
+            content = @c.encode(node.content, :basic, :hexadecimal)
+            p << content
+            format == "metanorma" and
+              passthrough_validate(node, node.content, content)
           end
         end
       end
+
+      PASSTHROUGH_ERR = <<~ERRMSG.freeze
+        This is not valid Metanorma XML. If you intended a different format, such as HTML, you need to specify `format=` on the pass markup;
+        refer to https://www.metanorma.org/author/topics/blocks/passthroughs/
+      ERRMSG
+
+      # need to validate Metanorma XML before it passes to textcleanup,
+      # where passthrough wrapper and escaped tags are removed:
+      # <passthrough format="metanorma>&lt;tag&gt</passthrough> => <tag>
+      # Do not treat not well-formed XML as invalid,
+      # as it may be fragment, e.g. unterminated start of element markup
+      def passthrough_validate(node, content, encoded_content)
+        valid, = validate_document_fragment(content.dup)
+        err =
+          "Invalid passthrough content: #{encoded_content}\n#{PASSTHROUGH_ERR}"
+        !valid and
+          @log.add("Metanorma XML Syntax", node, err, severity: 0)
+      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup.rb

@@ -19,6 +19,7 @@ require_relative "cleanup_image"
 require_relative "cleanup_reqt"
 require_relative "cleanup_text"
 require_relative "cleanup_toc"
+require_relative "cleanup_review"
 require_relative "cleanup_dochistory"
 require "relaton_iev"
 

data/lib/metanorma/standoc/cleanup_bibdata.rb

@@ -58,7 +58,7 @@ module Metanorma
         end
       end
 
-      def indirect_eref_to_xref(eref, ident)
+      def indirect_eref_to_xref(eref, ident, id_map=nil)
         loc = eref.at("./localityStack[locality[@type = 'anchor']]") ||
           eref.at("./locality[@type = 'anchor']")
         loc = loc&.remove&.text || ident
@@ -66,18 +66,30 @@ module Metanorma
         eref.delete("bibitemid")
         eref.delete("citeas")
         eref["target"] = loc
-        eref.document.at("//*[@id = '#{loc}']") and return
+        if id_map
+          return if id_map.has_key?(loc)
+        else
+          eref.document.at("//*[@id = '#{loc}']") and return
+        end
         eref.children = %(** Missing target #{loc})
         eref["target"] = ident
       end
 
       def resolve_local_indirect_erefs(xmldoc, refs, prefix)
+        # Pre-index elements by ID
+        id_map = xmldoc.xpath("//*[@id]").each_with_object({}) do |node, map|
+          map[node["id"]] = node
+        end
+
+        # Pre-index all <eref> elements by bibitemid
+        eref_map = xmldoc.xpath("//eref[@bibitemid]").group_by { |e| e["bibitemid"] }
+
         refs.each_with_object([]) do |r, m|
           id = r.sub(/^#{prefix}_/, "")
-          n = xmldoc.at("//*[@id = '#{id}']")
+          n = id_map[id]
           if n&.at("./ancestor-or-self::*[@type = '#{prefix}']")
-            xmldoc.xpath("//eref[@bibitemid = '#{r}']").each do |e|
-              indirect_eref_to_xref(e, id)
+            eref_map[r]&.each do |e|
+              indirect_eref_to_xref(e, id, id_map)
             end
           else m << r
           end

data/lib/metanorma/standoc/cleanup_inline.rb

@@ -196,6 +196,9 @@ module Metanorma
           p.name = "passthrough"
           p.children = select_odd_chars(p.children.to_xml)
         end
+        doc.xpath("//passthrough[@format = 'metanorma']").each do |p|
+          p.replace(p.children)
+        end
         doc.xpath("//identifier").each do |p|
           p.children = select_odd_chars(p.children.to_xml)
         end

data/lib/metanorma/standoc/cleanup_maths.rb

@@ -26,9 +26,6 @@ module Metanorma
         asciimath2mathml_err(elem.to_xml, e)
       end
 
-      # https://medium.com/@rickwang_wxc/in-ruby-given-a-string-detect-if-it-is-valid-numeric-c58275eace60
-      NUMERIC_REGEX = %r{^((\+|-)?\d*\.?\d+)([eE](\+|-){1}\d+)?$}
-
       MATHML_NS = "http://www.w3.org/1998/Math/MathML".freeze
 
       def asciimath_parse(expr, elem)
@@ -169,8 +166,7 @@ module Metanorma
           return
         end
         f == "default" or return f
-        if @numberfmt_default.empty?
-          "notation='basic'"
+        if @numberfmt_default.empty? then "notation='basic'"
         else @numberfmt_default&.map { |k, v| "#{k}='#{v}'" }&.join(",")
         end
       end
@@ -191,6 +187,8 @@ module Metanorma
           .each { |x| mathml_number_format(x) }
         mathml_unitsml(xmldoc)
       end
+
+      include ::Metanorma::Standoc::Regex
     end
   end
 end

data/lib/metanorma/standoc/cleanup_review.rb

@@ -0,0 +1,76 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def review_cleanup(xmldoc)
+        reviews = xmldoc.xpath("//review")
+        reviews.empty? and return
+        ctr = xmldoc.root.add_child("<review-container/>").first
+        reviews.each do |r|
+          review_set_location(r)
+          ctr << r
+        end
+      end
+
+      def review_insert_bookmark(review, id)
+        parent = review.parent
+        children = parent.children
+        index = children.index(review)
+        x = find_review_sibling(children, index, :previous) ||
+          find_review_sibling(children, index, :following)
+        ins = x || review.before("<p> </p>").previous.at(".//text()")
+        ins.previous = "<bookmark id='#{id}'/>"
+      end
+
+      # we know node is a block: dig for a place bookmark can go
+      def available_bookmark_destination(node)
+        ret = case node.name
+              when "title", "name", "p" then node
+              when "sourcecode" then node.at(".//name")
+              when "admonition", "note", "example", "li", "quote", "dt", "dd",
+                "permission", "requirement", "recommendation"
+                node.at(".//p | .//name") || node
+              when "formula"
+                node.at(".//p | .//name | .//dt")
+              when "ol", "ul" then node.at(".//p | .//name") || node.at("./li")
+              when "dl" then node.at(".//p | .//name") || node.at("./dt | ./dd")
+              when "table" then node.at(".//td[text()] | .//th[text()]")
+              end or return nil
+        first_non_stem_text(ret)
+      end
+
+      def first_non_stem_text(ret)
+        first_non_stem_text = nil
+        ret.traverse do |n|
+          if n.text? && n.ancestors("stem").empty? && !n.text.strip.empty?
+            first_non_stem_text = n
+            break
+          end
+        end
+        first_non_stem_text
+      end
+
+      def find_review_sibling(children, index, direction = :previous)
+        range = if direction == :previous then (index - 1).downto(0)
+                else (index + 1).upto(children.size - 1)
+                end
+        range.each do |i|
+          node = children[i]
+          if node.element? && !node.text.empty? && node.text.strip != "" &&
+              ret = available_bookmark_destination(node)
+            return ret
+          end
+        end
+        nil
+      end
+
+      def review_set_location(review)
+        unless review["from"]
+          id = "_#{UUIDTools::UUID.random_create}"
+          review_insert_bookmark(review, id)
+          review["from"] = id
+        end
+        review["to"] ||= review["from"]
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/cleanup_section.rb

@@ -96,9 +96,7 @@ module Metanorma
           y.name == "annex" || !y.ancestors("annex").empty? and next
           y.wrap("<annex/>")
           y.parent["id"] = "_#{UUIDTools::UUID.random_create}"
-          %w(obligation language script).each do |w|
-            y.parent[w] = y[w]
-          end
+          %w(obligation language script).each { |w| y.parent[w] = y[w] }
         end
       end
 
@@ -208,33 +206,6 @@ module Metanorma
           ins.previous = x.remove
         end
       end
-
-      def insert_before(xmldoc, xpath)
-        unless ins = xmldoc.at(xpath).children.first
-          xmldoc.at(xpath) << " "
-          ins = xmldoc.at(xpath).children.first
-        end
-        ins
-      end
-
-      def review_cleanup(xmldoc)
-        reviews = xmldoc.xpath("//review")
-        reviews.empty? and return
-        ctr = xmldoc.root.add_child("<review-container/>").first
-        reviews.each do |r|
-          review_set_location(r)
-          ctr << r
-        end
-      end
-
-      def review_set_location(review)
-        unless review["from"]
-          id = "_#{UUIDTools::UUID.random_create}"
-          review.previous = "<bookmark id='#{id}'/>"
-          review["from"] = id
-        end
-        review["to"] ||= review["from"]
-      end
     end
   end
 end

data/lib/metanorma/standoc/cleanup_text.rb

@@ -16,16 +16,6 @@ module Metanorma
         !path.intersection(ancestors).empty?
       end
 
-      def linebreak_cleanup(xmldoc)
-        xmldoc.traverse do |x|
-          x.text? && x.text.include?("\n") or next
-          ancestor_include?(x, PRESERVE_LINEBREAK_ELEMENTS) and next
-          ancestor_include?(x, STRIP_LINEBREAK_ELEMENTS) or next
-          x.replace(Metanorma::Utils
-            .line_sanitise(x.text.lines.map(&:rstrip)).join)
-        end
-      end
-
       # process example/p, example/sourcecode, not example on its own:
       # this is about stripping lines for blocks containing inline elems & text
       def linebreak_cleanup(xmldoc)
@@ -57,11 +47,15 @@ module Metanorma
 
       def gather_text_for_linebreak_cleanup(block)
         x = block.xpath(".//text()").map do |e|
-          { elem: e, text: e.text,
+          { elem: e, text: e.text, stem: ancestor_include?(e, %w(stem)),
             skip: ancestor_include?(e, PRESERVE_LINEBREAK_ELEMENTS) }
         end
         x.empty? and return x
         x.each { |e| e[:skip] ||= !e[:text].include?("\n") }
+        x.each_with_index do |e, i|
+          # do not treat stem linebreaks as meaningful
+          e[:skip] ||= x[i + 1]&.dig(:stem)
+        end
         x[-1][:last] = true
         x
       end
@@ -71,6 +65,9 @@ module Metanorma
         if @smartquotes then smartquotes_cleanup1(xmldoc)
         else dumbquote_cleanup(xmldoc)
         end
+        xmldoc.xpath("//passthrough[@formats = 'straightquotes']").each do |x|
+          x.replace(x.children)
+        end
       end
 
       def smartquotes_cleanup1(xmldoc)
@@ -91,7 +88,7 @@ module Metanorma
       # "abc<tag/>", def => "abc",<tag/> def
       # TODO?
       def uninterrupt_quotes_around_xml1(xmldoc)
-          xmldoc.xpath("//text()[preceding-sibling::*[1]]").each do |n|
+        xmldoc.xpath("//text()[preceding-sibling::*[1]]").each do |n|
           uninterrupt_quotes_around_xml_skip(n) and next
           uninterrupt_quotes_around_xml1(n.previous)
         end
@@ -102,7 +99,7 @@ module Metanorma
            identifier metanorma-extension).freeze
 
       PRESERVE_LINEBREAK_ELEMENTS =
-        %w(pre sourcecode passthrough metanorma-extension).freeze
+        %w(pre sourcecode passthrough metanorma-extension stem).freeze
 
       STRIP_LINEBREAK_ELEMENTS =
         %w(title name variant-title figure example review admonition
@@ -157,29 +154,12 @@ module Metanorma
           block?(x) and prev = ""
           empty_tag_with_text_content?(x) and prev = "dummy"
           x.text? or next
-
-          # ancestors = x.path.gsub(/\[\d+\]/, "").split(%r{/})[1..-2]
-          # ancestors.intersection(IGNORE_QUOTES_ELEMENTS).empty? or next
           ancestor_include?(x, IGNORE_QUOTES_ELEMENTS) and next
           dumb2smart_quotes1(x, prev)
           prev = x.text
         end
       end
 
-      def dumb2smart_quotesx(xmldoc)
-        # TODO?>
-        prev = ""
-        xmldoc.xpath("//* | //text()").each do |x|
-          x.is_a?(Nokogiri::XML::Node) or next
-          block?(x) and prev = ""
-          empty_tag_with_text_content?(x) and prev = "dummy"
-          x.text? or next
-ancestor_include?(x, IGNORE_QUOTES_ELEMENTS) and next
- dumb2smart_quotes1(x, prev)
-          prev = x.text
-        end
-      end
-
       def dumb2smart_quotes1(curr, prev)
         /[-'"(<>]|\.\.|\dx/.match?(curr.text) or return
 
@@ -190,7 +170,7 @@ ancestor_include?(x, IGNORE_QUOTES_ELEMENTS) and next
 
       def dumbquote_cleanup(xmldoc)
         xmldoc.traverse do |n|
-          next unless n.text? && /\u2019/.match?(n.text)
+          next unless n.text? && n.text.include?("\u2019")
 
           n.replace(@c.encode(
                       @c.decode(n.text)

data/lib/metanorma/standoc/cleanup_xref.rb

@@ -1,33 +1,6 @@
 module Metanorma
   module Standoc
     module Cleanup
-      # extending localities to cover ISO referencing
-      CONN_REGEX_STR = "(?<conn>and|or|from|to)!".freeze
-
-      LOCALITIES = "section|clause|part|paragraph|chapter|page|line|" \
-        "table|annex|figure|example|note|formula|list|time|anchor|" \
-        "locality:[^ \\t\\n\\r:,;=]+".freeze
-
-      LOCALITY_REGEX_STR = <<~REGEXP.freeze
-        ^((#{CONN_REGEX_STR})?
-            (?<locality>#{LOCALITIES})(\\s+|=)
-               (?<ref>[^"][^ \\t\\n,:;-]*|"[^"]+")
-                 (-(?<to>[^"][^ \\t\\n,:;-]*|"[^"]"))?|
-          (?<locality2>whole|title|locality:[^ \\t\\n\\r:,;=]+))(?<punct>[,:;]?)\\s*
-         (?<text>.*)$
-      REGEXP
-
-      def to_regex(str)
-        Regexp.new(str.gsub(/\s/, ""), Regexp::IGNORECASE | Regexp::MULTILINE)
-      end
-
-      LOCALITY_REGEX_VALUE_ONLY_STR = <<~REGEXP.freeze
-        ^(?<conn0>(#{CONN_REGEX_STR}))
-          (?!whole|title|locality:)
-          (?<value>[^=,;:\\t\\n\\r]+)
-          (?<punct>[,;\\t\\n\\r]|$)
-      REGEXP
-
       def tq(text)
         text.sub(/^"/, "").sub(/"$/, "")
       end
@@ -43,16 +16,6 @@ module Metanorma
         d.children.empty? and d.remove
       end
 
-      LOCALITY_REGEX_STR_TRIPLEDASH = <<~REGEXP.freeze
-        ^(?<locality>(#{CONN_REGEX_STR})?
-            (#{LOCALITIES})(\\s+|=))
-               (?<ref>[^"][^ \\t\\n,:;-]*
-                 -[^ \\t\\n,:;"-]+
-                 -[^ \\t\\n,:;"]+)
-          (?<text>[,:;]?\\s*
-         .*)$
-      REGEXP
-
       # treat n-n-n locality as "n-n-n", do not parse as a range
       def locality_normalise(text)
         re = to_regex(LOCALITY_REGEX_STR_TRIPLEDASH)
@@ -261,6 +224,8 @@ module Metanorma
           extract_localities(x)
         end
       end
+
+      include ::Metanorma::Standoc::Regex
     end
   end
 end

data/lib/metanorma/standoc/front.rb

@@ -1,5 +1,4 @@
 require "date"
-require "htmlentities"
 require "pathname"
 require_relative "./front_contributor"
 require "isoics"
@@ -13,7 +12,7 @@ module Metanorma
       end
 
       def metadata_id_build(node)
-        part, subpart = node&.attr("partnumber")&.split(/-/)
+        part, subpart = node&.attr("partnumber")&.split("-")
         id = node.attr("docnumber") || ""
         id += "-#{part}" if part
         id += "-#{subpart}" if subpart
@@ -189,9 +188,12 @@ module Metanorma
         metadata_flavor(node, ext)
         metadata_committee(node, ext)
         metadata_ics(node, ext)
+        structured_id(node, ext)
         metadata_coverpage_images(node, ext)
       end
 
+      def structured_id(node, xml); end
+
       def metadata_doctype(node, xml)
         xml.doctype doctype(node)
       end
@@ -218,7 +220,8 @@ module Metanorma
           at = { language: lang, format: "text/plain" }
           xml.title **attr_code(at) do |t|
             title = Metanorma::Utils::asciidoc_sub(
-              node.attr("title") || node.attr("title-en") || node.attr("doctitle"))
+              node.attr("title") || node.attr("title-en") || node.attr("doctitle"),
+            )
             t << title
           end
         end

data/lib/metanorma/standoc/init.rb

@@ -1,4 +1,5 @@
 require_relative "utils"
+require_relative "regex"
 
 module Metanorma
   module Standoc