metanorma-standoc 1.11.3 → 2.0.2

data/lib/metanorma/standoc/macros_terms.rb

@@ -0,0 +1,194 @@
+require "csv"
+
+module Metanorma
+  module Standoc
+    class PreferredTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :preferred
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted, attrs["text"]).convert
+        %{<preferred><expression><name>#{out}</name></expression></preferred>}
+      end
+    end
+
+    class AltTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :alt
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted,
+                                      attrs["text"]).convert
+        %{<admitted><expression><name>#{out}</name></expression></admitted>}
+      end
+    end
+
+    class AdmittedTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :admitted
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted,
+                                      attrs["text"]).convert
+        %{<admitted><expression><name>#{out}</name></expression></admitted>}
+      end
+    end
+
+    class DeprecatedTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :deprecated
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted,
+                                      attrs["text"]).convert
+        %{<deprecates><expression><name>#{out}</name></expression></deprecates>}
+      end
+    end
+
+    class DomainTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :domain
+      parse_content_as :text
+      using_format :short
+
+      def process(parent, _target, attrs)
+        out = Asciidoctor::Inline.new(parent, :quoted,
+                                      attrs["text"]).convert
+        %{<domain>#{out}</domain>}
+      end
+    end
+
+    class TermRefInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :term
+      name_positional_attributes "name", "termxref"
+      using_format :short
+
+      def process(_parent, _target, attrs)
+        termref = attrs["termxref"] || attrs["name"]
+        "<concept type='term'><termxref>#{attrs['name']}</termxref>"\
+          "<renderterm>#{termref}</renderterm><xrefrender/></concept>"
+      end
+    end
+
+    class SymbolRefInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :symbol
+      name_positional_attributes "name", "termxref"
+      using_format :short
+
+      def process(_parent, _target, attrs)
+        termref = attrs["termxref"] || attrs["name"]
+        "<concept type='symbol'><termxref>#{attrs['name']}</termxref>"\
+          "<renderterm>#{termref}</renderterm><xrefrender/></concept>"
+      end
+    end
+
+    # Possibilities:
+    # {{<<id>>, term}}
+    # {{<<id>>, term, text}}
+    # {{<<termbase:id>>, term}}
+    # {{<<termbase:id>>, term, text}}
+    # {{term}} equivalent to term:[term]
+    # {{term, text}} equivalent to term:[term, text]
+    # text may optionally be followed by crossreference-rendering, options=""
+    class ConceptInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :concept
+      match /\{\{(?<content>|.*?[^\\])\}\}/
+      using_format :short
+
+      def preprocess_attrs(target)
+        m = /^(?<id>&lt;&lt;.+?&gt;&gt;)?(?<rest>.*)$/.match(target)
+        ret = { id: m[:id]&.sub(/^&lt;&lt;/, "")&.sub(/&gt;&gt;$/, "") }
+        if m2 = /^(?<rest>.*?)(?<opt>,opt(ion)?s=.+)$/
+            .match(m[:rest].sub(/^,/, ""))
+          ret[:opt] = CSV.parse_line(m2[:opt].sub(/^,opt(ion)?s=/, "")
+            .sub(/^"(.+)"$/, "\\1").sub(/^'(.+)'$/, "\\1"))
+          begin
+            attrs = CSV.parse_line(m2[:rest]) || []
+          rescue StandardError
+            raise "error processing #{m2[:rest]} as CSV"
+          end
+        else
+          begin
+            attrs = CSV.parse_line(m[:rest].sub(/^,/, "")) || []
+          rescue StandardError
+            raise "error processing #{m[:rest]} as CSV"
+          end
+        end
+        ret.merge(term: attrs[0], word: attrs[1] || attrs[0],
+                  render: attrs[2])
+      end
+
+      def generate_attrs(opts)
+        ret = ""
+        opts.include?("noital") and ret += " ital='false'"
+        opts.include?("noref") and ret += " ref='false'"
+        opts.include?("ital") and ret += " ital='true'"
+        opts.include?("ref") and ret += " ref='true'"
+        opts.include?("nolinkmention") and ret += " linkmention='false'"
+        opts.include?("linkmention") and ret += " linkmention='true'"
+        opts.include?("nolinkref") and ret += " linkref='false'"
+        opts.include?("linkref") and ret += " linkref='true'"
+        ret
+      end
+
+      def process(parent, target, _attrs)
+        attrs = preprocess_attrs(target)
+        term = Asciidoctor::Inline.new(parent, :quoted,
+                                       attrs[:term]).convert
+        word = Asciidoctor::Inline.new(parent, :quoted,
+                                       attrs[:word]).convert
+        xref = Asciidoctor::Inline.new(parent, :quoted,
+                                       attrs[:render]).convert
+        opt = generate_attrs(attrs[:opt] || [])
+        if attrs[:id] then "<concept#{opt} key='#{attrs[:id]}'><refterm>"\
+          "#{term}</refterm><renderterm>#{word}</renderterm>"\
+          "<xrefrender>#{xref}</xrefrender></concept>"
+        else "<concept#{opt}><termxref>#{term}</termxref><renderterm>"\
+          "#{word}</renderterm><xrefrender>#{xref}</xrefrender></concept>"
+        end
+      rescue StandardError => e
+        raise("processing {{#{target}}}: #{e.message}")
+      end
+    end
+
+    # Possibilities:
+    # related:relation[<<id>>, term]
+    # related:relation[<<termbase:id>>, term]
+    # related:relation[term] equivalent to a crossreference to term:[term]
+    class RelatedTermInlineMacro < Asciidoctor::Extensions::InlineMacroProcessor
+      use_dsl
+      named :related
+      parse_content_as :text
+
+      def preprocess_attrs(target)
+        m = /^(?<id>&lt;&lt;.+?&gt;&gt;, ?)?(?<rest>.*)$/.match(target)
+        { id: m[:id]&.sub(/^&lt;&lt;/, "")&.sub(/&gt;&gt;, ?$/, ""),
+          term: m[:rest] }
+      end
+
+      def process(parent, target, attrs)
+        out = preprocess_attrs(attrs["text"])
+        term = Asciidoctor::Inline.new(parent, :quoted,
+                                       out[:term]).convert
+        if out[:id] then "<related type='#{target}' key='#{out[:id]}'>"\
+          "<refterm>#{term}</refterm></related>"
+        else "<related type='#{target}'><termxref>#{term}</termxref>"\
+          "<xrefrender>#{term}</xrefrender></related>"
+        end
+      rescue StandardError => e
+        raise("processing related:#{target}[#{attrs['text']}]: #{e.message}")
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/ref.rb

@@ -0,0 +1,243 @@
+require_relative "ref_utility"
+
+module Metanorma
+  module Standoc
+    module Refs
+      def iso_publisher(bib, code)
+        code.sub(/ .*$/, "").split("/").each do |abbrev|
+          bib.contributor do |c|
+            c.role **{ type: "publisher" }
+            c.organization do |org|
+              organization(org, abbrev, true)
+            end
+          end
+        end
+      end
+
+      def isorefrender1(bib, match, year, allp = "")
+        bib.title(**plaintxt) { |i| i << ref_normalise(match[:text]) }
+        docid(bib, match[:usrlbl]) if match[:usrlbl]
+        docid(bib, id_and_year(match[:code], year) + allp)
+        docnumber(bib, match[:code])
+      end
+
+      def isorefmatchescode(match)
+        yr = norm_year(match[:year])
+        { code: match[:code], year: yr, match: match,
+          title: match[:text], usrlbl: match[:usrlbl],
+          lang: (@lang || :all) }
+      end
+
+      def isorefmatchesout(item, xml)
+        if item[:doc] then use_retrieved_relaton(item, xml)
+        else
+          xml.bibitem **attr_code(ref_attributes(item[:ref][:match])) do |t|
+            isorefrender1(t, item[:ref][:match], item[:ref][:year])
+            item[:ref][:year] and t.date **{ type: "published" } do |d|
+              set_date_range(d, item[:ref][:year])
+            end
+            iso_publisher(t, item[:ref][:match][:code])
+          end
+        end
+      end
+
+      def isorefmatches2code(match)
+        { code: match[:code], no_year: true,
+          note: match[:fn], year: nil, match: match,
+          title: match[:text], usrlbl: match[:usrlbl],
+          lang: (@lang || :all) }
+      end
+
+      def isorefmatches2out(item, xml)
+        if item[:doc] then use_retrieved_relaton(item, xml)
+        else isorefmatches2_1(xml, item[:ref][:match])
+        end
+      end
+
+      def isorefmatches2_1(xml, match)
+        xml.bibitem **attr_code(ref_attributes(match)) do |t|
+          isorefrender1(t, match, "--")
+          t.date **{ type: "published" } do |d|
+            d.on "--"
+          end
+          iso_publisher(t, match[:code])
+          unless match[:fn].nil?
+            t.note(**plaintxt.merge(type: "Unpublished-Status")) do |p|
+              p << (match[:fn]).to_s
+            end
+          end
+        end
+      end
+
+      def isorefmatches3code(match)
+        yr = norm_year(match[:year])
+        hasyr = !yr.nil? && yr != "--"
+        { code: match[:code], match: match, yr: yr, hasyr: hasyr,
+          year: hasyr ? yr : nil,
+          all_parts: true, no_year: yr == "--",
+          text: match[:text], usrlbl: match[:usrlbl],
+          lang: (@lang || :all) }
+      end
+
+      def isorefmatches3out(item, xml)
+        if item[:doc] then use_retrieved_relaton(item, xml)
+        else
+          isorefmatches3_1(xml, item[:ref][:match], item[:ref][:yr],
+                           item[:ref][:hasyr], item[:doc])
+        end
+      end
+
+      def isorefmatches3_1(xml, match, yr, _hasyr, _ref)
+        xml.bibitem(**attr_code(ref_attributes(match))) do |t|
+          isorefrender1(t, match, yr, " (all parts)")
+          conditional_date(t, match, yr == "--")
+          iso_publisher(t, match[:code])
+          if match.names.include?("fn") && match[:fn]
+            t.note(**plaintxt.merge(type: "Unpublished-Status")) do |p|
+              p << (match[:fn]).to_s
+            end
+          end
+          t.extent **{ type: "part" } do |e|
+            e.referenceFrom "all"
+          end
+        end
+      end
+
+      def refitem_render1(match, code, bib)
+        if code[:type] == "path"
+          bib.uri code[:key].sub(/\.[a-zA-Z0-9]+$/, ""), **{ type: "URI" }
+          bib.uri code[:key].sub(/\.[a-zA-Z0-9]+$/, ""), **{ type: "citation" }
+        end
+        code[:id].sub!(/[:-](19|20)[0-9][0-9]$/, "")
+        docid(bib, match[:usrlbl]) if match[:usrlbl]
+        docid(bib, /^\d+$/.match?(code[:id]) ? "[#{code[:id]}]" : code[:id])
+        code[:type] == "repo" and
+          bib.docidentifier code[:key], **{ type: "repository" }
+      end
+
+      def refitem_render(xml, match, code)
+        xml.bibitem **attr_code(id: match[:anchor],
+                                hidden: code[:hidden]) do |t|
+          t.formattedref **{ format: "application/x-isodoc+xml" } do |i|
+            i << ref_normalise_no_format(match[:text])
+          end
+          yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code[:id])
+          refitem_render1(match, code, t)
+          docnumber(t, code[:id]) unless /^\d+$|^\(.+\)$/.match?(code[:id])
+          conditional_date(t, yr_match || match, false)
+        end
+      end
+
+      # TODO: alternative where only title is available
+      def refitemcode(item, node)
+        m = NON_ISO_REF.match(item) and return refitem1code(item, m).compact
+        @log.add("AsciiDoc Input", node, "#{MALFORMED_REF}: #{item}")
+        {}
+      end
+
+      def refitem1code(_item, match)
+        code = analyse_ref_code(match[:code])
+        ((code[:id] && code[:numeric]) || code[:nofetch]) and
+          return { code: nil, match: match, analyse_code: code,
+                   hidden: code[:hidden] }
+        year = refitem1yr(code[:id])
+        { code: code[:id], analyse_code: code,
+          year: year,
+          title: match[:text], match: match, hidden: code[:hidden],
+          usrlbl: match[:usrlbl], lang: (@lang || :all) }
+      end
+
+      def refitem1yr(code)
+        yr_match = /[:-](?<year>(19|20)[0-9][0-9])\b/.match(code)
+        yr_match ? yr_match[:year] : nil
+      end
+
+      def refitemout(item, xml)
+        return nil if item[:ref][:match].nil?
+
+        item[:doc] or return refitem_render(xml, item[:ref][:match],
+                                            item[:ref][:analyse_code])
+        use_retrieved_relaton(item, xml)
+      end
+
+      ISO_REF =
+        %r{^<ref\sid="(?<anchor>[^"]+)">
+      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+|IEV)
+      (:(?<year>[0-9][0-9-]+))?\]</ref>,?\s*(?<text>.*)$}xm.freeze
+
+      ISO_REF_NO_YEAR =
+        %r{^<ref\sid="(?<anchor>[^"]+)">
+      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9-]+):
+      (--|&\#821[12];)\]</ref>,?\s*
+        (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>)?,?\s?(?<text>.*)$}xm
+          .freeze
+
+      ISO_REF_ALL_PARTS =
+        %r{^<ref\sid="(?<anchor>[^"]+)">
+      \[(?<usrlbl>\([^)]+\))?(?<code>(ISO|IEC)[^0-9]*\s[0-9]+)
+      (:(?<year>--|&\#821[12];|[0-9][0-9-]+))?\s
+      \(all\sparts\)\]</ref>,?\s*
+        (<fn[^>]*>\s*<p>(?<fn>[^\]]+)</p>\s*</fn>,?\s?)?(?<text>.*)$}xm.freeze
+
+      NON_ISO_REF = %r{^<ref\sid="(?<anchor>[^"]+)">
+      \[(?<usrlbl>\([^)]+\))?(?<code>[^\]]+?)\]</ref>,?\s*(?<text>.*)$}xm
+        .freeze
+
+      def reference1_matches(item)
+        matched = ISO_REF.match item
+        matched2 = ISO_REF_NO_YEAR.match item
+        matched3 = ISO_REF_ALL_PARTS.match item
+        [matched, matched2, matched3]
+      end
+
+      def reference1code(item, node)
+        matched, matched2, matched3 = reference1_matches(item)
+        if matched3.nil? && matched2.nil? && matched.nil?
+          refitemcode(item, node).merge(process: 0)
+        elsif !matched.nil? then isorefmatchescode(matched).merge(process: 1)
+        elsif !matched2.nil? then isorefmatches2code(matched2).merge(process: 2)
+        elsif !matched3.nil? then isorefmatches3code(matched3).merge(process: 3)
+        end
+      end
+
+      def reference1out(item, xml)
+        case item[:ref][:process]
+        when 0 then refitemout(item, xml)
+        when 1 then isorefmatchesout(item, xml)
+        when 2 then isorefmatches2out(item, xml)
+        when 3 then isorefmatches3out(item, xml)
+        end
+      end
+
+      def reference_preproc(node)
+        refs = node.items.each_with_object([]) do |b, m|
+          m << reference1code(b.text, node)
+        end
+        results = refs.each_with_index.with_object(Queue.new) do |(ref, i), res|
+          fetch_ref_async(ref.merge(ord: i), i, res)
+        end
+        [refs, results]
+      end
+
+      def reference(node)
+        refs, results = reference_preproc(node)
+        ret = reference_queue(refs, results)
+        noko do |xml|
+          ret.each { |b| reference1out(b, xml) }
+        end.join
+      end
+
+      def reference_queue(refs, results)
+        refs.each.with_object([]) do |_, m|
+          ref, i, doc = results.pop
+          m[i.to_i] = { ref: ref }
+          if doc.is_a?(RelatonBib::RequestError)
+            @log.add("Bibliography", nil, "Could not retrieve #{ref[:code]}: "\
+                                          "no access to online site")
+          else m[i.to_i][:doc] = doc
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metanorma/standoc/ref_sect.rb

@@ -0,0 +1,153 @@
+module Metanorma
+  module Standoc
+    module Section
+      def in_biblio?
+        @biblio
+      end
+
+      def in_norm_ref?
+        @norm_ref
+      end
+
+      def bibliography_parse(attrs, xml, node)
+        x = biblio_prep(attrs, xml, node) and return x
+        @biblio = true
+        attrs = attrs.merge(normative: node.attr("normative") || false)
+        xml.references **attr_code(attrs) do |xml_section|
+          xml_section.title { |t| t << node.title }
+          xml_section << node.content
+        end
+        @biblio = false
+      end
+
+      def bibitem_parse(attrs, xml, node)
+        norm_ref = @norm_ref
+        biblio = @biblio
+        @biblio = false
+        @norm_ref = false
+        ret = clause_parse(attrs, xml, node)
+        @biblio = biblio
+        @norm_ref = norm_ref
+        ret
+      end
+
+      def norm_ref_parse(attrs, xml, node)
+        x = biblio_prep(attrs, xml, node) and return x
+        @norm_ref = true
+        attrs = attrs.merge(normative: node.attr("normative") || true)
+        xml.references **attr_code(attrs) do |xml_section|
+          xml_section.title { |t| t << node.title }
+          xml_section << node.content
+        end
+        @norm_ref = false
+      end
+
+      def biblio_prep(attrs, xml, node)
+        if node.option? "bibitem"
+          bibitem_parse(attrs, xml, node)
+        else
+          node.attr("style") == "bibliography" or
+            @log.add("AsciiDoc Input", node,
+                     "Section not marked up as [bibliography]!")
+          nil
+        end
+      end
+
+      def global_ievcache_name
+        "#{Dir.home}/.iev/cache"
+      end
+
+      def local_ievcache_name(cachename)
+        return nil if cachename.nil?
+
+        cachename += "_iev" unless cachename.empty?
+        cachename = "iev" if cachename.empty?
+        "#{cachename}/cache"
+      end
+
+      def fetch_ref(xml, code, year, **opts)
+        return nil if opts[:no_year]
+
+        code = code.sub(/^\([^)]+\)/, "")
+        hit = @bibdb&.fetch(code, year, opts)
+        return nil if hit.nil?
+
+        xml.parent.add_child(smart_render_xml(hit, code, opts))
+        xml
+      rescue RelatonBib::RequestError
+        @log.add("Bibliography", nil, "Could not retrieve #{code}: "\
+                                      "no access to online site")
+        nil
+      end
+
+      def fetch_ref_async(ref, idx, res)
+        if ref[:code].nil? || ref[:no_year] || @bibdb.nil?
+          res << [ref, idx, nil]
+        else
+          @bibdb.fetch_async(ref[:code], ref[:year], ref) do |doc|
+            res << [ref, idx, doc]
+          end
+        end
+      end
+
+      def emend_biblio(xml, code, title, usrlbl)
+        unless xml.at("/bibitem/docidentifier[not(@type = 'DOI')][text()]")
+          @log.add("Bibliography", nil,
+                   "ERROR: No document identifier retrieved for #{code}")
+          xml.root << "<docidentifier>#{code}</docidentifier>"
+        end
+        unless xml.at("/bibitem/title[text()]")
+          @log.add("Bibliography", nil,
+                   "ERROR: No title retrieved for #{code}")
+          xml.root << "<title>#{title || '(MISSING TITLE)'}</title>"
+        end
+        usrlbl and xml.at("/bibitem/docidentifier").next =
+                     "<docidentifier type='metanorma'>#{mn_code(usrlbl)}</docidentifier>"
+      end
+
+      def smart_render_xml(xml, code, opts)
+        xml.respond_to? :to_xml or return nil
+        xml = Nokogiri::XML(xml.to_xml(lang: opts[:lang]))
+        emend_biblio(xml, code, opts[:title], opts[:usrlbl])
+        xml.xpath("//date").each { |d| Metanorma::Utils::endash_date(d) }
+        xml.traverse do |n|
+          n.text? and n.replace(Metanorma::Utils::smartformat(n.text))
+        end
+        xml.to_xml.sub(/<\?[^>]+>/, "")
+      end
+
+      def use_retrieved_relaton(item, xml)
+        xml.parent.add_child(smart_render_xml(item[:doc], item[:ref][:code],
+                                              item[:ref]))
+        use_my_anchor(xml, item[:ref][:match][:anchor], item.dig(:ref, :analyse_code, :hidden))
+      end
+
+      def init_bib_caches(node)
+        return if @no_isobib
+
+        global = !@no_isobib_cache && !node.attr("local-cache-only")
+        local = node.attr("local-cache") || node.attr("local-cache-only")
+        local = nil if @no_isobib_cache
+        @bibdb = Relaton::DbCache.init_bib_caches(
+          local_cache: local,
+          flush_caches: node.attr("flush-caches"),
+          global_cache: global,
+        )
+      end
+
+      def init_iev_caches(node)
+        unless @no_isobib_cache || @no_isobib
+          node.attr("local-cache-only") or
+            @iev_globalname = global_ievcache_name
+          @iev_localname = local_ievcache_name(node.attr("local-cache") ||
+                                               node.attr("local-cache-only"))
+          if node.attr("flush-caches")
+            FileUtils.rm_f @iev_globalname unless @iev_globalname.nil?
+            FileUtils.rm_f @iev_localname unless @iev_localname.nil?
+          end
+        end
+        # @iev = Iev::Db.new(globalname, localname) unless @no_isobib
+      end
+    end
+  end
+end

data/lib/{asciidoctor/standoc/ref_date_id.rb → metanorma/standoc/ref_utility.rb}

@@ -1,4 +1,4 @@
-module Asciidoctor
+module Metanorma
   module Standoc
     module Refs
       def set_date_range(date, text)
@@ -31,8 +31,9 @@ module Asciidoctor
         end
       end
 
-      def use_my_anchor(ref, id)
+      def use_my_anchor(ref, id, hidden)
         ref.parent.elements.last["id"] = id
+        hidden and ref.parent.elements.last["hidden"] = hidden
         ref
       end
 
@@ -55,7 +56,9 @@ module Asciidoctor
       end
 
       def mn_code(code)
-        code.sub(/^\(/, "[").sub(/\).*$/, "]").sub(/^nofetch\((.+)\)$/, "\\1")
+        code.sub(/^\(/, "[").sub(/\).*$/, "]")
+          .sub(/^hidden\((.+)\)$/, "\\1")
+          .sub(/^nofetch\((.+)\)$/, "\\1")
       end
 
       def analyse_ref_nofetch(ret)
@@ -64,6 +67,12 @@ module Asciidoctor
         ret.merge(id: m[:id], nofetch: true)
       end
 
+      def analyse_ref_hidden(ret)
+        return ret unless m = /^hidden\((?<id>.+)\)$/.match(ret[:id])
+
+        ret.merge(id: m[:id], hidden: true)
+      end
+
       def analyse_ref_repo_path(ret)
         return ret unless m =
                             /^(?<type>repo|path):\((?<key>[^,]+),?(?<id>.*)\)$/.match(ret[:id])
@@ -79,12 +88,41 @@ module Asciidoctor
       end
 
       # ref id = (usrlbl)code[:-]year
-      # code = nofetch(code) | (repo|path):(key,code) | \[? number \]? | ident
+      # code = nofetch(code) | hidden(code) | (repo|path):(key,code) |
+      # \[? number \]? | ident
       def analyse_ref_code(code)
         ret = { id: code }
         return ret if code.blank?
 
-        analyse_ref_nofetch(analyse_ref_repo_path(analyse_ref_numeric(ret)))
+        analyse_ref_nofetch(
+          analyse_ref_hidden(analyse_ref_repo_path(analyse_ref_numeric(ret))),
+        )
+      end
+
+      # if no year is supplied, interpret as no_year reference
+      def no_year_generic_ref(code)
+        /^(BSI|BS)\b/.match?(code)
+      end
+
+      def plaintxt
+        { format: "text/plain" }
+      end
+
+      def ref_attributes(match)
+        { id: match[:anchor], type: "standard" }
+      end
+
+      MALFORMED_REF =
+        "no anchor on reference, markup may be malformed: see "\
+        "https://www.metanorma.com/author/topics/document-format/bibliography/ , "\
+        "https://www.metanorma.com/author/iso/topics/markup/#bibliographies".freeze
+
+      def ref_normalise(ref)
+        ref.gsub(/&amp;amp;/, "&amp;").gsub(%r{^<em>(.*)</em>}, "\\1")
+      end
+
+      def ref_normalise_no_format(ref)
+        ref.gsub(/&amp;amp;/, "&amp;")
       end
     end
   end