kramdown-rfc2629 1.4.5 → 1.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 762c0dd40266895bcdc3b41b825a3adf0a5eefd93c4104177ed2ff0f48b3ac53
-  data.tar.gz: 182b9ddc22db2d012c82ba6be12b48d7e033748e9417cb8014249a6759db5aa5
+  metadata.gz: e94ef372b40394892134b787b7deb9a1c69060622452f6d7eb41d0165346de14
+  data.tar.gz: 55e0579bc857461d2ca7502957853264e4b176f5760b79e0dacdd3245a9f132c
 SHA512:
-  metadata.gz: eca4702ba83030ff120267f67116e938452810b69461e186bc4e950b245f7ce02bb88fe4cf09c1ceb019af10fcb5740e4e966192959a15adb859ba614449a4be
-  data.tar.gz: d11ea0b4af875edec76612cda7c854443099d5dc42109f16e3aa258bb5b7874f4b52fb351ffd2141cc593cb0809088a3a8abf0430d5ab35522b8c75c77af16ab
+  metadata.gz: 53a957d89977260a054963342c8ef5d2aa9e270689fc6f78e1abed296489a18cb12a66c1b8ddfad6404599180c3cc61cd271e83a969064e272d39ec3b1a5373d
+  data.tar.gz: 6f2af11238cb255f3f1506fa6bf31264dc8224972a68c41a33a99e0722e3e91db5f7296d37d460dddf301d90a8ec400ba14d9b8d04938415832c93089cdef8c5

data/bin/kramdown-rfc-cache-subseries-bibxml

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+# prerequisite:
+# gem install net-http-persistent
+#
+# generates referencegroup files for BCP and STD series (like bibxml9)
+# unfortunately, needs to re-fetch rfc-index.xml
+# uses pretty slow built-in XML parser, so it will take a while even on regen
+#
+# uses ENV["KRAMDOWN_REFCACHEDIR"] for where you want to have your bibxml9 data
+#
+
+require 'rexml/document'
+require 'fileutils'
+
+begin
+  require 'net/http/persistent'
+rescue
+  warn "*** please install net-http-persistent:"
+  warn "   gem install net-http-persistent"
+  warn "(prefix by sudo only if required)."
+  exit 72                       # EX_OSFILE
+end
+
+
+TARGET_DIR = ENV["KRAMDOWN_REFCACHEDIR"] || (
+  path = File.expand_path("~/.cache/xml2rfc")
+  warn "*** set environment variable KRAMDOWN_REFCACHEDIR to #{path} to actually use the cache"
+  path
+)
+
+FileUtils.mkdir_p(TARGET_DIR)
+FileUtils.chdir(TARGET_DIR)
+
+$http = Net::HTTP::Persistent.new name: 'subseries'
+
+KRAMDOWN_PERSISTENT_VERBOSE = true
+
+      def get_and_write_resource_persistently(url, fn, age_verbose=false)
+        t1 = Time.now
+        response = $http.request(URI(url))
+        if response.code != "200"
+          raise "*** Status code #{response.code} while fetching #{url}"
+        else
+          File.write(fn, response.body)
+        end
+        t2 = Time.now
+        warn "#{url} -> #{fn} (#{"%.3f" % (t2 - t1)} s)" if KRAMDOWN_PERSISTENT_VERBOSE
+        if age_verbose
+          if age = response.get_fields("age")
+            warn "(working from a web cache, index is #{age.first} seconds stale)"
+          end
+        end
+      end
+
+CLEAR_RET = "\e[K\r" # XXX all the world is ECMA-48 (ISO 6429), no?
+
+      def noisy(name)
+        print "#{name}...#{CLEAR_RET}"
+      end
+      def clear_noise
+        print CLEAR_RET
+      end
+
+def normalize_name(n)
+  n.sub(/([A-Z])0+/) {$1}
+end
+
+def regress_name(n)
+  n.sub(/([A-Z])(\d+)/) {"#$1#{"%04d" % $2.to_i}"}
+end
+
+def regress_name_dot(n)
+  n.sub(/([A-Z])(\d+)/) {"#$1.#{"%04d" % $2.to_i}"}
+end
+
+def get_ref(rfcname)
+  name = "reference.#{regress_name_dot(rfcname)}.xml"
+  begin
+    file = File.read(name)      # no age check
+  rescue Errno::ENOENT
+    get_and_write_resource_persistently("https://www.rfc-editor.org/refs/bibxml/" << name, name)
+    file = File.read(name)
+  end
+  d = REXML::Document.new(file)
+  d.xml_decl.nowrite
+  "<!-- #{name} -->\n" << d.to_s.lstrip
+end
+
+def create_bib(series, subname, rfcnames)
+  p [series, subname, rfcnames]
+  subname_norm = normalize_name(subname)
+  refs = %{<?xml version='1.0' encoding='UTF-8'?>\n}
+  refs << %{<referencegroup anchor='#{subname_norm}' target='https://www.rfc-editor.org/info/#{subname_norm.downcase}'>\n}
+  refs << rfcnames.map {|x| get_ref(x)}.join
+  refs << "</referencegroup>\n"
+  File.write("reference.#{regress_name_dot(subname)}.xml", refs)
+end
+
+def handle_sub(series, entry)
+  ids = entry.get_elements("doc-id")
+  warn "** ids #{ids} #{entry}" unless ids.size == 1
+  subname = ids.first.text
+  isalso = entry.get_elements("is-also")
+  if isalso.size == 1
+    rfcs = isalso.first.get_elements("doc-id")
+    if rfcs.size > 0
+      rfcnames = rfcs.map {|r| r.text}
+      create_bib(series, subname, rfcnames)
+    end
+  end
+end
+
+RFCINDEX_SOURCE = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+RFCINDEX_COPY = File.basename(RFCINDEX_SOURCE)
+
+get_and_write_resource_persistently(RFCINDEX_SOURCE, RFCINDEX_COPY, true) unless ENV["KRAMDOWN_DONT_REFRESH_RFCINDEX"]
+
+doc = REXML::Document.new(File.read(RFCINDEX_COPY))
+REXML::XPath.each(doc.root, "/rfc-index/bcp-entry") { |e| handle_sub("BCP", e) }
+REXML::XPath.each(doc.root, "/rfc-index/std-entry") { |e| handle_sub("STD", e) }

data/bin/kramdown-rfc2629

@@ -14,7 +14,7 @@ Encoding.default_external = "UTF-8" # wake up, smell the coffee
 
 def boilerplate(key)
   case key.downcase
-  when /\Abcp14(\+)?\z/i
+  when /\Abcp14(\+)?(-tagged)?\z/i
     ret = <<RFC8174
 The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL
 NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED",
@@ -28,6 +28,26 @@ These words may also appear in this document in
 lower case as plain English words, absent their normative meanings.
 PLUS
     end
+    if $2
+      if $options.v3
+        ret << <<TAGGED
+
+*[MUST]: <bcp14>
+*[MUST NOT]: <bcp14>
+*[REQUIRED]: <bcp14>
+*[SHALL]: <bcp14>
+*[SHALL NOT]: <bcp14>
+*[SHOULD]: <bcp14>
+*[SHOULD NOT]: <bcp14>
+*[RECOMMENDED]: <bcp14>
+*[NOT RECOMMENDED]: <bcp14>
+*[MAY]: <bcp14>
+*[OPTIONAL]: <bcp14>
+TAGGED
+      else
+        warn "** need --v3 to tag bcp14"
+      end
+    end
     ret
   else
     warn "** Unknwon boilerplate key: #{key}"
@@ -69,6 +89,16 @@ def yaml_load(input, *args)
  end
 end
 
+XREF_SECTIONS_RE = ::Kramdown::Parser::RFC2629Kramdown::SECTIONS_RE
+XSR_PREFIX = "#{XREF_SECTIONS_RE} of "
+XSR_SUFFIX = ", (#{XREF_SECTIONS_RE})| \\((#{XREF_SECTIONS_RE})\\)"
+XREF_TXT = ::Kramdown::Parser::RFC2629Kramdown::XREF_TXT
+XREF_TXT_SUFFIX = " \\(#{XREF_TXT}\\)"
+
+def spacify_re(s)
+  s.gsub(' ', '[\u00A0\s]+')
+end
+
 def xml_from_sections(input)
 
   unless ENV["KRAMDOWN_NO_SOURCE"]
@@ -138,11 +168,20 @@ def xml_from_sections(input)
   # collect normative/informative tagging {{!RFC2119}} {{?RFC4711}}
   sechash.each do |k, v|
     next if k == "fluff"
-    v.gsub!(/{{(?:([?!])(-)?|(-))([\w._\-]+)(?:=([\w.\/_\-]+))?}}/) do |match|
-      norminform = $1
-      replacing = $2 || $3
-      word = $4
-      bibref = $5
+    v.gsub!(/{{(#{
+      spacify_re(XSR_PREFIX)
+    })?(?:([?!])(-)?|(-))([\w._\-]+)(?:=([\w.\/_\-]+))?(#{
+      XREF_TXT_SUFFIX
+    })?(#{
+      spacify_re(XSR_SUFFIX)
+    })?}}/) do |match|
+      xsr_prefix = $1
+      norminform = $2
+      replacing = $3 || $4
+      word = $5
+      bibref = $6
+      xrt_suffix = $7
+      xsr_suffix = $8
       if replacing
         if new = ref_replacements[word]
           word = new
@@ -166,7 +205,7 @@ def xml_from_sections(input)
       if norminform
         norm_ref[word] ||= norminform == '!' # one normative ref is enough
       end
-      "{{#{word}}}"
+      "{{#{xsr_prefix}#{word}#{xrt_suffix}#{xsr_suffix}}}"
     end
   end
 
@@ -249,6 +288,10 @@ def bibtagsys(bib, anchor=nil, stand_alone=true)
     rfc4d = "%04d" % $1.to_i
     [bib.upcase,
      "#{XML_RESOURCE_ORG_PREFIX}/bibxml/reference.RFC.#{rfc4d}.xml"]
+  elsif $options.v3 && bib =~ /\A(bcp|std)(\d+)/i
+    n4d = "%04d" % $2.to_i
+    [bib.upcase,
+     "#{XML_RESOURCE_ORG_PREFIX}/bibxml-rfcsubseries-new/reference.#{$1.upcase}.#{n4d}.xml"]
   elsif bib =~ /\A([-A-Z0-9]+)\./ &&
         (xro = Kramdown::Converter::Rfc2629::XML_RESOURCE_ORG_MAP[$1])
     dir, _ttl, rewrite_anchor = xro

data/data/encoding-fallbacks.txt

@@ -1,3 +1,4 @@
+00a0  
 00a1 !
 00a2 [cents]
 00a3 GBP

data/data/kramdown-rfc2629.erb

@@ -100,7 +100,7 @@
         <name>Contributors</name>
 <%   else -%>
     <section anchor="contributors" numbered="false" title="Contributors">
-<%     warn "*** Cannot process YAML contributors under V2 rules" if consec -%>
+<%     warn "*** To use YAML contributors, use --v3 (kdrfc -3)" if consec -%>
 <%   end -%>
 <%= sh -%>
 <% if $options.v3 && consec

data/kramdown-rfc2629.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'kramdown-rfc2629'
-  s.version = '1.4.5'
+  s.version = '1.4.10'
   s.summary = "Kramdown extension for generating RFC 7749 XML."
   s.description = %{An RFC7749 (XML2RFC) generating backend for Thomas Leitner's
 "kramdown" markdown parser.  Mostly useful for RFC writers.}
@@ -10,7 +10,8 @@ spec = Gem::Specification.new do |s|
   s.files = Dir['lib/**/*.rb'] + %w(README.md LICENSE kramdown-rfc2629.gemspec bin/kdrfc bin/kramdown-rfc2629 bin/doilit bin/kramdown-rfc-extract-markdown data/kramdown-rfc2629.erb data/encoding-fallbacks.txt data/math.json)
   s.require_path = 'lib'
   s.executables = ['kramdown-rfc2629', 'doilit', 'kramdown-rfc-extract-markdown',
-                   'kdrfc', 'kramdown-rfc-cache-i-d-bibxml']
+                   'kdrfc', 'kramdown-rfc-cache-i-d-bibxml',
+                   'kramdown-rfc-cache-subseries-bibxml']
   s.required_ruby_version = '>= 2.3.0'
   # s.requirements = 'wget'
   #  s.has_rdoc = true

data/lib/kramdown-rfc/parameterset.rb

@@ -34,13 +34,12 @@ module KramdownRFC
       val ||= defcontent
       Array(val).map do |val1|
         v = val1.to_s.strip
-        if markdown             # Uuh.  Heavy coupling.
-          doc = Kramdown::Document.new(v, $global_markdown_options)
-          $stderr.puts doc.warnings.to_yaml unless doc.warnings.empty?
-          contents = doc.to_rfc2629[3..-6] # skip <t>...</t>\n
-        else
-          contents = escape_html(v)
-        end
+        contents =
+          if markdown
+            ::Kramdown::Converter::Rfc2629::process_markdown(v)
+          else
+            escape_html(v)
+          end
         %{<#{[an, *Array(attr).map(&:to_s)].join(" ").strip}>#{contents}</#{an}>}
       end.join(" ")
     end

data/lib/kramdown-rfc2629.rb

@@ -42,32 +42,50 @@ module Kramdown
         @block_parsers.unshift(:block_pi)
       end
 
-      SECTIONS_RE = /Section(?:s (?:[\w.]+, )*[\w.]+,? and)? [\w.]+/
-
-      def handle_bares(s, attr, format, href)
-        sa = s.sub(/\A\S+\s/, '').split(/,? and /)
-        sa[0..0] = *sa[0].split(', ')
-        sz = sa.size
-        if sz != 1         # we have to redo xml2rfc's work here
-          @tree.children << Element.new(:text, "Sections ", {}) # XXX needs to split into Section/Appendix
-          sa.each_with_index do |sec, i|
-            attr1 = {"target" => href, "section" => sec, "sectionFormat" => "bare"}
-            @tree.children << Element.new(:xref, nil, attr1)
-            text = if i == 0 && sz == 2
-                     " and "
-                   elsif i == sz-1
-                     " of "
-                   elsif i == sz-2
-                     ", and "
-                   else
-                     ", "
-                   end
-            @tree.children << Element.new(:text, text, {})
+      XREF_BASE = /[\w.-]+/ # a token for a reference
+      XREF_TXT = /(?:[^\(]|\([^\)]*\))+/ # parenthesized text
+      XREF_RE = /#{XREF_BASE}(?: \(#{XREF_TXT}\))?/
+      XREF_RE_M = /\A(#{XREF_BASE})(?: \((#{XREF_TXT})\))?/ # matching version of XREF_RE
+      XREF_SINGLE = /(?:Section|Appendix) #{XREF_RE}/
+      XREF_MULTI = /(?:Sections|Appendices) (?:#{XREF_RE}, )*#{XREF_RE},? and #{XREF_RE}/
+      XREF_ANY = /(?:#{XREF_SINGLE}|#{XREF_MULTI})/
+      SECTIONS_RE = /(?:#{XREF_ANY} and )?#{XREF_ANY}/
+
+      def handle_bares(s, attr, format, href, last_join = nil)
+        if s.match(/\A(#{XREF_ANY}) and (#{XREF_ANY})\z/)
+          handle_bares($1, {}, nil, href, " and ")
+          handle_bares($2, {}, nil, href, " of ")
+          return
+        end
+
+        href = href.split(' ')[0] # Remove any trailing (...)
+        multi = last_join != nil
+        (sn, s) = s.split(' ', 2)
+        loop do
+          m = s.match(/\A#{XREF_RE_M}(, (?:and )?| and )?/)
+          break if not m
+
+          if not multi and not m[2] and not m[3]
+            # Modify |attr| if there is a single reference.  This can only be
+            # used if there is only one section reference and the section part
+            # has no title.
+            attr['section'] = m[1]
+            attr['sectionFormat'] = format
+            attr['text'] = m[2]
+            return
           end
-          # attr stays unchanged, no section added
-        else
-          attr['section'] = sa[-1]
-          attr['sectionFormat'] = format
+
+          if sn
+            @tree.children << Element.new(:text, "#{sn} ", {})
+            sn = nil
+          end
+
+          multi = true
+          s[m[0]] = ''
+
+          attr1 = { 'target' => href, 'section' => m[1], 'sectionFormat' => 'bare', 'text' => m[2] }
+          @tree.children << Element.new(:xref, nil, attr1)
+          @tree.children << Element.new(:text, m[3] || last_join || " of ", {})
         end
       end
 
@@ -102,12 +120,22 @@ module Kramdown
             when /\A(.*) \((#{SECTIONS_RE})\)\z/
               href = $1
               handle_bares($2, attr, "parens", href)
-            when /\A([\w.]+)<(.*)\z/
+            when /\A([\w.]+)<(.+)\z/
               href = $2
               attr['section'] = $1
               attr['sectionFormat'] = 'bare'
+            when /\A<<(.+)\z/
+              href = $1
+              attr['format'] = 'title'
+            when /\A<(.+)\z/
+              href = $1
+              attr['format'] = 'counter'
             end
           end
+          if href.match(XREF_RE_M)
+            href = $1
+            attr['text'] = $2
+          end
           href = href.gsub(/\A[0-9]/) { "_#{$&}" } # can't start an IDREF with a number
           attr['target'] = href
           el = Element.new(:xref, nil, attr)
@@ -282,6 +310,12 @@ module Kramdown
         generate_id(value).gsub(/-+/, '-')
       end
 
+      def self.process_markdown(v)             # Uuh.  Heavy coupling.
+        doc = ::Kramdown::Document.new(v, $global_markdown_options)
+        $stderr.puts doc.warnings.to_yaml unless doc.warnings.empty?
+        doc.to_rfc2629[3..-6] # skip <t>...</t>\n
+      end
+
       SVG_COLORS = Hash.new {|h, k| k}
       <<COLORS.each_line {|l| k, v = l.chomp.split; SVG_COLORS[k] = v}
 black	#000000
@@ -723,6 +757,7 @@ COLORS
 
       def convert_xref(el, indent, opts)
         gi = el.attr.delete('gi')
+        text = el.attr.delete('text')
         target = el.attr['target']
         if target[0] == "&"
           "#{target};"
@@ -732,7 +767,12 @@ COLORS
           else
             gi ||= "xref"
           end
-          "<#{gi}#{el_html_attributes(el)}/>"
+          if text
+            tail = ">#{Rfc2629::process_markdown(text)}</#{gi}>"
+          else
+            tail = "/>"
+          end
+          "<#{gi}#{el_html_attributes(el)}#{tail}"
         end
       end
 
@@ -827,13 +867,27 @@ COLORS
         end
       end
 
+      def self.bcp_std_ref(t, n)
+        warn "*** #{t} anchors not supported in v2 format" unless $options.v3
+        [name = "reference.#{t}.#{"%04d" % n.to_i}.xml",
+         "#{XML_RESOURCE_ORG_PREFIX}/bibxml-rfcsubseries-new/#{name}"] # FOR NOW
+      end
+
       # [subdirectory name, cache ttl in seconds, does it provide for ?anchor=]
       XML_RESOURCE_ORG_MAP = {
         "RFC" => ["bibxml", 86400*7, false,
-                  ->(fn, n){ "https://www.rfc-editor.org/refs/bibxml/#{fn}"}
+                  ->(fn, n){ [name = "reference.RFC.#{"%04d" % n.to_i}.xml",
+                              "https://www.rfc-editor.org/refs/bibxml/#{name}"] }
                  ],
         "I-D" => ["bibxml3", false, false,
-                  ->(fn, n){ "https://datatracker.ietf.org/doc/bibxml3/draft-#{n.sub(/\Adraft-/, '')}/xml" }
+                  ->(fn, n){ [fn,
+                              "https://datatracker.ietf.org/doc/bibxml3/draft-#{n.sub(/\Adraft-/, '')}/xml"] }
+                 ],
+        "BCP" => ["bibxml-rfcsubseries", 86400*7, false,
+                  ->(fn, n){ Rfc2629::bcp_std_ref("BCP", n) }
+                 ],
+        "STD" => ["bibxml-rfcsubseries", 86400*7, false,
+                  ->(fn, n){ Rfc2629::bcp_std_ref("STD", n) }
                  ],
         "W3C" => "bibxml4",
         "3GPP" => "bibxml5",
@@ -884,7 +938,7 @@ COLORS
             ttl ||= KRAMDOWN_REFCACHETTL  # everything but RFCs might change a lot
             puts "*** Huh: #{fn}" unless sub
             if altproc && !KRAMDOWN_USE_TOOLS_SERVER
-              url = altproc.call(fn, n)
+              fn, url = altproc.call(fn, n)
             else
               url = "#{XML_RESOURCE_ORG_PREFIX}/#{sub}/#{fn}"
             end
@@ -907,7 +961,7 @@ COLORS
                   }
                 else
                   REXML::XPath.each(d.root, "/reference/format") { |x|
-                    x.attributes["target"].sub!(%r{https?://www.ietf.org/internet-drafts/}, 
+                    x.attributes["target"].sub!(%r{https?://www.ietf.org/internet-drafts/},
                                                 %{https://www.ietf.org/archive/id/}) if t == "I-D"
                   }
                 end
@@ -1073,6 +1127,9 @@ COLORS
         title = nil if title.empty?
         value = el.value
         if item = title
+          if item == "<bcp14>" && $options.v3
+            return "<bcp14>#{el.value}</bcp14>"
+          end
           m = title.scan(Parser::RFC2629Kramdown::IREF_START)
           if m.empty?
             subitem = value

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kramdown-rfc2629
 version: !ruby/object:Gem::Version
-  version: 1.4.5
+  version: 1.4.10
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-29 00:00:00.000000000 Z
+date: 2021-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kramdown
@@ -62,6 +62,7 @@ executables:
 - kramdown-rfc-extract-markdown
 - kdrfc
 - kramdown-rfc-cache-i-d-bibxml
+- kramdown-rfc-cache-subseries-bibxml
 extensions: []
 extra_rdoc_files: []
 files:
@@ -70,6 +71,7 @@ files:
 - bin/doilit
 - bin/kdrfc
 - bin/kramdown-rfc-cache-i-d-bibxml
+- bin/kramdown-rfc-cache-subseries-bibxml
 - bin/kramdown-rfc-extract-markdown
 - bin/kramdown-rfc2629
 - data/encoding-fallbacks.txt