kramdown-rfc2629 1.4.6 → 1.4.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b0564f81e1af0b8fd748342c07d417a7d24a5eee8ec5a6cd64f79234ebc192a
-  data.tar.gz: 2f9e7421b5355b608bc5a41174317bccd4b55e95f1c2089228517003572f19e5
+  metadata.gz: 550eb712315978bfe6985b74bce8ed0c2c8d69c94ecd0e78e3950d8822644aed
+  data.tar.gz: 5b51583ad31b2c00d9e490e983b6465a51fb84a334fff1ea42e24212f0d24968
 SHA512:
-  metadata.gz: e0aa2949a7d5d28eacdceff68e37fae0338a7f92abab5ed21fcb99414042c528521d844d8e92d14202a0b914eec685149c69dcbdb53d3089a0f72723872d4b63
-  data.tar.gz: 9424d9efbe85b3ca435ff1691f052097b579ff02247af53a5b463b2eb3cf50bed3761ad4386d87a93df32e6b9a87a4924e1c47eaae0e01b2163473a9e65bf950
+  metadata.gz: 9033a5b37fa0fdc30bce1c4d02d987d96f895dfad5b3adb6419bd5bad4c5fbd0074edcaa0de955fc1857764d159df8f54b4a7bd4ef4720c3f9b0edc3645ebdd4
+  data.tar.gz: ee00920f4806f49f74d36343218453f8e886fd93f12949b37dbc92fabeab6947237065f997fa13218ded3b492a5d7a4c40901d9707f7973d28b8318f4dfcbcca

data/bin/kramdown-rfc-cache-subseries-bibxml

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+# prerequisite:
+# gem install net-http-persistent
+#
+# generates referencegroup files for BCP and STD series (like bibxml9)
+# unfortunately, needs to re-fetch rfc-index.xml
+# uses pretty slow built-in XML parser, so it will take a while even on regen
+#
+# uses ENV["KRAMDOWN_REFCACHEDIR"] for where you want to have your bibxml9 data
+#
+
+require 'rexml/document'
+require 'fileutils'
+
+begin
+  require 'net/http/persistent'
+rescue
+  warn "*** please install net-http-persistent:"
+  warn "   gem install net-http-persistent"
+  warn "(prefix by sudo only if required)."
+  exit 72                       # EX_OSFILE
+end
+
+
+TARGET_DIR = ENV["KRAMDOWN_REFCACHEDIR"] || (
+  path = File.expand_path("~/.cache/xml2rfc")
+  warn "*** set environment variable KRAMDOWN_REFCACHEDIR to #{path} to actually use the cache"
+  path
+)
+
+FileUtils.mkdir_p(TARGET_DIR)
+FileUtils.chdir(TARGET_DIR)
+
+$http = Net::HTTP::Persistent.new name: 'subseries'
+
+KRAMDOWN_PERSISTENT_VERBOSE = true
+
+      def get_and_write_resource_persistently(url, fn, age_verbose=false)
+        t1 = Time.now
+        response = $http.request(URI(url))
+        if response.code != "200"
+          raise "*** Status code #{response.code} while fetching #{url}"
+        else
+          File.write(fn, response.body)
+        end
+        t2 = Time.now
+        warn "#{url} -> #{fn} (#{"%.3f" % (t2 - t1)} s)" if KRAMDOWN_PERSISTENT_VERBOSE
+        if age_verbose
+          if age = response.get_fields("age")
+            warn "(working from a web cache, index is #{age.first} seconds stale)"
+          end
+        end
+      end
+
+CLEAR_RET = "\e[K\r" # XXX all the world is ECMA-48 (ISO 6429), no?
+
+      def noisy(name)
+        print "#{name}...#{CLEAR_RET}"
+      end
+      def clear_noise
+        print CLEAR_RET
+      end
+
+def normalize_name(n)
+  n.sub(/([A-Z])0+/) {$1}
+end
+
+def regress_name(n)
+  n.sub(/([A-Z])(\d+)/) {"#$1#{"%04d" % $2.to_i}"}
+end
+
+def regress_name_dot(n)
+  n.sub(/([A-Z])(\d+)/) {"#$1.#{"%04d" % $2.to_i}"}
+end
+
+def get_ref(rfcname)
+  name = "reference.#{regress_name_dot(rfcname)}.xml"
+  begin
+    file = File.read(name)      # no age check
+  rescue Errno::ENOENT
+    get_and_write_resource_persistently("https://www.rfc-editor.org/refs/bibxml/" << name, name)
+    file = File.read(name)
+  end
+  d = REXML::Document.new(file)
+  d.xml_decl.nowrite
+  "<!-- #{name} -->\n" << d.to_s.lstrip
+end
+
+def create_bib(series, subname, rfcnames)
+  p [series, subname, rfcnames]
+  subname_norm = normalize_name(subname)
+  refs = %{<?xml version='1.0' encoding='UTF-8'?>\n}
+  refs << %{<referencegroup anchor='#{subname_norm}' target='https://www.rfc-editor.org/info/#{subname_norm.downcase}'>\n}
+  refs << rfcnames.map {|x| get_ref(x)}.join
+  refs << "</referencegroup>\n"
+  File.write("reference.#{regress_name_dot(subname)}.xml", refs)
+end
+
+def handle_sub(series, entry)
+  ids = entry.get_elements("doc-id")
+  warn "** ids #{ids} #{entry}" unless ids.size == 1
+  subname = ids.first.text
+  isalso = entry.get_elements("is-also")
+  if isalso.size == 1
+    rfcs = isalso.first.get_elements("doc-id")
+    if rfcs.size > 0
+      rfcnames = rfcs.map {|r| r.text}
+      create_bib(series, subname, rfcnames)
+    end
+  end
+end
+
+RFCINDEX_SOURCE = "https://www.rfc-editor.org/rfc/rfc-index.xml"
+RFCINDEX_COPY = File.basename(RFCINDEX_SOURCE)
+
+get_and_write_resource_persistently(RFCINDEX_SOURCE, RFCINDEX_COPY, true) unless ENV["KRAMDOWN_DONT_REFRESH_RFCINDEX"]
+
+doc = REXML::Document.new(File.read(RFCINDEX_COPY))
+REXML::XPath.each(doc.root, "/rfc-index/bcp-entry") { |e| handle_sub("BCP", e) }
+REXML::XPath.each(doc.root, "/rfc-index/std-entry") { |e| handle_sub("STD", e) }

data/bin/kramdown-rfc2629

@@ -89,6 +89,16 @@ def yaml_load(input, *args)
  end
 end
 
+XREF_SECTIONS_RE = ::Kramdown::Parser::RFC2629Kramdown::SECTIONS_RE
+XSR_PREFIX = "#{XREF_SECTIONS_RE} of "
+XSR_SUFFIX = ", (#{XREF_SECTIONS_RE})| \\((#{XREF_SECTIONS_RE})\\)"
+XREF_TXT = ::Kramdown::Parser::RFC2629Kramdown::XREF_TXT
+XREF_TXT_SUFFIX = " \\(#{XREF_TXT}\\)"
+
+def spacify_re(s)
+  s.gsub(' ', '[\u00A0\s]+')
+end
+
 def xml_from_sections(input)
 
   unless ENV["KRAMDOWN_NO_SOURCE"]
@@ -158,11 +168,20 @@ def xml_from_sections(input)
   # collect normative/informative tagging {{!RFC2119}} {{?RFC4711}}
   sechash.each do |k, v|
     next if k == "fluff"
-    v.gsub!(/{{(?:([?!])(-)?|(-))([\w._\-]+)(?:=([\w.\/_\-]+))?}}/) do |match|
-      norminform = $1
-      replacing = $2 || $3
-      word = $4
-      bibref = $5
+    v.gsub!(/{{(#{
+      spacify_re(XSR_PREFIX)
+    })?(?:([?!])(-)?|(-))([\w._\-]+)(?:=([\w.\/_\-]+))?(#{
+      XREF_TXT_SUFFIX
+    })?(#{
+      spacify_re(XSR_SUFFIX)
+    })?}}/) do |match|
+      xsr_prefix = $1
+      norminform = $2
+      replacing = $3 || $4
+      word = $5
+      bibref = $6
+      xrt_suffix = $7
+      xsr_suffix = $8
       if replacing
         if new = ref_replacements[word]
           word = new
@@ -186,7 +205,7 @@ def xml_from_sections(input)
       if norminform
         norm_ref[word] ||= norminform == '!' # one normative ref is enough
       end
-      "{{#{word}}}"
+      "{{#{xsr_prefix}#{word}#{xrt_suffix}#{xsr_suffix}}}"
     end
   end
 
@@ -269,6 +288,10 @@ def bibtagsys(bib, anchor=nil, stand_alone=true)
     rfc4d = "%04d" % $1.to_i
     [bib.upcase,
      "#{XML_RESOURCE_ORG_PREFIX}/bibxml/reference.RFC.#{rfc4d}.xml"]
+  elsif $options.v3 && bib =~ /\A(bcp|std)(\d+)/i
+    n4d = "%04d" % $2.to_i
+    [bib.upcase,
+     "#{XML_RESOURCE_ORG_PREFIX}/bibxml-rfcsubseries-new/reference.#{$1.upcase}.#{n4d}.xml"]
   elsif bib =~ /\A([-A-Z0-9]+)\./ &&
         (xro = Kramdown::Converter::Rfc2629::XML_RESOURCE_ORG_MAP[$1])
     dir, _ttl, rewrite_anchor = xro

data/data/kramdown-rfc2629.erb

@@ -100,7 +100,7 @@
         <name>Contributors</name>
 <%   else -%>
     <section anchor="contributors" numbered="false" title="Contributors">
-<%     warn "*** Cannot process YAML contributors under V2 rules" if consec -%>
+<%     warn "*** To use YAML contributors, use --v3 (kdrfc -3)" if consec -%>
 <%   end -%>
 <%= sh -%>
 <% if $options.v3 && consec

data/kramdown-rfc2629.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'kramdown-rfc2629'
-  s.version = '1.4.6'
+  s.version = '1.4.11'
   s.summary = "Kramdown extension for generating RFC 7749 XML."
   s.description = %{An RFC7749 (XML2RFC) generating backend for Thomas Leitner's
 "kramdown" markdown parser.  Mostly useful for RFC writers.}
@@ -10,7 +10,8 @@ spec = Gem::Specification.new do |s|
   s.files = Dir['lib/**/*.rb'] + %w(README.md LICENSE kramdown-rfc2629.gemspec bin/kdrfc bin/kramdown-rfc2629 bin/doilit bin/kramdown-rfc-extract-markdown data/kramdown-rfc2629.erb data/encoding-fallbacks.txt data/math.json)
   s.require_path = 'lib'
   s.executables = ['kramdown-rfc2629', 'doilit', 'kramdown-rfc-extract-markdown',
-                   'kdrfc', 'kramdown-rfc-cache-i-d-bibxml']
+                   'kdrfc', 'kramdown-rfc-cache-i-d-bibxml',
+                   'kramdown-rfc-cache-subseries-bibxml']
   s.required_ruby_version = '>= 2.3.0'
   # s.requirements = 'wget'
   #  s.has_rdoc = true

data/lib/kramdown-rfc/parameterset.rb

@@ -34,13 +34,12 @@ module KramdownRFC
       val ||= defcontent
       Array(val).map do |val1|
         v = val1.to_s.strip
-        if markdown             # Uuh.  Heavy coupling.
-          doc = Kramdown::Document.new(v, $global_markdown_options)
-          $stderr.puts doc.warnings.to_yaml unless doc.warnings.empty?
-          contents = doc.to_rfc2629[3..-6] # skip <t>...</t>\n
-        else
-          contents = escape_html(v)
-        end
+        contents =
+          if markdown
+            ::Kramdown::Converter::Rfc2629::process_markdown(v)
+          else
+            escape_html(v)
+          end
         %{<#{[an, *Array(attr).map(&:to_s)].join(" ").strip}>#{contents}</#{an}>}
       end.join(" ")
     end

data/lib/kramdown-rfc2629.rb

@@ -42,32 +42,50 @@ module Kramdown
         @block_parsers.unshift(:block_pi)
       end
 
-      SECTIONS_RE = /Section(?:s (?:[\w.]+, )*[\w.]+,? and)? [\w.]+/
-
-      def handle_bares(s, attr, format, href)
-        sa = s.sub(/\A\S+\s/, '').split(/,? and /)
-        sa[0..0] = *sa[0].split(', ')
-        sz = sa.size
-        if sz != 1         # we have to redo xml2rfc's work here
-          @tree.children << Element.new(:text, "Sections ", {}) # XXX needs to split into Section/Appendix
-          sa.each_with_index do |sec, i|
-            attr1 = {"target" => href, "section" => sec, "sectionFormat" => "bare"}
-            @tree.children << Element.new(:xref, nil, attr1)
-            text = if i == 0 && sz == 2
-                     " and "
-                   elsif i == sz-1
-                     " of "
-                   elsif i == sz-2
-                     ", and "
-                   else
-                     ", "
-                   end
-            @tree.children << Element.new(:text, text, {})
+      XREF_BASE = /[\w.-]+/ # a token for a reference
+      XREF_TXT = /(?:[^\(]|\([^\)]*\))+/ # parenthesized text
+      XREF_RE = /#{XREF_BASE}(?: \(#{XREF_TXT}\))?/
+      XREF_RE_M = /\A(#{XREF_BASE})(?: \((#{XREF_TXT})\))?/ # matching version of XREF_RE
+      XREF_SINGLE = /(?:Section|Appendix) #{XREF_RE}/
+      XREF_MULTI = /(?:Sections|Appendices) (?:#{XREF_RE}, )*#{XREF_RE},? and #{XREF_RE}/
+      XREF_ANY = /(?:#{XREF_SINGLE}|#{XREF_MULTI})/
+      SECTIONS_RE = /(?:#{XREF_ANY} and )?#{XREF_ANY}/
+
+      def handle_bares(s, attr, format, href, last_join = nil)
+        if s.match(/\A(#{XREF_ANY}) and (#{XREF_ANY})\z/)
+          handle_bares($1, {}, nil, href, " and ")
+          handle_bares($2, {}, nil, href, " of ")
+          return
+        end
+
+        href = href.split(' ')[0] # Remove any trailing (...)
+        multi = last_join != nil
+        (sn, s) = s.split(' ', 2)
+        loop do
+          m = s.match(/\A#{XREF_RE_M}(, (?:and )?| and )?/)
+          break if not m
+
+          if not multi and not m[2] and not m[3]
+            # Modify |attr| if there is a single reference.  This can only be
+            # used if there is only one section reference and the section part
+            # has no title.
+            attr['section'] = m[1]
+            attr['sectionFormat'] = format
+            attr['text'] = m[2]
+            return
           end
-          # attr stays unchanged, no section added
-        else
-          attr['section'] = sa[-1]
-          attr['sectionFormat'] = format
+
+          if sn
+            @tree.children << Element.new(:text, "#{sn} ", {})
+            sn = nil
+          end
+
+          multi = true
+          s[m[0]] = ''
+
+          attr1 = { 'target' => href, 'section' => m[1], 'sectionFormat' => 'bare', 'text' => m[2] }
+          @tree.children << Element.new(:xref, nil, attr1)
+          @tree.children << Element.new(:text, m[3] || last_join || " of ", {})
         end
       end
 
@@ -114,6 +132,10 @@ module Kramdown
               attr['format'] = 'counter'
             end
           end
+          if href.match(XREF_RE_M)
+            href = $1
+            attr['text'] = $2
+          end
           href = href.gsub(/\A[0-9]/) { "_#{$&}" } # can't start an IDREF with a number
           attr['target'] = href
           el = Element.new(:xref, nil, attr)
@@ -288,6 +310,12 @@ module Kramdown
         generate_id(value).gsub(/-+/, '-')
       end
 
+      def self.process_markdown(v)             # Uuh.  Heavy coupling.
+        doc = ::Kramdown::Document.new(v, $global_markdown_options)
+        $stderr.puts doc.warnings.to_yaml unless doc.warnings.empty?
+        doc.to_rfc2629[3..-6] # skip <t>...</t>\n
+      end
+
       SVG_COLORS = Hash.new {|h, k| k}
       <<COLORS.each_line {|l| k, v = l.chomp.split; SVG_COLORS[k] = v}
 black	#000000
@@ -335,6 +363,19 @@ COLORS
         end
       end
 
+      def svg_clean_kgt(s)
+        d = REXML::Document.new(s)
+        REXML::XPath.each(d.root, "//rect|//line|//path") do |x|
+          x.attributes["fill"] = "none"
+          x.attributes["stroke"] = "black"
+          x.attributes["stroke-width"] = "1.5"
+        end
+        d.to_s
+      rescue => detail
+        warn "*** Can't clean SVG: #{detail}"
+        d
+      end
+
       def svg_clean(s)          # expensive, risky
         d = REXML::Document.new(s)
         REXML::XPath.each(d.root, "//*[@shape-rendering]") { |x| x.attributes["shape-rendering"] = nil }  #; warn x.inspect }
@@ -379,9 +420,12 @@ COLORS
         file = Tempfile.new("kramdown-rfc")
         file.write(result)
         file.close
+        dont_clean = false
+        dont_check = false
         case t
         when "goat"
           result1, err, _s = Open3.capture3("goat #{file.path}", stdin_data: result);
+          dont_clean = true
         when "ditaa"        # XXX: This needs some form of option-setting
           result1, err, _s = Open3.capture3("ditaa #{file.path} --svg -o -", stdin_data: result);
         when "mscgen"
@@ -396,18 +440,27 @@ COLORS
           result1, err, _s = Open3.capture3("plantuml -pipe -tsvg", stdin_data: plantuml);
           result, err1, _s = Open3.capture3("plantuml -pipe -tutxt", stdin_data: plantuml) if t == "plantuml-utxt"
           err << err1.to_s
+        when "railroad", "railroad-utf8"
+          result1, err1, _s = Open3.capture3("kgt -l abnf -e svg", stdin_data: result);
+          result1 = svg_clean_kgt(result1); dont_clean = true
+          result, err, _s = Open3.capture3("kgt -l abnf -e rr#{t == "railroad" ? "text" : "utf8"}",
+                                            stdin_data: result);
+          err << err1.to_s
         when "math"
           result1, err, _s = Open3.capture3("tex2svg --font STIX --speech=false #{Shellwords.escape(' ' << result)}");
           result, err1, _s = Open3.capture3("asciitex -f #{file.path}")
           err << err1
         end
         capture_croak(t, err)
-        # warn ["goat:", result1.inspect]
+        # warn ["text:", result.inspect]
+        # warn ["svg:", result1.inspect]
         file.unlink
-        result1 = svg_clean(result1) unless t == "goat"
-        result1, err, _s = Open3.capture3("svgcheck -Xqa", stdin_data: result1);
-        capture_croak("svgcheck", err)
-        # warn ["svgcheck:", result1.inspect]
+        result1 = svg_clean(result1) unless dont_clean
+        unless dont_check
+          result1, err, _s = Open3.capture3("svgcheck -Xqa", stdin_data: result1);
+          # warn ["svgcheck:", result1.inspect]
+          capture_croak("svgcheck", err)
+        end
         if result1 == ''
           warn "*** could not create svg for #{result.inspect[0...20]}..."
           exit 65 # EX_DATAERR
@@ -462,7 +515,8 @@ COLORS
             end
           end
           case t
-          when "goat", "ditaa", "mscgen", "plantuml", "plantuml-utxt", "mermaid", "math"
+          when "goat", "ditaa", "mscgen", "plantuml", "plantuml-utxt", 
+               "railroad", "railroad-utf8", "mermaid", "math"
             if gi
               warn "*** Can't set GI #{gi} for composite SVG artset"
             end
@@ -729,6 +783,7 @@ COLORS
 
       def convert_xref(el, indent, opts)
         gi = el.attr.delete('gi')
+        text = el.attr.delete('text')
         target = el.attr['target']
         if target[0] == "&"
           "#{target};"
@@ -738,7 +793,12 @@ COLORS
           else
             gi ||= "xref"
           end
-          "<#{gi}#{el_html_attributes(el)}/>"
+          if text
+            tail = ">#{Rfc2629::process_markdown(text)}</#{gi}>"
+          else
+            tail = "/>"
+          end
+          "<#{gi}#{el_html_attributes(el)}#{tail}"
         end
       end
 
@@ -833,13 +893,27 @@ COLORS
         end
       end
 
+      def self.bcp_std_ref(t, n)
+        warn "*** #{t} anchors not supported in v2 format" unless $options.v3
+        [name = "reference.#{t}.#{"%04d" % n.to_i}.xml",
+         "#{XML_RESOURCE_ORG_PREFIX}/bibxml-rfcsubseries-new/#{name}"] # FOR NOW
+      end
+
       # [subdirectory name, cache ttl in seconds, does it provide for ?anchor=]
       XML_RESOURCE_ORG_MAP = {
         "RFC" => ["bibxml", 86400*7, false,
-                  ->(fn, n){ "https://www.rfc-editor.org/refs/bibxml/#{fn}"}
+                  ->(fn, n){ [name = "reference.RFC.#{"%04d" % n.to_i}.xml",
+                              "https://www.rfc-editor.org/refs/bibxml/#{name}"] }
                  ],
         "I-D" => ["bibxml3", false, false,
-                  ->(fn, n){ "https://datatracker.ietf.org/doc/bibxml3/draft-#{n.sub(/\Adraft-/, '')}/xml" }
+                  ->(fn, n){ [fn,
+                              "https://datatracker.ietf.org/doc/bibxml3/draft-#{n.sub(/\Adraft-/, '')}/xml"] }
+                 ],
+        "BCP" => ["bibxml-rfcsubseries", 86400*7, false,
+                  ->(fn, n){ Rfc2629::bcp_std_ref("BCP", n) }
+                 ],
+        "STD" => ["bibxml-rfcsubseries", 86400*7, false,
+                  ->(fn, n){ Rfc2629::bcp_std_ref("STD", n) }
                  ],
         "W3C" => "bibxml4",
         "3GPP" => "bibxml5",
@@ -890,7 +964,7 @@ COLORS
             ttl ||= KRAMDOWN_REFCACHETTL  # everything but RFCs might change a lot
             puts "*** Huh: #{fn}" unless sub
             if altproc && !KRAMDOWN_USE_TOOLS_SERVER
-              url = altproc.call(fn, n)
+              fn, url = altproc.call(fn, n)
             else
               url = "#{XML_RESOURCE_ORG_PREFIX}/#{sub}/#{fn}"
             end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kramdown-rfc2629
 version: !ruby/object:Gem::Version
-  version: 1.4.6
+  version: 1.4.11
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-29 00:00:00.000000000 Z
+date: 2021-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kramdown
@@ -62,6 +62,7 @@ executables:
 - kramdown-rfc-extract-markdown
 - kdrfc
 - kramdown-rfc-cache-i-d-bibxml
+- kramdown-rfc-cache-subseries-bibxml
 extensions: []
 extra_rdoc_files: []
 files:
@@ -70,6 +71,7 @@ files:
 - bin/doilit
 - bin/kdrfc
 - bin/kramdown-rfc-cache-i-d-bibxml
+- bin/kramdown-rfc-cache-subseries-bibxml
 - bin/kramdown-rfc-extract-markdown
 - bin/kramdown-rfc2629
 - data/encoding-fallbacks.txt