kramdown-rfc2629 1.6.31 → 1.6.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78da707841d2475396ae7578307bf10ab16440755d79591078490738b9dd89ed
-  data.tar.gz: dc0143966fdc29ebe27e1cb07c28e4541b27038fac6d5a783d3dc2368e50d657
+  metadata.gz: d4c663bed0e8e2dc848a9e532ba201b9e9a26b4090368c0d83f8c518275df710
+  data.tar.gz: a2ac5d0be0615b41ecd65964ca7dfd61e3163fbb3aa612994fa5ac05ae1504ca
 SHA512:
-  metadata.gz: 71abd171f1fce10fd52d5ba1d1cbedcd7c982090acf5d120c23be6702528a2eb5e09da2da9a6fc2d161eba5fc0980f5850eced87c403b7d21971e52588f849b8
-  data.tar.gz: ea9ffd32a3189c778f3e74ead8ea43e8bc3961e9886a3b2055377da32f4ec36148eeee56c120fa627c3739cb41c9ab037d48f1300801ce3a6e61d96564003cb6
+  metadata.gz: 90ef1269825ef2b9484fc358cf92fa7ffa1ce255ea6e0b520cf99e98ed9f463551bf060d9af84de1ef87f0bd2e8a12442c4b9dc866858e1ed18bf23d4084829a
+  data.tar.gz: 840c014bc4efcabb1491e81126f577541bb237b0fa92a06fdaa293556eed480997829a357728da96caadc480347d32ddfb3be76a8e7027c986c775adff9babc7

data/bin/kramdown-rfc-extract-sourcecode

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby -KU
+require 'rexml/document'        # for SVG and bibxml acrobatics
+require 'yaml'
+require 'shellwords'
+require 'fileutils'
+require_relative '../lib/kramdown-rfc/rfc8792'
+
+def clean(s)
+  s.gsub!(/\//, ":")
+  s.gsub!(/[^:\w]/, "_")        # handles leading dot, too --   s.gsub!(/\A([.]+)/) {"_" * $1.size } otherwise
+  s
+end
+
+target = nil
+dir = nil
+unfold = true
+targets = [:list, :files, :zip, :yaml]
+require 'optparse'
+begin
+  op = OptionParser.new do |opts|
+    opts.banner = "Usage: kramdown-rfc-extract-sourcecode [options] document.xml"
+    opts.on("-tFMT", "--to=FMT", targets, "Target format #{targets.map(&:to_s)}") do |v|
+      target = v
+    end
+    opts.on("-dDIR", "--dir=DIR", "Target directory (default: sourcecode)") do |v|
+      dir = v
+    end
+    opts.on("-f", "--[no-]unfold", "RFC8972-unfold (default: yes)") do |v|
+      unfold = v
+    end
+  end
+  op.parse!
+rescue Exception => e
+  warn e
+  exit 1
+end
+if dir
+  target ||= :files
+  unless [:files, :zip].include? target
+    warn "** Unused argument --dir=#{dir}"
+  end
+end
+dir ||= "sourcecode"
+
+target ||= :list
+
+gensym = "unnamed-000"
+taken = Hash.new { |h, k| h[k] = Hash.new }
+warned = Hash.new { |h, k| h[k] = Hash.new }
+d = REXML::Document.new(ARGF)
+REXML::XPath.each(d.root, "//sourcecode|//artwork") do |x|
+  if ty = x[:type]
+    ty = clean(ty)
+    name = x[:name] || gensym.succ!
+    name = clean(name)
+    if taken[ty][name]
+      unless warned[ty][name]
+        warn "Concatenating to #{ty}/#{name}."
+        warned[ty][name] = true
+      end
+    else
+      taken[ty][name] = ''
+    end
+    taken[ty][name] << handle_artwork_sourcecode(x.text, unfold)
+  end
+end
+
+def make_directory_from(dir, taken)
+  if File.exist?(dir)
+    bak = "#{dir}.bak"
+    begin
+      FileUtils.mv(dir, bak)
+    rescue Errno::EEXIST
+      bak.succ!
+      retry
+    end
+  end
+  FileUtils.mkdir_p(dir)
+  taken.each { |dir1, v|
+    FileUtils.mkdir_p("#{dir}/#{dir1}")
+    v.each { |fn, value|
+      IO.write("#{dir}/#{dir1}/#{fn}", value)
+    }
+  }
+end
+
+case target
+when :yaml
+  puts taken.to_yaml
+when :list
+  puts Hash[
+         taken.map {|k, v|
+           [k, v.keys]
+         }
+       ].to_yaml
+when :files
+  make_directory_from(dir, taken)
+when :zip
+  make_directory_from(dir, taken)
+  zip = "#{dir}.zip"
+  if File.exist?(zip)
+    bak = "#{zip}.bak"
+    begin
+      FileUtils.mv(zip, bak)
+    rescue Errno::EEXIST        # This doesn't actually happen.  XXX
+      bak.succ!
+      retry
+    end
+  end
+  cmd = ["zip", "-mr", zip, dir].shelljoin
+  warn cmd
+  system(cmd)
+end

data/kramdown-rfc2629.gemspec

@@ -1,6 +1,6 @@
 spec = Gem::Specification.new do |s|
   s.name = 'kramdown-rfc2629'
-  s.version = '1.6.31'
+  s.version = '1.6.33'
   s.summary = "Kramdown extension for generating RFCXML (RFC 799x)."
   s.description = %{An RFCXML (RFC 799x) generating backend for Thomas Leitner's
 "kramdown" markdown parser.  Mostly useful for RFC writers.}
@@ -13,10 +13,11 @@ spec = Gem::Specification.new do |s|
   s.add_dependency('unicode-scripts', '~> 1.0')
   s.add_dependency('net-http-persistent', '~> 4.0')
   s.add_dependency('differ', '~>0.1')
-  s.files = Dir['lib/**/*.rb'] + %w(README.md LICENSE kramdown-rfc2629.gemspec bin/kdrfc bin/kramdown-rfc bin/kramdown-rfc2629 bin/doilit bin/echars bin/kramdown-rfc-extract-markdown data/kramdown-rfc2629.erb data/encoding-fallbacks.txt data/math.json bin/kramdown-rfc-cache-subseries-bibxml bin/kramdown-rfc-autolink-iref-cleanup bin/de-gfm bin/kramdown-rfc-clean-svg-ids)
+  s.files = Dir['lib/**/*.rb'] + %w(README.md LICENSE kramdown-rfc2629.gemspec bin/kdrfc bin/kramdown-rfc bin/kramdown-rfc2629 bin/doilit bin/echars bin/kramdown-rfc-extract-markdown bin/kramdown-rfc-extract-sourcecode data/kramdown-rfc2629.erb data/encoding-fallbacks.txt data/math.json bin/kramdown-rfc-cache-subseries-bibxml bin/kramdown-rfc-autolink-iref-cleanup bin/de-gfm bin/kramdown-rfc-clean-svg-ids)
   s.require_path = 'lib'
   s.executables = ['kramdown-rfc', 'kramdown-rfc2629', 'doilit', 'echars',
                    'kramdown-rfc-extract-markdown',
+                   'kramdown-rfc-extract-sourcecode',
                    'kdrfc', 'kramdown-rfc-cache-i-d-bibxml',
                    'kramdown-rfc-cache-subseries-bibxml',
                    'kramdown-rfc-autolink-iref-cleanup',

data/lib/kramdown-rfc/command.rb

@@ -13,13 +13,6 @@ KDRFC_VERSION=Gem.loaded_specs["kramdown-rfc2629"].version rescue "unknown-versi
 
 Encoding.default_external = "UTF-8" # wake up, smell the coffee
 
-# Note that this doesn't attempt to handle HT characters
-def remove_indentation(s)
-  l = s.lines
-  indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
-  l.map {|li| li.sub(/^ {0,#{indent}}/, "")}.join
-end
-
 def add_quote(s)
   l = s.lines
   l.map {|li| "> #{li}"}.join
@@ -141,8 +134,8 @@ def do_the_tls_dance
   end
 end
 
-RE_NL = /(?:\n|\r|\r\n)/
-RE_SECTION = /---(?: +(\w+)(-?))?\s*#{RE_NL}(.*?#{RE_NL})(?=---(?:\s+\w+-?)?\s*#{RE_NL}|\Z)/m
+RE_NL = /(?:\r\n|\n|\r)/
+RE_SECTION = /---(?: +(\w+)(-?))? *#{RE_NL}(.*?#{RE_NL})(?=---(?:\s+\w+-?)?\s*#{RE_NL}|\Z)/m
 
 NMDTAGS = ["{:/nomarkdown}\n\n", "\n\n{::nomarkdown}\n"]
 
@@ -240,6 +233,12 @@ def xml_from_sections(input)
 
   sections = input.scan(RE_SECTION)
   # resulting in an array; each section is [section-label, nomarkdown-flag, section-text]
+  line = 1                      # skip "---"
+  sections.each do |section|
+    section << line
+    line += 1 + section[2].lines.count
+  end
+  # warn "#{line-1} lines"
 
   # the first section is a YAML with front matter parameters (don't put a label here)
   # We put back the "---" plus gratuitous blank lines to hack the line number in errors
@@ -278,7 +277,7 @@ def xml_from_sections(input)
   # all the other sections are put in a Hash, possibly concatenated from parts there
   sechash = Hash.new{ |h,k| h[k] = ""}
   snames = []                   # a stack of section names
-  sections.each do |sname, nmdflag, text|
+  sections.each do |sname, nmdflag, text, line|
     # warn [:SNAME, sname, nmdflag, text[0..10]].inspect
     nmdin, nmdout = {
       "-" => ["", ""],          # stay in nomarkdown
@@ -289,7 +288,7 @@ def xml_from_sections(input)
     else
       snames.pop                # just "---" -> pop label (previous now current)
     end
-    sechash[snames.last] << "#{nmdin}#{text}#{nmdout}"
+    sechash[snames.last] << "#{nmdin}<?line #{line}?>\n#{text}#{nmdout}"
   end
 
   ref_replacements = { }
@@ -540,7 +539,9 @@ if input[-1] != "\n"
 end
 process_includes(input) unless ENV["KRAMDOWN_SAFE"]
 input.gsub!(/^\{::boilerplate\s+(.*?)\}/) {
-  boilerplate($1)
+  bp = boilerplate($1)
+  delta = bp.lines.count
+  bp + "<?line -#{delta+1}?>\n"
 }
 if input =~ /[\t]/
    warn "*** Input contains HT (\"tab\") characters. Undefined behavior will ensue."

data/lib/kramdown-rfc/rfc8792.rb

@@ -1,4 +1,45 @@
+
+# Note that this doesn't attempt to handle HT characters
+def remove_indentation(s)
+  l = s.lines
+  indent = l.grep(/\S/).map {|l| l[/^\s*/].size}.min
+  l.map {|li| li.sub(/^ {0,#{indent}}/, "")}.join
+end
+
+def trim_empty_lines_around(s)  # this deletes the trailing newline, which may need to be reconstructed
+  s.sub(/\A(\r?\n)*/, '').sub(/(\r?\n)*\z/, '')
+end
+
+def fix_unterminated_line(s)
+  s.sub(/[^\n]\z/) { "#$&\n" } # XXX
+end
+
+def handle_artwork_sourcecode(s, unfold = true)
+  s = trim_empty_lines_around(s)
+  s = unfold8792(s) if unfold
+  fix_unterminated_line(s)
+end
+
 FOLD_MSG = "NOTE: '\\' line wrapping per RFC 8792".freeze
+UNFOLD_RE = /\A.*#{FOLD_MSG.sub("\\", "(\\\\\\\\\\\\\\\\?)")}.*\n\r?\n/
+
+def unfold8792(s)
+  if s =~ UNFOLD_RE
+    indicator = $1
+    s = $'
+    sub = case indicator
+          when "\\"
+            s.gsub!(/\\\n[ \t]*/, '')
+          when "\\\\"
+            s.gsub!(/\\\n[ \t]*\\/, '')
+          else
+            fail "indicator"    # Cannot happen
+          end
+    warn "** encountered RFC 8792 header without folded lines" unless sub
+  end
+  s
+end
+
 MIN_FOLD_COLUMNS = FOLD_MSG.size
 FOLD_COLUMNS = 69
 RE_IDENT = /\A[A-Za-z0-9_]\z/

data/lib/kramdown-rfc2629.rb

@@ -23,6 +23,7 @@ require 'json'                  # for math
 require 'rexml/document'        # for SVG and bibxml acrobatics
 
 require 'kramdown-rfc/doi'      # for fetching information for a DOI
+require 'kramdown-rfc/rfc8792'
 
 class Object
   def deep_clone
@@ -315,15 +316,21 @@ module Kramdown
       def initialize(*doc)
         super
         @sec_level = 1
+        @location_delta = 100000 # until reset
+        @location_correction = 0 # pre-scanning corrections
         @in_dt = 0
         @footnote_names_in_use = {}
       end
 
+      def correct_location(location)
+        location + @location_delta + @location_correction
+      end
+
       def convert(el, indent = -INDENTATION, opts = {})
         if el.children[-1].type == :raw
           raw = convert1(el.children.pop, indent, opts)
         end
-        "#{convert1(el, indent, opts)}#{end_sections(1, indent)}#{raw}"
+        "#{convert1(el, indent, opts)}#{end_sections(1, indent, el.options[:location])}#{raw}"
       end
 
       def convert1(el, indent, opts = {})
@@ -658,34 +665,80 @@ COLORS
               if anchor = el.attr['anchor']
                 "##{anchor}"
               elsif lineno = el.options[:location]
-                "approx. line #{lineno}" # XXX
+                "#{correct_location(lineno)}"
               else
                 "UNKNOWN"
               end
+            preprocs = el.attr.delete("pre")
+            checks = el.attr.delete("check")
+            postprocs = el.attr.delete("post")
             case t
             when "json"
+              checks ||= "json"
+            when /\A(.*)-from-yaml\z/
+              t = $1
+              preprocs ||= "yaml2json"
+            end
+            preprocs = (preprocs || '').split("-")
+            checks = (checks || '').split("-")
+            postprocs = (postprocs || '').split("-")
+            result = sourcecode_checkproc(preprocs, checks, postprocs, loc_str, result)
+            "#{' '*indent}<figure#{el_html_attributes(el)}><#{gi}#{html_attributes(artwork_attr)}><![CDATA[#{result}#{result =~ /\n\Z/ ? '' : "\n"}]]></#{gi}></figure>\n"
+          end
+        end
+      end
+
+      def sourcecode_proc(proc, loc_str, result)
+        case proc
+        when "dedent"
+          result = remove_indentation(result)
+        when /\Afold(\d*)(left(\d*))?(dry)?\z/
+          fold = [$1.to_i,            # col 0 for ''
+                  ($3.to_i if $2),    # left 0 for '', nil if no "left"
+                  $4]                 # dry
+          result = fix_unterminated_line(fold8792_1(trim_empty_lines_around(result), *fold)) # XXX
+        when "yaml2json"
+          begin
+            y = YAML.safe_load(result, aliases: true, filename: loc_str)
+            result = JSON.pretty_generate(y)
+          rescue => e
+            warn "*** #{loc_str}: YAML isn't: #{e.message}\n"
+          end
+        else
+          warn "*** #{loc_str}: unknown proc '#{proc}'"
+        end
+        result
+      end
+
+      def sourcecode_checkproc(preprocs, checks, postprocs, loc_str, result)
+        preprocs.each do |proc|
+          result = sourcecode_proc(proc, loc_str, result)
+        end if preprocs
+        check_input = result
+        checks.each do |check|
+          case check
+          when "skipheader"
+            check_input = handle_artwork_sourcecode(check_input).sub(/.*?\n\n/m, '')
+          when "json"
+            # check for 8792; undo if needed:
+            begin
+              JSON.load(handle_artwork_sourcecode(check_input))
+            rescue => e
+              err1 = "*** #{loc_str}: JSON isn't: #{JSON.dump(e.message[0..40])}\n"
               begin
-                JSON.load(result)
-              rescue => e
-                err1 = "*** #{loc_str}: JSON isn't: #{e.message[0..40]}\n"
-                begin
-                  JSON.load("{" << result << "}")
-                rescue => e
-                  warn err1 << "***  not even with braces added around: #{e.message[0..40]}"
-                end
-              end
-            when "json-from-yaml"
-              begin
-                y = YAML.safe_load(result, aliases: true, filename: loc_str)
-                result = JSON.pretty_generate(y)
-                t = "json"      # XXX, this could be another format!
+                JSON.load("{" << check_input << "}")
               rescue => e
-                warn "*** YAML isn't: #{e.message}\n"
+                warn err1 << "***  not even with braces added around: #{JSON.dump(e.message[0..40])}"
               end
             end
-            "#{' '*indent}<figure#{el_html_attributes(el)}><#{gi}#{html_attributes(artwork_attr)}><![CDATA[#{result}#{result =~ /\n\Z/ ? '' : "\n"}]]></#{gi}></figure>\n"
+          else
+            warn "*** #{loc_str}: unknown check '#{check}'"
           end
-        end
+        end if checks
+        postprocs.each do |proc|
+          result = sourcecode_proc(proc, loc_str, result)
+        end if postprocs
+        result
       end
 
       def mk_artwork(artwork_attr, typ, content)
@@ -707,7 +760,7 @@ COLORS
         end
       end
 
-      def end_sections(to_level, indent)
+      def end_sections(to_level, indent, location)
         if indent < 0
           indent = 0
         end
@@ -716,7 +769,7 @@ COLORS
           @sec_level = to_level
           "#{' '*indent}</section>\n" * delta
         else
-          $stderr.puts "Incorrect section nesting: Need to start with 1"
+          $stderr.puts "** #{correct_location(location)}: Bad section nesting: start heading level at 1 and increment by 1"
         end
       end
 
@@ -768,7 +821,7 @@ COLORS
         clean, irefs = clean_pcdata(inner_a(el, indent, opts))
         el.attr['title'] = clean
         end
-        "#{end_sections(el.options[:level], indent)}#{' '*indent}<section#{@sec_level += 1; el_html_attributes(el)}>#{irefs}\n"
+        "#{end_sections(el.options[:level], indent, el.options[:location])}#{' '*indent}<section#{@sec_level += 1; el_html_attributes(el)}>#{irefs}\n"
       end
 
       def convert_hr(el, indent, opts) # misuse for page break
@@ -885,6 +938,15 @@ COLORS
       end
 
       def convert_xml_comment(el, indent, opts)
+        if el.value =~ /\A<\?line (([-+]?)[0-9]+)\?>\z/
+          lineno = $1.to_i
+          case $2
+          when ''               # absolute
+            @location_delta = lineno - el.options[:location]
+          when '+', '-'         # correction (pre-scanning!)
+            @location_correction += lineno
+          end
+        end
         if el.options[:category] == :block && !el.options[:parent_is_raw]
           ' '*indent + el.value + "\n"
         else
@@ -1277,7 +1339,7 @@ COLORS
       end
 
       def convert_raw(el, indent, opts)
-        end_sections(1, indent) +
+        end_sections(1, indent, el.options[:location]) +
         el.value + (el.options[:category] == :block ? "\n" : '')
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kramdown-rfc2629
 version: !ruby/object:Gem::Version
-  version: 1.6.31
+  version: 1.6.33
 platform: ruby
 authors:
 - Carsten Bormann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-20 00:00:00.000000000 Z
+date: 2023-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kramdown
@@ -146,6 +146,7 @@ executables:
 - doilit
 - echars
 - kramdown-rfc-extract-markdown
+- kramdown-rfc-extract-sourcecode
 - kdrfc
 - kramdown-rfc-cache-i-d-bibxml
 - kramdown-rfc-cache-subseries-bibxml
@@ -167,6 +168,7 @@ files:
 - bin/kramdown-rfc-cache-subseries-bibxml
 - bin/kramdown-rfc-clean-svg-ids
 - bin/kramdown-rfc-extract-markdown
+- bin/kramdown-rfc-extract-sourcecode
 - bin/kramdown-rfc2629
 - data/encoding-fallbacks.txt
 - data/kramdown-rfc2629.erb