rsyntaxtree 2.3.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c57f743f27cc6830495cfe9da3ce4483fec13491af8e80d4437357cc9c37968
-  data.tar.gz: d051b212a5abe64711fd42622bce06cea77c1e8bfc78a8315dfc1246f0dc22d0
+  metadata.gz: 2cab83f32f9b3adab3a84ef896406e8683cf3a9abcecfe132540bda198475a10
+  data.tar.gz: 29d0804c7d76c1661103e9dcbc922ea7b916ba503f3fb0c9a8f2d78c8e8008d6
 SHA512:
-  metadata.gz: e214718246e31cf0d42a03f91ee4b7e714cf1576f3f60fb70b6d0fb9d2d22a24e2b04e832d9e8be2df0b664b5506bf8a0994f659384a85469a4023535409c615
-  data.tar.gz: 5f7638450414b9cb53ac9fe2cdc08e7776af24ea50a0de75166dfbb3d47b9d4a27ab1d45a026012ef6bd9fdcc76400b1cf88f7df5095dac20cd78c629c3ccc33
+  metadata.gz: 22ddead5114613c3a1c82ce9774911e4c414721495b66ed152f904f55ba8aef3d8dde957a473dc563c764635f2892f58f7f2d0d74dd6142427e3de279e34d5c4
+  data.tar.gz: 4be4281b2e805ac45c0c01a0cc4d67b837fac67118c807812e88038ecf08fef63952897ba8fab3c66d2ec2b5a66d84428de4e038d1409ee766ab371c17fa907c

data/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## [2.4.0] - 2026-09-02
+
+### Added
+- `RSyntaxTree.escape(text, as:, hyphen:, apostrophe:)` returns notation that
+  draws as the given text, for a program writing notation from strings it did
+  not choose. The context (`:word`, `:phrase`, `:label`, `:cell`) decides what
+  whitespace means; the hyphen mode has to be given because no one spelling of
+  a hyphen serves both; `apostrophe: :keep` keeps a straight apostrophe. The
+  test draws every result and reads the text back, so the rules are exercised
+  rather than restated. The one text with no spelling under `hyphen: literal`
+  — a line of nothing but hyphens, which is the rule — raises `ArgumentError`.
+- `\'` keeps a straight apostrophe, which the notation otherwise sets as a
+  curly one. `\@` is an at sign.
+
+### Fixed
+- `\+` before digits is a plus sign. The path markers at the end of a label
+  (`+1`, `+>2`) were detected without regard to a backslash, so no label could
+  contain "+1" at all: every "C++11" and "2+2" was read as one end of a
+  movement path.
+- A label containing a backslash could lose characters in the drawing. The
+  text was placed into the SVG as a replacement string, which Ruby reads for
+  backreferences: `\+` and `\'` were two of them.
+- An escaped character the tokenizer did not know (`\'`, `\@`) lost its
+  backslash before the markup was read.
+
 ## [2.3.0] - 2026-09-01
 
 ### Added

data/lib/rsyntaxtree/element.rb

@@ -33,11 +33,8 @@ module RSyntaxTree
       @vertical_indent = 0     # Drawing offset
       content = content.strip
 
-      @path = if /.+?\^?((?:\+-?>?<?\d+)+)\^?\z/m =~ content
-                $1.sub(/\A\+/, "").split("+")
-              else
-                []
-              end
+      _body, path_text = Element.split_path(content)
+      @path = path_text.delete_prefix("^").delete_suffix("^").sub(/\A\+/, "").split("+")
 
       @fontset = fontset
       @fontsize = fontsize
@@ -64,7 +61,7 @@ module RSyntaxTree
         end
       end
 
-      @raw_content = content.sub(/\^?(?:\+-?>?<?\d+)+\^?\z/, '')
+      @raw_content = Element.split_path(content).first
 
       parsed = Markup.parse(prepare_markup(content))
 
@@ -125,9 +122,33 @@ module RSyntaxTree
     # something else: a `---` rule line of its own, and the `+-2` markers
     # at the end. Shared by swap_hyphen_markup and escape_hyphens.
     def self.hyphen_safe_lines(text)
-      path = text[/\^?(?:\+-?>?<?\d+)+\^?\z/]
-      body = path ? text[0...-path.length] : text
-      [body.split('\n', -1), path.to_s]
+      body, path = split_path(text)
+      [body.split('\n', -1), path]
+    end
+
+    # The run of path markers a label ends in: `+1`, `+>2`, `+-3`, several of
+    # them, an optional `^` on either side. A marker's `+` is a marker only
+    # when the backslashes before it are even in number: `x\+1` is a plus
+    # sign and a digit, `x\\+1` a backslash and a marker. Detection used to
+    # ignore the backslash, so a label could not contain "+1" at all — every
+    # "C++11" and "2+2" was read as one end of a movement path.
+    PATH_MARKERS = /(?:\+-?>?<?\d+)+/
+
+    # Splits a label into what is drawn and its trailing path text, the
+    # latter empty when there is none. A label that is nothing but markers
+    # has no path: there would be nothing to draw them from.
+    def self.split_path(text)
+      i = 0
+      while (i = text.index("+", i))
+        tail = text[i..]
+        if i.positive? && tail.match?(/\A#{PATH_MARKERS}\^?\z/) &&
+           text[0...i][/\\*\z/].length.even?
+          i -= 1 if i > 1 && text[i - 1] == "^"
+          return [text[0...i], text[i..]]
+        end
+        i += 1
+      end
+      [text, ""]
     end
 
     def swap_hyphen_markup(text)
@@ -366,7 +387,7 @@ module RSyntaxTree
             # as a curly apostrophe (U+2019) for smarter typography, e.g. the
             # X-bar prime in "T'". Applied before metrics so the measured glyph
             # matches the rendered one.
-            text = text.gsub("'", "’")
+            text = text.gsub("'", "’").gsub(Markup::KEPT_APOSTROPHE, "'")
             e[:text] = text.gsub(" ", WHITESPACE_BLOCK)
                           .gsub(">", '&#62;')
                           .gsub("<", '&#60;')

data/lib/rsyntaxtree/escape.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+#==========================
+# escape.rb
+#==========================
+#
+# Turns arbitrary text into notation that draws as that text.
+# Copyright (c) 2007-2026 Yoichiro Hasebe <yohasebe@gmail.com>
+
+module RSyntaxTree
+  # The characters the notation reads as markup, each written as itself by
+  # a backslash. The backslash comes first so that the others are escaped
+  # once, not twice. A yen sign is here because the reader treats it as a
+  # backslash for keyboards that have no other.
+  ESCAPED_CHARACTERS = %w[\\ [ ] < > ^ + * _ = ~ | # { } % @ ¥].freeze
+
+  ESCAPE_CONTEXTS = %i[word phrase label cell].freeze
+
+  # The web interface's transport spellings of characters a form cannot
+  # carry, read out of the input before anything else. Under hyphen: markup
+  # the hyphens are escaped and nothing matches; under :literal they would
+  # be read, so a backslash goes in after the first hyphen — an escape the
+  # reader does not know, whose backslash it drops — and the text survives.
+  TRANSPORT_PLACEHOLDERS = %w[AMP PERCENT PRIME SCOLON OABRACKET CABRACKET].freeze
+
+  # A line of three or more hyphens and nothing else is the horizontal rule.
+  RULE_LINE = /\A-{3,}\z/
+
+  # Notation that draws as +text+, for a program writing notation from
+  # strings it did not choose — a tagger's tokens, a corpus's words.
+  # Written here rather than in every such program, because a copy of the
+  # escaping rules kept elsewhere would drift from the notation as it
+  # changes, and the drift would show as a different figure drawn without
+  # complaint. The test for this method draws each result and checks that
+  # the text came out, so the rules are never restated, only exercised.
+  #
+  # +as+ says where the text will stand, which decides what whitespace
+  # means: a :word is one leaf or one label (a space inside it becomes <>),
+  # a :phrase is a leaf of several words (spaces stay, so the leaf gets a
+  # triangle), a :label is a node label (like a word, with a newline kept
+  # as a line break), and a :cell is one cell of a column-aligned or
+  # matrix label (tabs and newlines become the column and row breaks).
+  #
+  # +hyphen+ must match the option the figure is drawn with. Under :markup
+  # a hyphen is escaped; under :literal a bare hyphen is already itself,
+  # and the escaped form means an underline instead — so there is no
+  # notation that reads as a hyphen under both, and the caller has to say.
+  #
+  # +apostrophe+ :curly leaves a straight apostrophe to be set as a curly
+  # one, which is what the notation does; :keep escapes it so it stays
+  # straight, for text that must come out exactly as given.
+  #
+  # One text has no spelling under hyphen: :literal — a line of nothing but
+  # three or more hyphens, which is the horizontal rule and draws as one —
+  # and asking for it raises ArgumentError rather than returning notation
+  # that draws an empty leaf. Empty and all-whitespace text is returned as
+  # the whitespace it holds, which draws as a blank; a caller building a
+  # tree from tokens will usually want to drop such tokens first.
+  def self.escape(text, as: :word, hyphen: :markup, apostrophe: :curly)
+    unless ESCAPE_CONTEXTS.include?(as)
+      raise ArgumentError, "as: must be one of #{ESCAPE_CONTEXTS.join(', ')}"
+    end
+    raise ArgumentError, "hyphen: must be :markup or :literal" unless %i[markup literal].include?(hyphen)
+    raise ArgumentError, "apostrophe: must be :curly or :keep" unless %i[curly keep].include?(apostrophe)
+
+    s = text.to_s
+    if as == :phrase
+      words = s.split(/\s+/).reject(&:empty?)
+      # One word alone is the whole leaf; with company, hyphens are text.
+      refuse_rule_line(s, hyphen) if hyphen == :literal && words.length == 1 && words.first.match?(RULE_LINE)
+      return words.map { |w| escape_text(w, :word, hyphen, apostrophe) }.join(" ")
+    end
+
+    if hyphen == :literal
+      # A newline is a line break in a label or a cell, and a space in a word.
+      lines = %i[label cell].include?(as) ? s.split(/\r\n?|\n/, -1) : [s]
+      lines.each { |line| refuse_rule_line(s, hyphen) if line.match?(RULE_LINE) }
+    end
+    escape_text(s, as, hyphen, apostrophe)
+  end
+
+  # The character work, once the text has been accepted.
+  def self.escape_text(s, as, hyphen, apostrophe)
+    # Every replacement is a block: as a replacement string a backslash is
+    # read for backreferences, and `\\'` and `\\+` are two of them.
+    out = s.gsub(/[#{Regexp.escape(ESCAPED_CHARACTERS.join)}]/) { |c| "\\#{c}" }
+    out = out.gsub("-") { "\\-" } if hyphen == :markup
+    if hyphen == :literal
+      out = out.gsub(/-(#{TRANSPORT_PLACEHOLDERS.join('|')})-/) { "-\\#{Regexp.last_match(1)}-" }
+    end
+    out = out.gsub("'") { "\\'" } if apostrophe == :keep
+    out = out.gsub(/\r\n?|\n/) { "\\n" } if %i[label cell].include?(as)
+    out = out.gsub("\t") { "\\t" } if as == :cell
+    # A CRLF is one break, so one space, as it is one break in a label.
+    out.gsub(/\r\n|\s/) { "<>" }
+  end
+  private_class_method :escape_text
+
+  def self.refuse_rule_line(text, hyphen)
+    raise ArgumentError,
+          "no notation draws #{text.inspect} under hyphen: #{hyphen.inspect}: a line of " \
+          "nothing but hyphens is the horizontal rule. Draw with hyphen: :markup."
+  end
+  private_class_method :refuse_rule_line
+end

data/lib/rsyntaxtree/markup_parser.rb

@@ -39,9 +39,13 @@ class MarkupParser < Parslet::Parser
 
   rule(:path) { (str('+') >> str('-').maybe >> (str('>') | str('<')).maybe >> match('\d').repeat(1)).as(:path) }
   # rule(:escaped) { str('\\') >> match('[#<>{}\\^+*_=~\|\n\-]').as(:chr) }
-  rule(:escaped) { str('\\') >> match('[#<>{}\\\\^+*_=~\\|\\n\\-\\[\\]%]').as(:chr) }
+  rule(:escaped) { str('\\') >> match('[#<>{}\\\\^+*_=~\\|\\n\\-\\[\\]%@]').as(:chr) }
+  # A straight apostrophe is set as a curly one; `\'` asks for the straight
+  # one itself. It comes through as a placeholder so the substitution, which
+  # runs later on the joined text, can tell the two apart.
+  rule(:kept_apostrophe) { str("\\'").as(:kept_apostrophe) }
   rule(:non_escaped) { ((match('[#<>{}\\^+*_=~\|\-]') | str('\\n') | str('\\t')).absent? >> any).as(:chr) }
-  rule(:text) { (escaped | non_escaped).repeat(1).as(:text) }
+  rule(:text) { (kept_apostrophe | escaped | non_escaped).repeat(1).as(:text) }
 
   # Column separator. Every line of a label is cut at these into cells, and
   # each column is laid out at the width of its widest cell, which is what an
@@ -87,7 +91,7 @@ class MarkupParser < Parslet::Parser
   rule(:matrix_line) { (rule_line | double_rule_line | matrix_markup.repeat(1).as(:line)) >> (cr | str('#)').present?) }
   rule(:matrix_markup) { (matrix | tabstop | matrix_text | decoration | shape | bstroke) }
   # Text inside a matrix stops at the closing delimiter as well.
-  rule(:matrix_text) { (escaped | (str('#)').absent? >> non_escaped)).repeat(1).as(:text) }
+  rule(:matrix_text) { (kept_apostrophe | escaped | (str('#)').absent? >> non_escaped)).repeat(1).as(:text) }
 
   rule(:markup) { (matrix | tabstop | text | decoration | shape | bstroke) }
 
@@ -107,10 +111,15 @@ class MarkupParser < Parslet::Parser
 end
 
 module Markup
+  # Stands in for an escaped apostrophe until the curly substitution has
+  # run. A private-use character no label carries.
+  KEPT_APOSTROPHE = "\uE000"
+
   @parser = MarkupParser.new
 
   @evaluator = Parslet::Transform.new do
     rule(chr: simple(:chr)) { chr.to_s }
+    rule(kept_apostrophe: simple(:kept)) { KEPT_APOSTROPHE }
     rule(text: sequence(:text)) { { text: text.join(""), decoration: [] } }
 
     rule(tabstop: subtree(:empty)) {

data/lib/rsyntaxtree/notation_core.md

@@ -24,9 +24,16 @@ figure that was meant.
   `\n` or `\t` in it: there it can split a value that carries markup, and it
   breaks a nested #( ... #) matrix. Inside any label with columns, write a
   space as `<>`: 'a<>toy', not 'a toy'.
-- A straight apostrophe is typeset as the curly ’ (U+2019).
+- A straight apostrophe is typeset as the curly ’ (U+2019); \' keeps it
+  straight.
+- A `+` followed by digits at the end of a label is a movement path (+1, +>2).
+  \+ is a plus sign, so C\+\+11 and 2\+2 draw as written.
 - A backslash takes the character after it, whatever it is: \q draws q, and
   C:\path draws C:path. A backslash itself is \\.
+- A program building labels from text it did not choose should not copy
+  these rules: RSyntaxTree.escape(text, as: :word) returns notation that
+  draws as the text (as: :phrase, :label or :cell for the other places;
+  hyphen: to match the hyphen option; apostrophe: :keep to keep it straight).
 - The prefixes of a label compose in one order, and only this one:
   `^` → `#`/`##`/`###` → `%` → `@color:`. So ^#%@red:NP is read whole, while
   @red:%NP leaves the % as a literal character and silently drops the shade.

data/lib/rsyntaxtree/string_parser.rb

@@ -256,7 +256,7 @@ module RSyntaxTree
         # `\#` arrived at the grammar as a bare '#', which opens an enclosure,
         # so a label written with a hash in it lost the hash and everything
         # after it went inside brackets instead.
-        when /[nt{}<>^+*_=~|%\-#]/
+        when /[nt{}<>^+*_=~|%\-#@']/
           if escape
             token += '\\' + ch
             escape = false

data/lib/rsyntaxtree/svg_graph.rb

@@ -623,7 +623,11 @@ module RSyntaxTree
         element.ink_top = first_baseline - f_m.ink_above
         element.ink_bottom = text_y + (l_m.ink_height - l_m.ink_above)
       end
-      @tree_data += text_data.sub(/CONTENT/, new_text)
+      # The block form, so the label goes in as it is. As a replacement
+      # string it was read for backreferences, and `\+` — the last matched
+      # group, of which there is none — took a backslash and a plus sign out
+      # of any label that carried them.
+      @tree_data += text_data.sub(/CONTENT/) { new_text }
     end
 
     # Draws one run of a label — a stretch of text with its decorations, or the

data/lib/rsyntaxtree/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RSyntaxTree
-  VERSION = "2.3.0"
+  VERSION = "2.4.0"
 end

data/lib/rsyntaxtree.rb

@@ -210,6 +210,7 @@ require_relative 'rsyntaxtree/tikz_generator'
 require_relative 'rsyntaxtree/version'
 require_relative 'rsyntaxtree/string_parser'
 require_relative 'rsyntaxtree/format_converter'
+require_relative 'rsyntaxtree/escape'
 
 require 'cgi'
 require 'rsvg2'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rsyntaxtree
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Yoichiro Hasebe
@@ -128,6 +128,7 @@ files:
 - lib/rsyntaxtree/color_names.rb
 - lib/rsyntaxtree/element.rb
 - lib/rsyntaxtree/elementlist.rb
+- lib/rsyntaxtree/escape.rb
 - lib/rsyntaxtree/format_converter.rb
 - lib/rsyntaxtree/json_graph.rb
 - lib/rsyntaxtree/markup_parser.rb