slaw 10.4.1 → 10.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aa5a1bf4a3098846a8f22ab9cff0c1a44f376407b9cc9c2f314ec69d9445c073
-  data.tar.gz: 918bc6711f33db010f38c80606a52c7f9ff446778230dcbeafa27d34abc5eceb
+  metadata.gz: ba413f53b9d24192d6ce5a168eed83ca55a317dcb3768009cfdf3bf902a23327
+  data.tar.gz: f664b13ce90bb21b65c0fffb2f780dc5937429c31ac2a9d6270259e0669f2f61
 SHA512:
-  metadata.gz: 1231195c46118f5098b4b31deca137981d03dacdcdb57175c30779cd4abc0339a95d480d3d08962e8d8dfae8d6c67880c1f219867583fe3df15b531391b19086
-  data.tar.gz: 7bdbb9416a5c77eddab03eafef40b8aae830c2bcb71dc2cb4dc15f746be7404fd7122dbc242150e6ae90a9dca4f3a144f59013e3d36d5662de30f815aaf6ab00
+  metadata.gz: 0f0929aa1fcce2c86f6340fec7731f5e865e0b58b793b4f7d64ba48ec2463299498b4b96fea3487a11ab57a2ae932273e27c593395cd2ca263347006fa36a88a
+  data.tar.gz: 72aceb516d091bd396deeb828873058a3cda366524c3e1819590ae77f021ab9a806baf0108f1239dceaaf0e7a64713e1c954ec385b5ed03563f1d702cd4b0984

data/README.md

@@ -86,6 +86,10 @@ You can create your own grammar by creating a gem that provides these files and
 
 ## Changelog
 
+### 10.5.0 (20 April 2021)
+
+* Handle escaping inlines when unparsing.
+
 ### 10.4.1 (14 April 2021)
 
 * Handle escaping in inlines, so that forward slashes in link text are unescaped correctly, eg `[https:\/\/example.com](https://example.com)`

data/lib/slaw/grammars/inlines.treetop

@@ -20,7 +20,7 @@ module Slaw
       end
 
       rule inline_item
-        remark / image / ref / bold / italics / superscript / subscript / [^\n]
+        remark / image / ref / bold / italics / superscript / subscript / '\\'? [^\n]
         <InlineItem>
       end
 

data/lib/slaw/grammars/inlines_nodes.rb

@@ -37,8 +37,12 @@ module Slaw
 
       class InlineItem < Treetop::Runtime::SyntaxNode
         def to_xml(b, idprefix)
-          # handle escaped characters foo\/bar -> foo/bar
-          b.text(text_value.gsub(/\\(.)?/, '\1'))
+          if text_value.start_with? '\\'
+            # handle escaped characters: \a -> a
+            b.text(text_value[1..])
+          else
+            b.text(text_value)
+          end
         end
       end
 

data/lib/slaw/grammars/za/act_text.xsl

@@ -9,31 +9,141 @@
   <xsl:strip-space elements="*"/>
   <xsl:preserve-space elements="a:a a:affectedDocument a:b a:block a:caption a:change a:concept a:courtType a:date a:def a:del a:docCommittee a:docDate a:docIntroducer a:docJurisdiction a:docNumber a:docProponent a:docPurpose a:docStage a:docStatus a:docTitle a:docType a:docketNumber a:entity a:event a:extractText a:fillIn a:from a:heading a:i a:inline a:ins a:judge a:lawyer a:legislature a:li a:listConclusion a:listIntroduction a:location a:mmod a:mod a:mref a:narrative a:neutralCitation a:num a:object a:omissis a:opinion a:organization a:outcome a:p a:party a:person a:placeholder a:process a:quantity a:quotedText a:recordedTime a:ref a:relatedDocument a:remark a:rmod a:role a:rref a:scene a:session a:shortTitle a:signature a:span a:sub a:subheading a:summary a:sup a:term a:tocItem a:u a:vote"/>
 
+  <!-- replaces "value" in "text" with "replacement" -->
+  <xsl:template name="string-replace-all">
+    <xsl:param name="text" />
+    <xsl:param name="value" />
+    <xsl:param name="replacement" />
+
+    <xsl:choose>
+      <xsl:when test="$text = '' or $value = '' or not($value)">
+        <xsl:value-of select="$text" />
+      </xsl:when>
+      <xsl:when test="contains($text, $value)">
+        <xsl:value-of select="substring-before($text, $value)"/>
+        <xsl:value-of select="$replacement" />
+        <xsl:call-template name="string-replace-all">
+          <xsl:with-param name="text" select="substring-after($text, $value)" />
+          <xsl:with-param name="value" select="$value" />
+          <xsl:with-param name="replacement" select="$replacement" />
+        </xsl:call-template>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:value-of select="$text" />
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <!-- Escape inline markers with a backslash -->
+  <xsl:template name="escape-inlines">
+    <xsl:param name="text" />
+
+    <!-- This works from the inside out, first escaping backslash chars themselves, then escaping
+          the different types of inline markers -->
+    <xsl:call-template name="string-replace-all">
+      <xsl:with-param name="text">
+        <xsl:call-template name="string-replace-all">
+          <xsl:with-param name="text">
+            <xsl:call-template name="string-replace-all">
+              <xsl:with-param name="text">
+                <xsl:call-template name="string-replace-all">
+                  <xsl:with-param name="text">
+                    <xsl:call-template name="string-replace-all">
+                      <xsl:with-param name="text">
+                        <xsl:call-template name="string-replace-all">
+                          <xsl:with-param name="text">
+                            <xsl:call-template name="string-replace-all">
+                              <xsl:with-param name="text">
+                                <xsl:call-template name="string-replace-all">
+                                  <xsl:with-param name="text">
+                                    <xsl:call-template name="string-replace-all">
+                                      <xsl:with-param name="text">
+                                        <xsl:call-template name="string-replace-all">
+                                          <xsl:with-param name="text" select="$text" />
+                                          <xsl:with-param name="value"><xsl:value-of select="'\'" /></xsl:with-param>
+                                          <xsl:with-param name="replacement"><xsl:value-of select="'\\'" /></xsl:with-param>
+                                        </xsl:call-template>
+                                      </xsl:with-param>
+                                      <xsl:with-param name="value"><xsl:value-of select="'**'" /></xsl:with-param>
+                                      <xsl:with-param name="replacement"><xsl:value-of select="'\*\*'" /></xsl:with-param>
+                                    </xsl:call-template>
+                                  </xsl:with-param>
+                                  <xsl:with-param name="value"><xsl:value-of select="'//'" /></xsl:with-param>
+                                  <xsl:with-param name="replacement"><xsl:value-of select="'\/\/'" /></xsl:with-param>
+                                </xsl:call-template>
+                              </xsl:with-param>
+                              <xsl:with-param name="value"><xsl:value-of select="'_^'" /></xsl:with-param>
+                              <xsl:with-param name="replacement"><xsl:value-of select="'\_^'" /></xsl:with-param>
+                            </xsl:call-template>
+                          </xsl:with-param>
+                          <xsl:with-param name="value"><xsl:value-of select="'^_'" /></xsl:with-param>
+                          <xsl:with-param name="replacement"><xsl:value-of select="'\^_'" /></xsl:with-param>
+                        </xsl:call-template>
+                      </xsl:with-param>
+                      <xsl:with-param name="value"><xsl:value-of select="'^^'" /></xsl:with-param>
+                      <xsl:with-param name="replacement"><xsl:value-of select="'\^\^'" /></xsl:with-param>
+                  </xsl:call-template>
+                </xsl:with-param>
+                <xsl:with-param name="value"><xsl:value-of select="'!['" /></xsl:with-param>
+                <xsl:with-param name="replacement"><xsl:value-of select="'\!['" /></xsl:with-param>
+              </xsl:call-template>
+              </xsl:with-param>
+              <xsl:with-param name="value"><xsl:value-of select="']('" /></xsl:with-param>
+              <xsl:with-param name="replacement"><xsl:value-of select="'\]('" /></xsl:with-param>
+            </xsl:call-template>
+          </xsl:with-param>
+          <xsl:with-param name="value"><xsl:value-of select="'[['" /></xsl:with-param>
+          <xsl:with-param name="replacement"><xsl:value-of select="'\[\['" /></xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+      <xsl:with-param name="value"><xsl:value-of select="']]'" /></xsl:with-param>
+      <xsl:with-param name="replacement"><xsl:value-of select="'\]\]'" /></xsl:with-param>
+    </xsl:call-template>
+  </xsl:template>
+
   <!-- adds a backslash to the start of the value param, if necessary -->
-  <xsl:template name="escape">
+  <xsl:template name="escape-prefixes">
     <xsl:param name="value"/>
 
     <xsl:variable name="prefix" select="translate(substring($value, 1, 13), 'abcdefghijklmnopqrstuvwxyz', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ')" />
     <!-- '(' is considered special, so translate numbers into '(' so we can find and escape them -->
     <xsl:variable name="numprefix" select="translate(substring($value, 1, 3), '1234567890', '((((((((((')" />
 
-    <!-- p tags must escape initial content that looks like a block element marker -->
-    <xsl:if test="$prefix = 'BODY' or
-                  $prefix = 'PREAMBLE' or
-                  $prefix = 'PREFACE' or
-                  starts-with($prefix, 'CHAPTER ') or
-                  starts-with($prefix, 'PART ') or
-                  starts-with($prefix, 'SUBPART ') or
-                  starts-with($prefix, 'SCHEDULE ') or
-                  starts-with($prefix, 'HEADING ') or
-                  starts-with($prefix, 'SUBHEADING ') or
-                  starts-with($prefix, 'LONGTITLE ') or
-                  starts-with($prefix, 'CROSSHEADING ') or
-                  starts-with($prefix, '{|') or
-                  starts-with($numprefix, '(')">
-      <xsl:text>\</xsl:text>
-    </xsl:if>
-    <xsl:value-of select="$value"/>
+    <xsl:variable name="slash">
+      <!-- p tags must escape initial content that looks like a block element marker -->
+      <xsl:if test="$prefix = 'BODY' or
+                    $prefix = 'PREAMBLE' or
+                    $prefix = 'PREFACE' or
+                    starts-with($prefix, 'CHAPTER ') or
+                    starts-with($prefix, 'PART ') or
+                    starts-with($prefix, 'SUBPART ') or
+                    starts-with($prefix, 'SCHEDULE ') or
+                    starts-with($prefix, 'HEADING ') or
+                    starts-with($prefix, 'SUBHEADING ') or
+                    starts-with($prefix, 'LONGTITLE ') or
+                    starts-with($prefix, 'CROSSHEADING ') or
+                    starts-with($prefix, '{|') or
+                    starts-with($numprefix, '(')">
+        <xsl:value-of select="'\'" />
+      </xsl:if>
+    </xsl:variable>
+
+    <xsl:value-of select="concat($slash, $value)" />
+  </xsl:template>
+
+  <!-- adds a backslash to the start of the text param, if necessary -->
+  <xsl:template name="escape">
+    <xsl:param name="value"/>
+
+    <xsl:variable name="escaped">
+      <xsl:call-template name="escape-inlines">
+        <xsl:with-param name="text" select="$value" />
+      </xsl:call-template>
+    </xsl:variable>
+
+    <xsl:call-template name="escape-prefixes">
+      <xsl:with-param name="value" select="$escaped" />
+    </xsl:call-template>
   </xsl:template>
 
   <xsl:template match="a:act">
@@ -157,12 +267,19 @@
   </xsl:template>
 
   <!-- first text nodes of these elems must be escaped if they have special chars -->
-  <xsl:template match="a:p[not(ancestor::a:table)]/text()[1] | a:listIntroduction/text()[1] | a:intro/text()[1]">
+  <xsl:template match="a:p[not(ancestor::a:table)]/text()[not(preceding-sibling::*)] | a:listIntroduction/text()[not(preceding-sibling::*)] | a:intro/text()[not(preceding-sibling::*)]">
     <xsl:call-template name="escape">
       <xsl:with-param name="value" select="." />
     </xsl:call-template>
   </xsl:template>
 
+  <!-- escape inlines in text nodes -->
+  <xsl:template match="text()">
+    <xsl:call-template name="escape-inlines">
+      <xsl:with-param name="text" select="." />
+    </xsl:call-template>
+  </xsl:template>
+
 
   <!-- attachments/schedules -->
   <xsl:template match="a:attachment">
@@ -192,31 +309,24 @@
       <xsl:value-of select="." />
       <xsl:text>" </xsl:text>
     </xsl:for-each>
-    <xsl:text>
-|-</xsl:text>
+    <xsl:text>&#10;|-</xsl:text>
 
     <xsl:apply-templates />
-    <xsl:text>
-|}
-
-</xsl:text>
+    <xsl:text>&#10;|}&#10;&#10;</xsl:text>
   </xsl:template>
 
   <xsl:template match="a:tr">
     <xsl:apply-templates />
-    <xsl:text>
-|-</xsl:text>
+    <xsl:text>&#10;|-</xsl:text>
   </xsl:template>
 
   <xsl:template match="a:th|a:td">
     <xsl:choose>
       <xsl:when test="local-name(.) = 'th'">
-        <xsl:text>
-! </xsl:text>
+        <xsl:text>&#10;! </xsl:text>
       </xsl:when>
       <xsl:when test="local-name(.) = 'td'">
-        <xsl:text>
-| </xsl:text>
+        <xsl:text>&#10;| </xsl:text>
       </xsl:when>
     </xsl:choose>
 

data/lib/slaw/version.rb

@@ -1,3 +1,3 @@
 module Slaw
-  VERSION = "10.4.1"
+  VERSION = "10.5.0"
 end

data/spec/fixtures/roundtrip-escapes.txt

@@ -0,0 +1,20 @@
+BODY
+
+1. Section that tests escapes
+
+text \\ with a single slash
+
+some **inlines \/\/ [with \/\/ slashes](#foo)**
+
+inlines that \*\* should \/\/ be \[\[ escaped \![ and \]\]
+
+refs [https:\/\/example.com with ] and and \]( and **nested \*\* stars \*\***](#foo)
+
+nested ** stars \*\* in bold \*\***
+
+nested // slashes \/\/ in italics \/\///
+
+nested ** stars in // italics \*\* // and bold **
+
+super ^^with \^\^ hats \^\^^^ and sub _^\_^ with \^_ end tokens \^_^_
+

data/spec/generator_spec.rb

@@ -78,45 +78,81 @@ XML
 
 1. Section
 
-\Chapter 2 ignored
+\\Chapter 2 ignored
 
 Chapters
 
-\Part 2 ignored
+\\Part 2 ignored
 
 participation
 
-\Schedule 2 ignored
+\\Schedule 2 ignored
 
 Schedules
 
-\HEADING x
+\\HEADING x
 
-\SUBHEADING x
+\\SUBHEADING x
 
 BODY not escaped
 
-\BODY
+\\BODY
 
 PREAMBLE not escaped
 
-\PREAMBLE
+\\PREAMBLE
 
 PREFACE not escaped
 
-\PREFACE
+\\PREFACE
 
-\2. ignored
+\\2. ignored
 
-\2.1 ignored
+\\2.1 ignored
 
-\(2) ignored
+\\(2) ignored
 
-\(a) ignored
+\\(a) ignored
 
-\(2a) ignored
+\\(2a) ignored
 
-\{| ignored
+\\{| ignored
+
+'
+    end
+
+    it 'should escape inlines when unparsing' do
+      doc = xml2doc(section(<<'XML'
+        <num>1.</num>
+        <heading>Section</heading>
+        <paragraph id="section-1.paragraph-0">
+          <content>
+            <p>text \ with a single slash</p>
+            <p>some <b>inlines // <ref href="#foo">with // slashes</ref></b></p>
+            <p>inlines that ** should // be [[ escaped ![ and ]]</p>
+            <p>refs <ref href="#foo">https://example.com with ] and ]( and <b>nested **</b></ref></p>
+            <p>super <sup>with ^^</sup> and sub <sub>_^ with ^_</sub></p>
+          </content>
+        </paragraph>
+XML
+      ))
+
+      text = subject.text_from_act(doc)
+      # NOTE: in single quoted strings, backslash sequences aren't considered special, EXCEPT a double backslash
+      # which is actually a single backslash. So \\ needs to be \\\\ while \* is just \*. The mind boggles.
+      text.should == 'BODY
+
+1. Section
+
+text \\\\ with a single slash
+
+some **inlines \/\/ [with \/\/ slashes](#foo)**
+
+inlines that \*\* should \/\/ be \[\[ escaped \![ and \]\]
+
+refs [https:\/\/example.com with ] and \]( and **nested \*\***](#foo)
+
+super ^^with \^\^^^ and sub _^\_^ with \^_^_
 
 '
     end
@@ -148,7 +184,7 @@ XML
 
 1. Section
 
-\(2) A special meeting [[ foo ]]:
+\\(2) A special meeting [[ foo ]]:
 
 (a) the chairperson so directs; or
 
@@ -269,4 +305,13 @@ Subject to approval in terms of this By-Law.
 '
     end
   end
+
+  describe 'round trip' do
+    it 'should be idempotent for escapes' do
+      text = File.open('spec/fixtures/roundtrip-escapes.txt', 'r').read()
+      act = subject.generate_from_text(text)
+      xml = act.to_xml(encoding: 'utf-8')
+      subject.text_from_act(act).should == text
+    end
+  end
 end

data/spec/za/act_block_spec.rb

@@ -117,16 +117,19 @@ EOS
     it 'should handle escaped content' do
       node = parse :body, <<EOS
 \\1. ignored
+foo \\\\bar
 
 \\CROSSHEADING cross\\heading
 
 1. Sec\\tion
 \\Chapter 2 ignored
+Some text with a \\\\real backslash
 EOS
       to_xml(node).should == '<body>
   <hcontainer eId="hcontainer_1" name="hcontainer">
     <content>
       <p>1. ignored</p>
+      <p>foo \\bar</p>
       <p>CROSSHEADING crossheading</p>
     </content>
   </hcontainer>
@@ -136,6 +139,7 @@ EOS
     <hcontainer eId="sec_1__hcontainer_1" name="hcontainer">
       <content>
         <p>Chapter 2 ignored</p>
+        <p>Some text with a \\real backslash</p>
       </content>
     </hcontainer>
   </section>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slaw
 version: !ruby/object:Gem::Version
-  version: 10.4.1
+  version: 10.5.0
 platform: ruby
 authors:
 - Greg Kempe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-14 00:00:00.000000000 Z
+date: 2021-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -144,6 +144,7 @@ files:
 - spec/counters_spec.rb
 - spec/extract/extractor_spec.rb
 - spec/fixtures/community-fire-safety.xml
+- spec/fixtures/roundtrip-escapes.txt
 - spec/generator_spec.rb
 - spec/parse/blocklists_spec.rb
 - spec/parse/builder_spec.rb
@@ -182,6 +183,7 @@ test_files:
 - spec/counters_spec.rb
 - spec/extract/extractor_spec.rb
 - spec/fixtures/community-fire-safety.xml
+- spec/fixtures/roundtrip-escapes.txt
 - spec/generator_spec.rb
 - spec/parse/blocklists_spec.rb
 - spec/parse/builder_spec.rb