treetop 1.6.6 → 1.6.11

data/Treetop.tmbundle/info.plist

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>name</key>
+	<string>Treetop</string>
+	<key>uuid</key>
+	<string>83A8B700-143D-4BD6-B4EA-D73796E8F883</string>
+</dict>
+</plist>

data/doc/pitfalls_and_advanced_techniques.markdown

@@ -49,3 +49,9 @@ This says that `'end'` must be followed by a space, but this space is not consum
     end
     
 In general, when the syntax gets tough, it helps to focus on what you really mean. A keyword is a character not followed by another character that isn't a space.
+
+## Poor Performance with Large Unicode Strings
+
+Treetop may perform poorly when parsing very large (more than 100KB) unicode strings. This is due to the fact that substring lookups on Ruby unicode strings are linear-time operations, and not constant-time operations like they are on ASCII encoded strings. This means that parse times for larger strings can be exponentially worse than for smaller strings.
+
+If your input and grammar only expect ASCII strings, you can achieve significant performance improvements for large strings by re-encoding them to ASCII using `input.encode(Encoding::US_ASCII)`. See [this issue on GitHub](https://github.com/cjheath/treetop/issues/31) for more information and other possible workarounds for unicode strings.

data/doc/syntactic_recognition.markdown

@@ -32,9 +32,9 @@ The main keywords are:
 
 * `rule` : This defines a parsing rule within the grammar. It is followed by a name by which this rule can be referenced within other rules. It is then followed by a parsing expression defining the rule.
 
-A grammar may be surrounded by one or more nested `module` statements, which provides a namespace for the generated Ruby parser.
+A grammar may be surrounded by one or more nested `module` or `class` statements, which provides a namespace for the generated Ruby parser. Note that you cannot specify a superclass for a class, so if your class has a superclass, it must be declared elsewhere and loaded first.
 
-Treetop will emit a module called `GrammarName` and a parser class called `GrammarNameParser` (in the module namespace, if specified).
+Treetop will emit a module called `GrammarName` and a parser class called `GrammarNameParser` (in the namespace, if specified).
 
 #Parsing Expressions
 Each rule associates a name with a _parsing expression_. Parsing expressions are a generalization of vanilla regular expressions. Their key feature is the ability to reference other expressions in the grammar by name.

data/lib/treetop/compiler/grammar_compiler.rb

@@ -33,10 +33,13 @@ module Treetop
 
   # compile a treetop source file and load it
   def self.load(path)
-    adjusted_path = path =~ /\.(treetop|tt)\Z/ ? path : path + '.treetop'
-    File.open(adjusted_path) do |source_file|
+    unless path =~ Treetop::Polyglot::VALID_GRAMMAR_EXT_REGEXP
+      ext = Treetop::Polyglot::VALID_GRAMMAR_EXT.select {|ext| File.exist?(path+".#{ext}")}.shift
+      path += ".#{ext}" unless ext.nil?
+    end
+    File.open(path) do |source_file|
       source = source_file.read
-      source.gsub!(/\b__FILE__\b/, %Q{"#{adjusted_path}"})
+      source.gsub!(/\b__FILE__\b/, %Q{"#{path}"})
       load_from_string(source)
     end
   end

data/lib/treetop/compiler/metagrammar.rb

@@ -320,102 +320,122 @@ module Treetop
 
         i0, s0 = index, []
         i1, s1 = index, []
+        i2 = index
         if (match_len = has_terminal?('module', false, index))
-          r2 = instantiate_node(SyntaxNode,input, index...(index + match_len))
+          r3 = instantiate_node(SyntaxNode,input, index...(index + match_len))
           @index += match_len
         else
           terminal_parse_failure('\'module\'')
-          r2 = nil
+          r3 = nil
+        end
+        if r3
+          r3 = SyntaxNode.new(input, (index-1)...index) if r3 == true
+          r2 = r3
+        else
+          if (match_len = has_terminal?('class', false, index))
+            r4 = instantiate_node(SyntaxNode,input, index...(index + match_len))
+            @index += match_len
+          else
+            terminal_parse_failure('\'class\'')
+            r4 = nil
+          end
+          if r4
+            r4 = SyntaxNode.new(input, (index-1)...index) if r4 == true
+            r2 = r4
+          else
+            @index = i2
+            r2 = nil
+          end
         end
         s1 << r2
         if r2
-          r3 = _nt_space
-          s1 << r3
-          if r3
-            i4, s4 = index, []
+          r5 = _nt_space
+          s1 << r5
+          if r5
+            i6, s6 = index, []
             if has_terminal?(@regexps[gr = '\A[A-Z]'] ||= Regexp.new(gr), :regexp, index)
-              r5 = true
+              r7 = true
               @index += 1
             else
               terminal_parse_failure('[A-Z]')
-              r5 = nil
+              r7 = nil
             end
-            s4 << r5
-            if r5
-              s6, i6 = [], index
+            s6 << r7
+            if r7
+              s8, i8 = [], index
               loop do
-                r7 = _nt_alphanumeric_char
-                if r7
-                  s6 << r7
+                r9 = _nt_alphanumeric_char
+                if r9
+                  s8 << r9
                 else
                   break
                 end
               end
-              r6 = instantiate_node(SyntaxNode,input, i6...index, s6)
-              s4 << r6
-              if r6
-                s8, i8 = [], index
+              r8 = instantiate_node(SyntaxNode,input, i8...index, s8)
+              s6 << r8
+              if r8
+                s10, i10 = [], index
                 loop do
-                  i9, s9 = index, []
+                  i11, s11 = index, []
                   if (match_len = has_terminal?('::', false, index))
-                    r10 = instantiate_node(SyntaxNode,input, index...(index + match_len))
+                    r12 = instantiate_node(SyntaxNode,input, index...(index + match_len))
                     @index += match_len
                   else
                     terminal_parse_failure('\'::\'')
-                    r10 = nil
+                    r12 = nil
                   end
-                  s9 << r10
-                  if r10
+                  s11 << r12
+                  if r12
                     if has_terminal?(@regexps[gr = '\A[A-Z]'] ||= Regexp.new(gr), :regexp, index)
-                      r11 = true
+                      r13 = true
                       @index += 1
                     else
                       terminal_parse_failure('[A-Z]')
-                      r11 = nil
+                      r13 = nil
                     end
-                    s9 << r11
-                    if r11
-                      s12, i12 = [], index
+                    s11 << r13
+                    if r13
+                      s14, i14 = [], index
                       loop do
-                        r13 = _nt_alphanumeric_char
-                        if r13
-                          s12 << r13
+                        r15 = _nt_alphanumeric_char
+                        if r15
+                          s14 << r15
                         else
                           break
                         end
                       end
-                      r12 = instantiate_node(SyntaxNode,input, i12...index, s12)
-                      s9 << r12
+                      r14 = instantiate_node(SyntaxNode,input, i14...index, s14)
+                      s11 << r14
                     end
                   end
-                  if s9.last
-                    r9 = instantiate_node(SyntaxNode,input, i9...index, s9)
-                    r9.extend(ModuleDeclaration0)
+                  if s11.last
+                    r11 = instantiate_node(SyntaxNode,input, i11...index, s11)
+                    r11.extend(ModuleDeclaration0)
                   else
-                    @index = i9
-                    r9 = nil
+                    @index = i11
+                    r11 = nil
                   end
-                  if r9
-                    s8 << r9
+                  if r11
+                    s10 << r11
                   else
                     break
                   end
                 end
-                r8 = instantiate_node(SyntaxNode,input, i8...index, s8)
-                s4 << r8
+                r10 = instantiate_node(SyntaxNode,input, i10...index, s10)
+                s6 << r10
               end
             end
-            if s4.last
-              r4 = instantiate_node(SyntaxNode,input, i4...index, s4)
-              r4.extend(ModuleDeclaration1)
+            if s6.last
+              r6 = instantiate_node(SyntaxNode,input, i6...index, s6)
+              r6.extend(ModuleDeclaration1)
             else
-              @index = i4
-              r4 = nil
+              @index = i6
+              r6 = nil
             end
-            s1 << r4
-            if r4
-              r14 = _nt_space
-              s1 << r14
+            s1 << r6
+            if r6
+              r16 = _nt_space
+              s1 << r16
             end
           end
         end
@@ -428,44 +448,44 @@ module Treetop
         end
         s0 << r1
         if r1
-          i15 = index
-          r16 = _nt_module_declaration
-          if r16
-            r16 = SyntaxNode.new(input, (index-1)...index) if r16 == true
-            r15 = r16
+          i17 = index
+          r18 = _nt_module_declaration
+          if r18
+            r18 = SyntaxNode.new(input, (index-1)...index) if r18 == true
+            r17 = r18
           else
-            r17 = _nt_grammar
-            if r17
-              r17 = SyntaxNode.new(input, (index-1)...index) if r17 == true
-              r15 = r17
+            r19 = _nt_grammar
+            if r19
+              r19 = SyntaxNode.new(input, (index-1)...index) if r19 == true
+              r17 = r19
             else
-              @index = i15
-              r15 = nil
+              @index = i17
+              r17 = nil
             end
           end
-          s0 << r15
-          if r15
-            i18, s18 = index, []
-            r19 = _nt_space
-            s18 << r19
-            if r19
+          s0 << r17
+          if r17
+            i20, s20 = index, []
+            r21 = _nt_space
+            s20 << r21
+            if r21
               if (match_len = has_terminal?('end', false, index))
-                r20 = instantiate_node(SyntaxNode,input, index...(index + match_len))
+                r22 = instantiate_node(SyntaxNode,input, index...(index + match_len))
                 @index += match_len
               else
                 terminal_parse_failure('\'end\'')
-                r20 = nil
+                r22 = nil
               end
-              s18 << r20
+              s20 << r22
             end
-            if s18.last
-              r18 = instantiate_node(SyntaxNode,input, i18...index, s18)
-              r18.extend(ModuleDeclaration3)
+            if s20.last
+              r20 = instantiate_node(SyntaxNode,input, i20...index, s20)
+              r20.extend(ModuleDeclaration3)
             else
-              @index = i18
-              r18 = nil
+              @index = i20
+              r20 = nil
             end
-            s0 << r18
+            s0 << r20
           end
         end
         if s0.last

data/lib/treetop/compiler/metagrammar.treetop

@@ -18,7 +18,7 @@ module Treetop
       end
 
       rule module_declaration
-        module_prefix:('module' space name:([A-Z] alphanumeric_char* ('::' [A-Z] alphanumeric_char*)*) space) module_contents:(module_declaration / grammar) suffix:(space 'end') {
+        module_prefix:(('module'/'class') space name:([A-Z] alphanumeric_char* ('::' [A-Z] alphanumeric_char*)*) space) module_contents:(module_declaration / grammar) suffix:(space 'end') {
           def compile
             module_prefix.text_value + module_contents.compile + suffix.text_value
           end

data/lib/treetop/compiler/ruby_builder.rb

@@ -9,11 +9,11 @@ module Treetop
       def initialize
         @level = 0
         @address_space = LexicalAddressSpace.new
-        @ruby = ""
+        @ruby = String.new("")
       end
       
       def <<(ruby_line)              
-        return if ruby_line.blank?
+        return if ruby_line == ''
         ruby << ruby_line.tabto(level) << "\n"
       end
 

data/lib/treetop/ruby_extensions/string.rb

@@ -13,12 +13,6 @@ class String
     self[0...index].count("\n") + 1
   end
 
-  unless method_defined?(:blank?)
-    def blank?
-      self == ""
-    end
-  end
-
   # The following methods are lifted from Facets 2.0.2
   def tabto(n)
     if self =~ /^( *)\S/

data/lib/treetop/runtime/compiled_parser.rb

@@ -52,10 +52,10 @@ module Treetop
       end
 
       def terminal_failures
-        if @terminal_failures.empty? || @terminal_failures[0].is_a?(TerminalParseFailure)
+        if @terminal_failures.empty? || @terminal_failures[-1].is_a?(TerminalParseFailure)
           @terminal_failures
         else
-          @terminal_failures.map! {|tf_ary| TerminalParseFailure.new(*tf_ary) }
+          @terminal_failures.map! {|tf_ary| tf_ary.is_a?(TerminalParseFailure) ? tf_ary : TerminalParseFailure.new(*tf_ary) }
         end
       end
 
@@ -75,6 +75,11 @@ module Treetop
         @max_terminal_failure_index = 0
       end
 
+      def forget_failures_to_here
+        @terminal_failures = []
+        @max_terminal_failure_index = -1
+      end
+
       def reset_index
         @index = 0
       end
@@ -113,12 +118,20 @@ module Treetop
       end
 
       def terminal_parse_failure(expected_string, unexpected = false)
+        if @max_terminal_failure_index == -1
+          @max_terminal_failure_index = 0
+          return nil
+        end
         return nil if index < max_terminal_failure_index
         if index > max_terminal_failure_index
           @max_terminal_failure_index = index
           @terminal_failures = []
         end
         @terminal_failures << [index, expected_string, unexpected]
+        # It's very slow, but this shows the last 5 nested rules:
+        # caller.reject{|l| l =~ /`loop'|`block in /}[0..5].reverse.map{|l| l.sub(/[^`]*`_nt_/,'').sub(/'/,'')}
+
+        terminal_failures
         return nil
       end
     end

data/lib/treetop/version.rb

@@ -2,7 +2,7 @@ module Treetop #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 1
     MINOR = 6
-    TINY  = 6
+    TINY  = 11
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/treetop.gemspec

@@ -1,166 +1,34 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: treetop 1.6.6 ruby lib
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'date'
+require 'treetop/version'
 
-Gem::Specification.new do |s|
-  s.name = "treetop"
-  s.version = "1.6.6"
+Gem::Specification.new do |spec|
+  spec.name = "treetop"
+  spec.version = Treetop::VERSION::STRING
+  spec.authors = ["Nathan Sobo", "Clifford Heath"]
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.require_paths = ["lib"]
-  s.authors = ["Nathan Sobo", "Clifford Heath"]
-  s.date = "2016-07-20"
-  s.description = "A Parsing Expression Grammar (PEG) Parser generator DSL for Ruby"
-  s.email = "cliffordheath@gmail.com"
-  s.executables = ["tt"]
-  s.extra_rdoc_files = [
+  spec.email = "cliffordheath@gmail.com"
+  spec.date = Date.today.strftime("%F")
+  spec.summary = "A Ruby-based text parsing and interpretation DSL"
+  spec.description = "A Parsing Expression Grammar (PEG) Parser generator DSL for Ruby"
+  spec.homepage = "https://github.com/cjheath/treetop"
+  spec.licenses = ["MIT"]
+
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|website|script|\.|benchmark)}) }
+  spec.executables = ["tt"]
+  spec.require_paths = ["lib"]
+  spec.extra_rdoc_files = [
     "LICENSE",
     "README.md"
   ]
-  s.files = [
-    "LICENSE",
-    "README.md",
-    "Rakefile",
-    "bin/tt",
-    "doc/contributing_and_planned_features.markdown",
-    "doc/grammar_composition.markdown",
-    "doc/index.markdown",
-    "doc/pitfalls_and_advanced_techniques.markdown",
-    "doc/semantic_interpretation.markdown",
-    "doc/site.rb",
-    "doc/sitegen.rb",
-    "doc/syntactic_recognition.markdown",
-    "doc/tt.1",
-    "doc/using_in_ruby.markdown",
-    "examples/indented_blocks/indented_blocks.tt",
-    "examples/indented_blocks/indented_blocks_test.rb",
-    "examples/lambda_calculus/arithmetic.rb",
-    "examples/lambda_calculus/arithmetic.treetop",
-    "examples/lambda_calculus/arithmetic_node_classes.rb",
-    "examples/lambda_calculus/arithmetic_test.rb",
-    "examples/lambda_calculus/lambda_calculus.rb",
-    "examples/lambda_calculus/lambda_calculus.treetop",
-    "examples/lambda_calculus/lambda_calculus_node_classes.rb",
-    "examples/lambda_calculus/lambda_calculus_test.rb",
-    "examples/lambda_calculus/test_helper.rb",
-    "lib/treetop.rb",
-    "lib/treetop/bootstrap_gen_1_metagrammar.rb",
-    "lib/treetop/compiler.rb",
-    "lib/treetop/compiler/grammar_compiler.rb",
-    "lib/treetop/compiler/lexical_address_space.rb",
-    "lib/treetop/compiler/metagrammar.rb",
-    "lib/treetop/compiler/metagrammar.treetop",
-    "lib/treetop/compiler/node_classes.rb",
-    "lib/treetop/compiler/node_classes/anything_symbol.rb",
-    "lib/treetop/compiler/node_classes/atomic_expression.rb",
-    "lib/treetop/compiler/node_classes/character_class.rb",
-    "lib/treetop/compiler/node_classes/choice.rb",
-    "lib/treetop/compiler/node_classes/declaration_sequence.rb",
-    "lib/treetop/compiler/node_classes/grammar.rb",
-    "lib/treetop/compiler/node_classes/inline_module.rb",
-    "lib/treetop/compiler/node_classes/nonterminal.rb",
-    "lib/treetop/compiler/node_classes/optional.rb",
-    "lib/treetop/compiler/node_classes/parenthesized_expression.rb",
-    "lib/treetop/compiler/node_classes/parsing_expression.rb",
-    "lib/treetop/compiler/node_classes/parsing_rule.rb",
-    "lib/treetop/compiler/node_classes/predicate.rb",
-    "lib/treetop/compiler/node_classes/predicate_block.rb",
-    "lib/treetop/compiler/node_classes/repetition.rb",
-    "lib/treetop/compiler/node_classes/sequence.rb",
-    "lib/treetop/compiler/node_classes/terminal.rb",
-    "lib/treetop/compiler/node_classes/transient_prefix.rb",
-    "lib/treetop/compiler/node_classes/treetop_file.rb",
-    "lib/treetop/compiler/ruby_builder.rb",
-    "lib/treetop/polyglot.rb",
-    "lib/treetop/ruby_extensions.rb",
-    "lib/treetop/ruby_extensions/string.rb",
-    "lib/treetop/runtime.rb",
-    "lib/treetop/runtime/compiled_parser.rb",
-    "lib/treetop/runtime/interval_skip_list.rb",
-    "lib/treetop/runtime/interval_skip_list/head_node.rb",
-    "lib/treetop/runtime/interval_skip_list/interval_skip_list.rb",
-    "lib/treetop/runtime/interval_skip_list/node.rb",
-    "lib/treetop/runtime/syntax_node.rb",
-    "lib/treetop/runtime/terminal_parse_failure.rb",
-    "lib/treetop/runtime/terminal_syntax_node.rb",
-    "lib/treetop/version.rb",
-    "spec/compiler/and_predicate_spec.rb",
-    "spec/compiler/anything_symbol_spec.rb",
-    "spec/compiler/character_class_spec.rb",
-    "spec/compiler/choice_spec.rb",
-    "spec/compiler/circular_compilation_spec.rb",
-    "spec/compiler/failure_propagation_functional_spec.rb",
-    "spec/compiler/grammar_compiler_spec.rb",
-    "spec/compiler/grammar_spec.rb",
-    "spec/compiler/multibyte_chars_spec.rb",
-    "spec/compiler/namespace_spec.rb",
-    "spec/compiler/nonterminal_symbol_spec.rb",
-    "spec/compiler/not_predicate_spec.rb",
-    "spec/compiler/occurrence_range_spec.rb",
-    "spec/compiler/one_or_more_spec.rb",
-    "spec/compiler/optional_spec.rb",
-    "spec/compiler/parenthesized_expression_spec.rb",
-    "spec/compiler/parsing_rule_spec.rb",
-    "spec/compiler/repeated_subrule_spec.rb",
-    "spec/compiler/semantic_predicate_spec.rb",
-    "spec/compiler/sequence_spec.rb",
-    "spec/compiler/terminal_spec.rb",
-    "spec/compiler/terminal_symbol_spec.rb",
-    "spec/compiler/test_grammar.treetop",
-    "spec/compiler/test_grammar.tt",
-    "spec/compiler/test_grammar_do.treetop",
-    "spec/compiler/test_grammar_magic_coding.treetop",
-    "spec/compiler/test_grammar_magic_encoding.treetop",
-    "spec/compiler/tt_compiler_spec.rb",
-    "spec/compiler/zero_or_more_spec.rb",
-    "spec/composition/a.treetop",
-    "spec/composition/b.treetop",
-    "spec/composition/c.treetop",
-    "spec/composition/d.treetop",
-    "spec/composition/f.treetop",
-    "spec/composition/grammar_composition_spec.rb",
-    "spec/composition/subfolder/e_includes_c.treetop",
-    "spec/ruby_extensions/string_spec.rb",
-    "spec/runtime/compiled_parser_spec.rb",
-    "spec/runtime/syntax_node_spec.rb",
-    "spec/spec_helper.rb",
-    "treetop.gemspec"
-  ]
-  s.homepage = "https://github.com/cjheath/treetop"
-  s.licenses = ["MIT"]
-  s.rubygems_version = "2.4.5"
-  s.summary = "A Ruby-based text parsing and interpretation DSL"
-
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<polyglot>, ["~> 0.3"])
-      s.add_development_dependency(%q<jeweler>, ["~> 2.0"])
-      s.add_development_dependency(%q<activesupport>, ["~> 5"])
-      s.add_development_dependency(%q<i18n>, ["~> 0.6"])
-      s.add_development_dependency(%q<rr>, ["~> 1.0"])
-      s.add_development_dependency(%q<rspec>, ["~> 3"])
-      s.add_development_dependency(%q<rake>, ["~> 11"])
-    else
-      s.add_dependency(%q<polyglot>, ["~> 0.3"])
-      s.add_dependency(%q<jeweler>, ["~> 2.0"])
-      s.add_dependency(%q<activesupport>, ["~> 5"])
-      s.add_dependency(%q<i18n>, ["~> 0.6"])
-      s.add_dependency(%q<rr>, ["~> 1.0"])
-      s.add_dependency(%q<rspec>, ["~> 3"])
-      s.add_dependency(%q<rake>, ["~> 11"])
-    end
-  else
-    s.add_dependency(%q<polyglot>, ["~> 0.3"])
-    s.add_dependency(%q<jeweler>, ["~> 2.0"])
-    s.add_dependency(%q<activesupport>, ["~> 5"])
-    s.add_dependency(%q<i18n>, ["~> 0.6"])
-    s.add_dependency(%q<rr>, ["~> 1.0"])
-    s.add_dependency(%q<rspec>, ["~> 3"])
-    s.add_dependency(%q<rake>, ["~> 11"])
-  end
+  spec.add_runtime_dependency(%q<polyglot>, ["~> 0.3"])
+  spec.add_development_dependency(%q<activesupport>, ["~> 4"])
+  spec.add_development_dependency(%q<i18n>, ["~> 0.6"])
+  spec.add_development_dependency(%q<rr>, ["~> 1.0"])
+  spec.add_development_dependency(%q<rspec>, ["~> 3"])
+  spec.add_development_dependency(%q<rake>, ["~> 11"])
 end