bel_parser 1.0.0.alpha.38 → 1.0.0.alpha.39

data/lib/bel_parser/parsers/common/string.rl

@@ -5,45 +5,102 @@
 
   include 'common.rl';
 
-  action start_string {
-    @buffers[:string] = []
+  action string_end {
+    trace('STRING string_end')
+    completed = @string_opened && @string_closed
+    chars = data[p_start...p_end]
+    ast_node = string(utf8_string(chars), complete: true)
+    @buffers[:string] = ast_node
   }
 
-  action append_string {
-    (@buffers[:string] ||= []) << fc
+  action yield_string {
+    trace('STRING yield_string')
+    yield @buffers[:string]
   }
 
-  action finish_string {
-    @buffers[:string] = string(utf8_string(@buffers[:string]))
+  action start_string {
+    trace('STRING start_string')
+    @string_opened = true
+    p_start = p
   }
 
-  action error_string {
-    @buffers[:string] ||= []
-    @buffers[:string] = string(utf8_string(@buffers[:string]).sub(/\n$/, ''))
+  action stop_string {
+    trace('STRING stop_string')
+    @string_closed = true
+    p_end = p
   }
 
-  action yield_complete_string {
-    yield @buffers[:string]
+  action eof_string {
+    trace('STRING eof_string')
+    p_end = p
+    chars = data[p_start...p_end]
+    ast_node = string(utf8_string(chars), complete: false)
+    @buffers[:string] = ast_node
   }
 
-  action yield_error_string {
-    @buffers[:string] ||= []
+  action string_node_eof {
+    trace('STRING string_node_eof')
     yield @buffers[:string]
   }
 
-  STRING  =
-    ('"' ('\\\"' | [^"])** '"') >start_string $append_string %finish_string $err(error_string);
+  action a_string_err {
+    trace('STRING a_string_err')
+    p_end = p
+    chars = data[p_start...p_end]
+    ast_node = string(utf8_string(chars), complete: false)
+    @buffers[:string] = ast_node
+  }
+
+  action string_node_err {
+    trace('STRING string_node_err')
+    p_end = p
+    chars = data[p_start...p_end]
+    ast_node = string(utf8_string(chars), complete: false)
+    yield ast_node
+  }
 
-  string :=
-    STRING $err(yield_error_string) %yield_complete_string
-    NL;
+  #single =
+  #  WS* SQ (NOT_SQESC | ESCAPED)*
+  #  >start_string
+  #  $eof(eof_string)
+  #  %stop_string
+  #  %string_end
+  #  SQ
+  #  ;
+
+  double =
+    WS* DQ (NOT_DQESC | ESCAPED)*
+    >start_string
+    $eof(eof_string)
+    %stop_string
+    %string_end
+    DQ
+    ;
+
+  a_string =
+    #(single | double)
+    double
+    $err(a_string_err)
+    ;
+
+  string_node :=
+    (
+      #single |
+      double
+    )
+    $err(string_node_err)
+    @eof(string_node_eof)
+    @yield_string
+    ;
 }%%
 =end
 # end: ragel
+# ('\"' | ^(0 .. 31 | 34))* ^'\\"'
 
 require_relative '../ast/node'
 require_relative '../mixin/buffer'
 require_relative '../nonblocking_io_wrapper'
+require_relative '../tracer'
 
 module BELParser
   module Parsers
@@ -69,25 +126,31 @@ module BELParser
           include Enumerable
           include BELParser::Parsers::Buffer
           include BELParser::Parsers::AST::Sexp
+          include BELParser::Parsers::Tracer
 
           def initialize(content)
             @content = content
-      # begin: ragel        
+      # begin: ragel
             %% write data;
-      # end: ragel        
+      # end: ragel
           end
 
           def each
-            @buffers = {}
-            data     = @content.unpack('C*')
-            p        = 0
-            pe       = data.length
-            eof      = data.length
-
-      # begin: ragel        
+            @buffers        = {}
+            @incomplete     = {}
+            @string_opened  = false
+            @string_closed  = false
+            data            = @content.unpack('C*')
+            p_start         = 0
+            p_end           = 0
+            p               = 0
+            pe              = data.length
+            eof             = data.length
+
+      # begin: ragel
             %% write init;
             %% write exec;
-      # end: ragel        
+      # end: ragel
           end
         end
       end

data/lib/bel_parser/parsers/expression/comment.rb

@@ -9,6 +9,7 @@
 require_relative '../ast/node'
 require_relative '../mixin/buffer'
 require_relative '../nonblocking_io_wrapper'
+require_relative '../tracer'
 
 module BELParser
   module Parsers
@@ -34,17 +35,18 @@ module BELParser
           include Enumerable
           include BELParser::Parsers::Buffer
           include BELParser::Parsers::AST::Sexp
+          include BELParser::Parsers::Tracer
 
           def initialize(content)
             @content = content
-      # begin: ragel        
+      # begin: ragel
             
 class << self
 	attr_accessor :_bel_trans_keys
 	private :_bel_trans_keys, :_bel_trans_keys=
 end
 self._bel_trans_keys = [
-	0, 0, 10, 47, 47, 47, 
+	0, 0, 47, 47, 47, 47, 
 	10, 10, 10, 10, 0, 
 	0, 0
 ]
@@ -54,7 +56,7 @@ class << self
 	private :_bel_key_spans, :_bel_key_spans=
 end
 self._bel_key_spans = [
-	0, 38, 1, 1, 1, 0
+	0, 1, 1, 1, 1, 0
 ]
 
 class << self
@@ -62,7 +64,7 @@ class << self
 	private :_bel_index_offsets, :_bel_index_offsets=
 end
 self._bel_index_offsets = [
-	0, 0, 39, 41, 43, 45
+	0, 0, 2, 4, 6, 8
 ]
 
 class << self
@@ -70,12 +72,8 @@ class << self
 	private :_bel_indicies, :_bel_indicies=
 end
 self._bel_indicies = [
-	0, 1, 1, 1, 1, 1, 1, 1, 
-	1, 1, 1, 1, 1, 1, 1, 1, 
-	1, 1, 1, 1, 1, 1, 1, 1, 
-	1, 1, 1, 1, 1, 1, 1, 1, 
-	1, 1, 1, 1, 1, 2, 1, 3, 
-	1, 1, 4, 6, 5, 1, 0
+	0, 1, 2, 1, 1, 3, 5, 4, 
+	1, 0
 ]
 
 class << self
@@ -83,7 +81,7 @@ class << self
 	private :_bel_trans_targs, :_bel_trans_targs=
 end
 self._bel_trans_targs = [
-	5, 0, 2, 3, 4, 4, 5
+	2, 0, 3, 4, 4, 5
 ]
 
 class << self
@@ -91,7 +89,15 @@ class << self
 	private :_bel_trans_actions, :_bel_trans_actions=
 end
 self._bel_trans_actions = [
-	1, 0, 0, 0, 2, 3, 4
+	0, 0, 0, 1, 0, 2
+]
+
+class << self
+	attr_accessor :_bel_eof_actions
+	private :_bel_eof_actions, :_bel_eof_actions=
+end
+self._bel_eof_actions = [
+	0, 0, 0, 0, 2, 0
 ]
 
 class << self
@@ -101,29 +107,33 @@ self.bel_start = 1;
 class << self
 	attr_accessor :bel_first_final
 end
-self.bel_first_final = 5;
+self.bel_first_final = 4;
 class << self
 	attr_accessor :bel_error
 end
 self.bel_error = 0;
 
 class << self
-	attr_accessor :bel_en_comment
+	attr_accessor :bel_en_comment_node
 end
-self.bel_en_comment = 1;
+self.bel_en_comment_node = 1;
 
 
-      # end: ragel        
+      # end: ragel
           end
 
           def each
-            @buffers = {}
-            stack    = []
-            data     = @content.unpack('C*')
-            p        = 0
-            pe       = data.length
-
-      # begin: ragel        
+            @buffers    = {}
+            @incomplete = {}
+            stack       = []
+            data        = @content.unpack('C*')
+            p           = 0
+            p_start     = 0
+            p_end       = 0
+            pe          = data.length
+            eof         = data.length
+
+      # begin: ragel
             
 begin
 	p ||= 0
@@ -168,33 +178,30 @@ begin
 	cs = _bel_trans_targs[_trans]
 	if _bel_trans_actions[_trans] != 0
 	case _bel_trans_actions[_trans]
-	when 3 then
-		begin
-
-    @buffers[:comment] << data[p].ord
-  		end
 	when 1 then
 		begin
 
-    yield @buffers[:comment] || comment(nil)
+    trace('COMMENT start_comment')
+    p_start = p;
   		end
 	when 2 then
 		begin
 
-    @buffers[:comment] = []
-  		end
-		begin
-
-    @buffers[:comment] << data[p].ord
+    trace('COMMENT stop_comment')
+    p_end = p;
   		end
-	when 4 then
 		begin
 
-    @buffers[:comment] = comment(
-                           utf8_string(@buffers[:comment]))
+    trace('COMMENT comment_end')
+    p_end = p
+    chars = data[p_start...p_end]
+    completed = !chars.empty?
+    ast_node = comment(utf8_string(chars), complete: completed)
+    @buffers[:comment] = ast_node
   		end
 		begin
 
+    trace('COMMENT yield_comment')
     yield @buffers[:comment] || comment(nil)
   		end
 	end
@@ -212,6 +219,31 @@ begin
 	end
 	end
 	if _goto_level <= _test_eof
+	if p == eof
+	  case _bel_eof_actions[cs]
+	when 2 then
+		begin
+
+    trace('COMMENT stop_comment')
+    p_end = p;
+  		end
+		begin
+
+    trace('COMMENT comment_end')
+    p_end = p
+    chars = data[p_start...p_end]
+    completed = !chars.empty?
+    ast_node = comment(utf8_string(chars), complete: completed)
+    @buffers[:comment] = ast_node
+  		end
+		begin
+
+    trace('COMMENT yield_comment')
+    yield @buffers[:comment] || comment(nil)
+  		end
+	  end
+	end
+
 	end
 	if _goto_level <= _out
 		break
@@ -219,7 +251,7 @@ begin
 end
 	end
 
-      # end: ragel        
+      # end: ragel
           end
         end
       end

data/lib/bel_parser/parsers/expression/comment.rl

@@ -6,27 +6,38 @@
   include 'common.rl';
 
   action start_comment {
-    @buffers[:comment] = []
+    trace('COMMENT start_comment')
+    p_start = p;
   }
 
-  action append_comment {
-    @buffers[:comment] << fc
+  action stop_comment {
+    trace('COMMENT stop_comment')
+    p_end = p;
   }
 
-  action finish_comment {
-    @buffers[:comment] = comment(
-                           utf8_string(@buffers[:comment]))
+  action comment_end {
+    trace('COMMENT comment_end')
+    p_end = p
+    chars = data[p_start...p_end]
+    completed = !chars.empty?
+    ast_node = comment(utf8_string(chars), complete: completed)
+    @buffers[:comment] = ast_node
   }
 
   action yield_comment {
+    trace('COMMENT yield_comment')
     yield @buffers[:comment] || comment(nil)
   }
 
-  COMMENT = '//' ^NL+ >start_comment $append_comment %finish_comment;
+  comment =
+    SS
+    ^NL+ >start_comment %stop_comment %comment_end
+    ;
 
-  comment :=
-    COMMENT? %yield_comment
-    NL;
+  comment_node :=
+    comment %yield_comment
+    NL?
+    ;
 }%%
 =end
 # end: ragel
@@ -34,6 +45,7 @@
 require_relative '../ast/node'
 require_relative '../mixin/buffer'
 require_relative '../nonblocking_io_wrapper'
+require_relative '../tracer'
 
 module BELParser
   module Parsers
@@ -59,25 +71,30 @@ module BELParser
           include Enumerable
           include BELParser::Parsers::Buffer
           include BELParser::Parsers::AST::Sexp
+          include BELParser::Parsers::Tracer
 
           def initialize(content)
             @content = content
-      # begin: ragel        
+      # begin: ragel
             %% write data;
-      # end: ragel        
+      # end: ragel
           end
 
           def each
-            @buffers = {}
-            stack    = []
-            data     = @content.unpack('C*')
-            p        = 0
-            pe       = data.length
-
-      # begin: ragel        
+            @buffers    = {}
+            @incomplete = {}
+            stack       = []
+            data        = @content.unpack('C*')
+            p           = 0
+            p_start     = 0
+            p_end       = 0
+            pe          = data.length
+            eof         = data.length
+
+      # begin: ragel
             %% write init;
             %% write exec;
-      # end: ragel        
+      # end: ragel
           end
         end
       end