ruby-ll 1.1.3 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a058e52fdce3a69cfacb367d31f25d9322b09d3a
-  data.tar.gz: 66c890ff03310a459735f8c14d2a452f4415e550
+  metadata.gz: c5d66bcacfb96b5c530ea468c1e6f9cc74a3d916
+  data.tar.gz: 4eba99c6eb0c02fd4945cc2767ec5487cd93b7d2
 SHA512:
-  metadata.gz: b2635328c10c6b8a4c9570e997a2e9ecd0536fa767b03b62306f0eb9285c984e6b68b22bb1519164c1cef5c4a12cc3c2f3537f7aefc874464c2d4834b36edea2
-  data.tar.gz: 8dfcc7a62283212ff857fd2ff2b33b68f78b2ee095021975900ddbd55f7abf7fc7795cd9a4081c2109a8e72b17cd21d9924c93b7a29daf9d5fcf77844cfca109
+  metadata.gz: 885e11cf3cfcb3f42644fbd4d0e79e34dbb5c9939e0916728517ba283a07b61b0f0dfaf6734285e112aa499aab4928a64b1fc14eecc2460139658e958542a6ee
+  data.tar.gz: 1f15f04b4abd4e573e12334e1db627ca899e24d91be250c048294e5503212a7eca375133723108732f07df7f7c8243f8beb665400cbe5a18669995fa1faf0841

data/README.md

@@ -278,8 +278,30 @@ return specific elements from it:
     numbers = A B { val[0] };
 
 Values returned by code blocks are passed to whatever other rule called it. This
-allows code blocks to be used for building ASTs and the likes. If no explicit
-code block is defined `val` is returned as is.
+allows code blocks to be used for building ASTs and the likes.
+
+If no explicit code block is defined then ruby-ll will generate one for you. If
+a branch consists out of only a single step (e.g. `A = B;`) then only the first
+value is returned, otherwise all values are returned.
+
+This means that in the following example the output will be whatever value "C"
+contains:
+
+    A = B { p val[0] };
+    B = C;
+
+However, here the output would be `[C, D]` as the `B` rule's branch contains
+multiple steps:
+
+    A = B { p val[0] };
+    B = C D;
+
+To summarize (`# =>` denotes the return value):
+
+    A = B;   # => B
+    A = B C; # => [B, C]
+
+You can override this behaviour simply by defining your own code block.
 
 ruby-ll parsers recurse into rules before unwinding, this means that the
 inner-most rule is processed first.
@@ -301,6 +323,50 @@ name as a terminal, as such the following is invalid:
 
 It's also an error to re-define an existing rule.
 
+### Operators
+
+Grammars can use two operators to define a sequence of terminals/non-terminals:
+the star (`*`) and plus (`+`) operators.
+
+The star operator indicates that something should occur 0 or more times. Here
+the "B" identifier could occur 0 times, once, twice or many more times:
+
+    A = B*;
+
+The plus operator indicates that something should occur at least once followed
+by any number of more occurrences. For example, this grammar states that "B"
+should occur at least once but can also occur, say, 10 times:
+
+    A = B+;
+
+Operators can be applied either to a single terminal/rule or a series of
+terminals/rules grouped together using parenthesis. For example, both are
+perfectly valid:
+
+    A = B+;
+    A = (B C)+;
+
+When calling an operator on a single terminal/rule the corresponding entry in
+the `val` array is simply set to the terminal/rule value. For example:
+
+    A = B+ { p val[0] };
+
+For input `B B B` this would output `[B, B, B]`.
+
+However, when grouping multiple terminals/rules using parenthesis every
+occurrence is wrapped in an Array. For example:
+
+    A = (B C)+ { p val[0] };
+
+For input `B C B C` this would output `[[B, C], [B, C]]`. To work around this
+you can simply move the group of identifiers to its own rule and only return
+whatever you need:
+
+    A  = A1+ { p val[0] };
+    A1 = B C { val[0] }; # only return "B"
+
+For input `B C B C` this would output `[B, B]`.
+
 ## Conflicts
 
 LL(1) grammars can have two kinds of conflicts in a rule:

data/doc/driver_architecture.md

@@ -0,0 +1,32 @@
+# Driver Architecture
+
+The actual parsing of input is handled by a so called "driver" represented as
+the class `LL::Driver`. This class is written in either C or Java depending on
+the Ruby platform that's being used. The rationale for this is simple:
+performance. While Ruby is a great language it's sadly not fast enough to handle
+parsing of large inputs in a way that doesn't either require lots of memory,
+time or both.
+
+Both the C and Java drivers try to use native data structures as much as
+possible instead of using Ruby structures. For example, their internal parsing
+stacks are native stacks. In case of Java this is an ArrayDeque, in case of C
+this is a vector created using the [kvec][kvec] library as C doesn't have a
+native vector structure.
+
+The driver operates by iterating over every token supplied by the `each_token`
+method (this method must be defined by a parser itself). For every input token a
+callback function in C/Java is executed that determines what to parse and how to
+parse it.
+
+The parsing process largely operates on integers, only using Ruby objects where
+absolutely required. For example, all steps of a rule's branch are represented
+as integers. Lookup tables are also simply arrays of integers with terminals
+being mapped directly to the indexes of these arrays. See ruby-ll's own parser
+for examples. Note that the integers for the `rules` Array are in reverse order,
+so everything that comes first is processed last.
+
+For more information on the internals its best to refer to the C driver code
+located in `ext/c/driver.c`. The Java code is largely based on this code safe
+for some code comments here and there.
+
+[kvec]: https://github.com/attractivechaos/klib/blob/master/kvec.h

data/ext/c/driver.c

@@ -5,6 +5,10 @@
 #define T_TERMINAL 1
 #define T_EPSILON 2
 #define T_ACTION 3
+#define T_STAR 4
+#define T_PLUS 5
+#define T_ADD_VALUE_STACK 6
+#define T_APPEND_VALUE_STACK 7
 
 ID id_config_const;
 ID id_each_token;
@@ -66,6 +70,8 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
     VALUE method;
     VALUE action_args;
     VALUE action_retval;
+    VALUE operator_buffer;
+    VALUE last_value;
     long num_args;
     long args_i;
 
@@ -113,8 +119,8 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
             }
         }
 
-        /* Rule */
-        if ( stack_type == T_RULE )
+        /* A rule or the "+" operator */
+        if ( stack_type == T_RULE || stack_type == T_PLUS )
         {
             production_i = state->config->table[stack_value][token_id];
 
@@ -132,6 +138,19 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
             }
             else
             {
+                /*
+                Append a "*" operator for all following occurrences as they are
+                optional
+                */
+                if ( stack_type == T_PLUS )
+                {
+                    kv_push(long, state->stack, T_STAR);
+                    kv_push(long, state->stack, stack_value);
+
+                    kv_push(long, state->stack, T_APPEND_VALUE_STACK);
+                    kv_push(long, state->stack, 0);
+                }
+
                 FOR(rule_i, state->config->rule_lengths[production_i])
                 {
                     kv_push(
@@ -142,6 +161,56 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
                 }
             }
         }
+        /* "*" operator */
+        else if ( stack_type == T_STAR )
+        {
+            production_i = state->config->table[stack_value][token_id];
+
+            if ( production_i != T_EOF )
+            {
+                kv_push(long, state->stack, T_STAR);
+                kv_push(long, state->stack, stack_value);
+
+                kv_push(long, state->stack, T_APPEND_VALUE_STACK);
+                kv_push(long, state->stack, 0);
+
+                FOR(rule_i, state->config->rule_lengths[production_i])
+                {
+                    kv_push(
+                        long,
+                        state->stack,
+                        state->config->rules[production_i][rule_i]
+                    );
+                }
+            }
+        }
+        /*
+        Adds a new array to the value stack that can be used to group operator
+        values together
+        */
+        else if ( stack_type == T_ADD_VALUE_STACK )
+        {
+            operator_buffer = rb_ary_new();
+
+            kv_push(VALUE, state->value_stack, operator_buffer);
+
+            RB_GC_GUARD(operator_buffer);
+        }
+        /*
+        Appends the last value on the value stack to the operator buffer that
+        preceeds it.
+        */
+        else if ( stack_type == T_APPEND_VALUE_STACK )
+        {
+            last_value = kv_pop(state->value_stack);
+
+            operator_buffer = kv_A(
+                state->value_stack,
+                kv_size(state->value_stack) - 1
+            );
+
+            rb_ary_push(operator_buffer, last_value);
+        }
         /* Terminal */
         else if ( stack_type == T_TERMINAL )
         {

data/ext/c/driver_config.c

@@ -166,7 +166,7 @@ VALUE ll_driver_config_set_actions(VALUE self, VALUE array)
 
     Data_Get_Struct(self, DriverConfig, config);
 
-    config->action_names       = ALLOC_N(ID, row_count);
+    config->action_names       = ALLOC_N(VALUE, row_count);
     config->action_arg_amounts = ALLOC_N(long, row_count);
 
     FOR(rindex, row_count)

data/ext/c/driver_config.h

@@ -30,7 +30,7 @@ typedef struct
     long **table;
 
     /* Array containing action names as Symbols */
-    ID *action_names;
+    VALUE *action_names;
 
     /* Array containing the arity for every action */
     long *action_arg_amounts;

data/ext/java/org/libll/Driver.java

@@ -27,11 +27,15 @@ import org.jruby.runtime.builtin.IRubyObject;
 @JRubyClass(name="LL::Driver", parent="Object")
 public class Driver extends RubyObject
 {
-    private static long T_EOF      = -1;
-    private static long T_RULE     = 0;
-    private static long T_TERMINAL = 1;
-    private static long T_EPSILON  = 2;
-    private static long T_ACTION   = 3;
+    private static long T_EOF                = -1;
+    private static long T_RULE               = 0;
+    private static long T_TERMINAL           = 1;
+    private static long T_EPSILON            = 2;
+    private static long T_ACTION             = 3;
+    private static long T_STAR               = 4;
+    private static long T_PLUS               = 5;
+    private static long T_ADD_VALUE_STACK    = 6;
+    private static long T_APPEND_VALUE_STACK = 7;
 
     /**
      * The current Ruby runtime.
@@ -132,8 +136,8 @@ public class Driver extends RubyObject
                         token_id = self.config.terminals.get(type);
                     }
 
-                    // Rule
-                    if ( stack_type == self.T_RULE )
+                    // A rule or the "+" operator
+                    if ( stack_type == self.T_RULE || stack_type == self.T_PLUS )
                     {
                         Long production_i = self.config.table
                             .get(stack_value.intValue())
@@ -152,6 +156,17 @@ public class Driver extends RubyObject
                         }
                         else
                         {
+                            // Append a "*" operator for all following
+                            // occurrences as they are optional
+                            if ( stack_type == self.T_PLUS )
+                            {
+                                stack.push(self.T_STAR);
+                                stack.push(stack_value);
+
+                                stack.push(self.T_APPEND_VALUE_STACK);
+                                stack.push(Long.valueOf(0));
+                            }
+
                             ArrayList<Long> row = self.config.rules
                                 .get(production_i.intValue());
 
@@ -161,6 +176,47 @@ public class Driver extends RubyObject
                             }
                         }
                     }
+                    // "*" operator
+                    else if ( stack_type == self.T_STAR )
+                    {
+                        Long production_i = self.config.table
+                            .get(stack_value.intValue())
+                            .get(token_id.intValue());
+
+                        if ( production_i != self.T_EOF )
+                        {
+                            stack.push(self.T_STAR);
+                            stack.push(stack_value);
+
+                            stack.push(self.T_APPEND_VALUE_STACK);
+                            stack.push(Long.valueOf(0));
+
+                            ArrayList<Long> row = self.config.rules
+                                .get(production_i.intValue());
+
+                            for ( int index = 0; index < row.size(); index++ )
+                            {
+                                stack.push(row.get(index));
+                            }
+                        }
+                    }
+                    // Adds a new array to the value stack that can be used to
+                    // group operator values together
+                    else if ( stack_type == self.T_ADD_VALUE_STACK )
+                    {
+                        RubyArray operator_buffer = self.runtime.newArray();
+
+                        value_stack.push(operator_buffer);
+                    }
+                    // Appends the last value on the value stack to the operator
+                    // buffer that preceeds it.
+                    else if ( stack_type == self.T_APPEND_VALUE_STACK )
+                    {
+                        IRubyObject last_value    = value_stack.pop();
+                        RubyArray operator_buffer = (RubyArray) value_stack.peek();
+
+                        operator_buffer.append(last_value);
+                    }
                     // Terminal
                     else if ( stack_type == self.T_TERMINAL )
                     {

data/lib/ll.rb

@@ -18,6 +18,7 @@ require_relative 'll/rule'
 require_relative 'll/branch'
 require_relative 'll/terminal'
 require_relative 'll/epsilon'
+require_relative 'll/operator'
 require_relative 'll/message'
 require_relative 'll/ast/node'
 require_relative 'll/erb_context'

data/lib/ll/branch.rb

@@ -25,7 +25,13 @@ module LL
     def first_set
       first = steps[0]
 
-      return first.is_a?(Rule) ? first.first_set : [first]
+      if first.is_a?(Rule)
+        return first.first_set
+      elsif first
+        return [first]
+      else
+        return []
+      end
     end
 
     ##

data/lib/ll/configuration_compiler.rb

@@ -8,11 +8,15 @@ module LL
     # @return [Hash]
     #
     TYPES = {
-      :eof      => -1,
-      :rule     => 0,
-      :terminal => 1,
-      :epsilon  => 2,
-      :action   => 3
+      :eof                => -1,
+      :rule               => 0,
+      :terminal           => 1,
+      :epsilon            => 2,
+      :action             => 3,
+      :star               => 4,
+      :plus               => 5,
+      :add_value_stack    => 6,
+      :append_value_stack => 7
     }.freeze
 
     ##
@@ -105,7 +109,21 @@ module LL
 
       grammar.rules.each do |rule|
         rule.branches.each do |branch|
-          bodies[:"_rule_#{index}"] = branch.ruby_code || DEFAULT_RUBY_CODE
+          if branch.ruby_code
+            code = branch.ruby_code
+
+          # If a branch only contains a single, non-epsilon step we can just
+          # return that value as-is. This makes parsing code a little bit
+          # easier.
+          elsif !branch.ruby_code and branch.steps.length == 1 \
+          and !branch.steps[0].is_a?(Epsilon)
+            code = 'val[0]'
+
+          else
+            code = DEFAULT_RUBY_CODE
+          end
+
+          bodies[:"_rule_#{index}"] = code
 
           index += 1
         end
@@ -133,17 +151,24 @@ module LL
           action_index += 1
 
           branch.steps.reverse_each do |step|
-            if step.is_a?(LL::Terminal)
+            if step.is_a?(Terminal)
               row << TYPES[:terminal]
               row << term_indices[step] + 1
 
-            elsif step.is_a?(LL::Rule)
+            elsif step.is_a?(Rule)
               row << TYPES[:rule]
               row << rule_indices[step]
 
-            elsif step.is_a?(LL::Epsilon)
+            elsif step.is_a?(Epsilon)
               row << TYPES[:epsilon]
               row << 0
+
+            elsif step.is_a?(Operator)
+              row << TYPES[step.type]
+              row << rule_indices[step.receiver]
+
+              row << TYPES[:add_value_stack]
+              row << 0
             end
           end
 

data/lib/ll/driver.rb

@@ -30,6 +30,10 @@ module LL
         message  = "Unexpected #{token_type}, expected #{expected} instead"
       when :eof
         message = "Received #{token_type} but there's nothing left to parse"
+      when :star
+        message = %Q{Unexpected #{token_type} for a "*" operator}
+      when :plus
+        message = %Q{Unexpected #{token_type} for a "+" operator}
       end
 
       return message

data/lib/ll/grammar_compiler.rb

@@ -179,7 +179,7 @@ module LL
     ##
     # Processes the assignment of terminals.
     #
-    # @see #process
+    # @see [#process]
     #
     def on_terminals(node, compiled_grammar)
       node.children.each do |child|
@@ -199,7 +199,7 @@ module LL
     ##
     # Processes an %inner directive.
     #
-    # @see #process
+    # @see [#process]
     #
     def on_inner(node, compiled_grammar)
       compiled_grammar.inner = process(node.children[0], compiled_grammar)
@@ -208,7 +208,7 @@ module LL
     ##
     # Processes a %header directive.
     #
-    # @see #process
+    # @see [#process]
     #
     def on_header(node, compiled_grammar)
       compiled_grammar.header = process(node.children[0], compiled_grammar)
@@ -217,7 +217,7 @@ module LL
     ##
     # Processes a node containing Ruby source code.
     #
-    # @see #process
+    # @see [#process]
     # @return [String]
     #
     def on_ruby(node, compiled_grammar)
@@ -227,7 +227,7 @@ module LL
     ##
     # Extracts the name from an identifier.
     #
-    # @see #process
+    # @see [#process]
     # @return [String]
     #
     def on_ident(node, compiled_grammar)
@@ -237,7 +237,7 @@ module LL
     ##
     # Processes an epsilon.
     #
-    # @see #process
+    # @see [#process]
     # @return [LL::Epsilon]
     #
     def on_epsilon(node, compiled_grammar)
@@ -247,7 +247,7 @@ module LL
     ##
     # Processes the assignment of a rule.
     #
-    # @see #process
+    # @see [#process]
     #
     def on_rule(node, compiled_grammar)
       name = process(node.children[0], compiled_grammar)
@@ -280,7 +280,7 @@ module LL
     ##
     # Creates a basic prototype for a rule.
     #
-    # @see #process
+    # @see [#process]
     #
     def on_rule_prototype(node, compiled_grammar)
       name = process(node.children[0], compiled_grammar)
@@ -295,7 +295,7 @@ module LL
     ##
     # Processes a single rule branch.
     #
-    # @see #process
+    # @see [#process]
     # @return [LL::Branch]
     #
     def on_branch(node, compiled_grammar)
@@ -313,33 +313,53 @@ module LL
     ##
     # Processes the steps of a branch.
     #
-    # @see #process
+    # @see [#process]
     # @return [Array]
     #
     def on_steps(node, compiled_grammar)
-      steps = []
+      return lookup_identifiers(node, compiled_grammar)
+    end
 
-      node.children.each do |step_node|
-        retval = process(step_node, compiled_grammar)
+    ##
+    # Processes the "*" operator.
+    #
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    # @return [LL::Operator]
+    #
+    def on_star(node, compiled_grammar)
+      steps = lookup_identifiers(node, compiled_grammar)
+      name  = "_ll_star#{node.source_line.line}#{node.source_line.column}"
+      rule  = Rule.new(name, node.source_line)
 
-        # Literal rule/terminal names.
-        if retval.is_a?(String)
-          step = compiled_grammar.lookup_identifier(retval)
+      rule.add_branch(steps, node.source_line)
 
-          undefined_identifier!(retval, step_node, compiled_grammar) unless step
-        # Epsilon
-        else
-          step = retval
-        end
+      rule.increment_references
 
-        if step
-          step.increment_references if step.respond_to?(:increment_references)
+      compiled_grammar.add_rule(rule)
 
-          steps << step
-        end
-      end
+      return Operator.new(:star, rule, node.source_line)
+    end
+
+    ##
+    # Processes the "+" operator.
+    #
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    # @return [LL::Operator]
+    #
+    def on_plus(node, compiled_grammar)
+      steps = lookup_identifiers(node, compiled_grammar)
+      name  = "_ll_plus#{node.source_line.line}#{node.source_line.column}"
+      rule  = Rule.new(name, node.source_line)
+
+      rule.add_branch(steps, node.source_line)
 
-      return steps
+      rule.increment_references
+
+      compiled_grammar.add_rule(rule)
+
+      return Operator.new(:plus, rule, node.source_line)
     end
 
     private
@@ -355,5 +375,35 @@ module LL
         node.source_line
       )
     end
+
+    ##
+    # @see [#process]
+    # @return [Array]
+    #
+    def lookup_identifiers(node, compiled_grammar)
+      idents = []
+
+      node.children.each do |ident_node|
+        retval = process(ident_node, compiled_grammar)
+
+        # Literal rule/terminal names.
+        if retval.is_a?(String)
+          ident = compiled_grammar.lookup_identifier(retval)
+
+          undefined_identifier!(retval, ident_node, compiled_grammar) if !ident
+        # Epsilon
+        else
+          ident = retval
+        end
+
+        if ident
+          ident.increment_references if ident.respond_to?(:increment_references)
+
+          idents << ident
+        end
+      end
+
+      return idents
+    end
   end # Compiler
 end # LL

data/lib/ll/lexer.rb

@@ -64,49 +64,49 @@ self._ll_lexer_indicies = [
 	27, 1, 1, 1, 1, 1, 1, 1, 
 	1, 1, 1, 1, 1, 1, 1, 1, 
 	1, 1, 1, 1, 1, 1, 26, 1, 
-	1, 28, 1, 29, 1, 1, 1, 1, 
-	1, 1, 1, 1, 1, 1, 25, 25, 
+	1, 28, 1, 29, 1, 1, 30, 31, 
+	32, 33, 1, 1, 1, 1, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	30, 31, 1, 32, 1, 1, 1, 25, 
+	34, 35, 1, 36, 1, 1, 1, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	25, 1, 1, 1, 1, 33, 1, 25, 
+	25, 1, 1, 1, 1, 37, 1, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	25, 34, 35, 1, 1, 1, 25, 36, 
-	36, 36, 36, 36, 36, 36, 36, 36, 
-	36, 36, 36, 36, 36, 36, 36, 36, 
-	36, 36, 36, 36, 36, 36, 36, 36, 
-	36, 36, 36, 36, 36, 36, 36, 36, 
-	36, 36, 36, 36, 36, 36, 36, 36, 
-	36, 36, 36, 36, 36, 36, 36, 25, 
+	25, 38, 39, 1, 1, 1, 25, 40, 
+	40, 40, 40, 40, 40, 40, 40, 40, 
+	40, 40, 40, 40, 40, 40, 40, 40, 
+	40, 40, 40, 40, 40, 40, 40, 40, 
+	40, 40, 40, 40, 40, 40, 40, 40, 
+	40, 40, 40, 40, 40, 40, 40, 40, 
+	40, 40, 40, 40, 40, 40, 40, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	25, 36, 36, 36, 36, 36, 36, 36, 
+	25, 40, 40, 40, 40, 40, 40, 40, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	25, 25, 36, 36, 36, 36, 25, 36, 
+	25, 25, 40, 40, 40, 40, 25, 40, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
 	25, 25, 25, 25, 25, 25, 25, 25, 
-	25, 25, 36, 36, 36, 36, 36, 25, 
-	37, 28, 39, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 38, 38, 38, 38, 38, 
-	38, 38, 38, 40, 38, 41, 38, 0
+	25, 25, 40, 40, 40, 40, 40, 25, 
+	41, 28, 43, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 42, 42, 42, 42, 42, 
+	42, 42, 42, 44, 42, 45, 42, 0
 ]
 
 class << self
@@ -118,8 +118,8 @@ self._ll_lexer_trans_targs = [
 	6, 22, 8, 9, 10, 22, 12, 13, 
 	22, 15, 16, 17, 18, 19, 20, 21, 
 	22, 23, 22, 22, 24, 1, 22, 22, 
-	22, 23, 22, 22, 22, 22, 25, 25, 
-	25, 25
+	22, 22, 22, 22, 22, 23, 22, 22, 
+	22, 22, 25, 25, 25, 25
 ]
 
 class << self
@@ -132,7 +132,7 @@ self._ll_lexer_trans_actions = [
 	3, 0, 0, 0, 0, 0, 0, 0, 
 	4, 7, 8, 9, 0, 0, 10, 11, 
 	12, 13, 14, 15, 16, 17, 18, 19, 
-	20, 21
+	20, 21, 22, 23, 24, 25
 ]
 
 class << self
@@ -164,8 +164,8 @@ end
 self._ll_lexer_eof_trans = [
 	0, 0, 0, 0, 0, 0, 0, 0, 
 	0, 0, 0, 0, 0, 0, 0, 0, 
-	0, 0, 0, 0, 0, 0, 0, 37, 
-	38, 0
+	0, 0, 0, 0, 0, 0, 0, 41, 
+	42, 0
 ]
 
 class << self
@@ -312,13 +312,13 @@ ts = p
 	cs = _ll_lexer_trans_targs[_trans]
 	if _ll_lexer_trans_actions[_trans] != 0
 	case _ll_lexer_trans_actions[_trans]
-	when 20 then
+	when 24 then
 # line 188 "lib/ll/lexer.rl"
 		begin
 te = p+1
  begin  brace_count += 1  end
 		end
-	when 21 then
+	when 25 then
 # line 190 "lib/ll/lexer.rl"
 		begin
 te = p+1
@@ -338,7 +338,7 @@ te = p+1
           end
          end
 		end
-	when 18 then
+	when 22 then
 # line 206 "lib/ll/lexer.rl"
 		begin
 te = p+1
@@ -367,33 +367,57 @@ te = p+1
 te = p+1
  begin  emit(:T_HEADER, ts, te)  end
 		end
-	when 12 then
+	when 16 then
 # line 220 "lib/ll/lexer.rl"
 		begin
 te = p+1
  begin  emit(:T_EQUALS, ts, te)  end
 		end
-	when 10 then
+	when 14 then
 # line 221 "lib/ll/lexer.rl"
 		begin
 te = p+1
  begin  emit(:T_COLON, ts, te)  end
 		end
-	when 11 then
+	when 15 then
 # line 222 "lib/ll/lexer.rl"
 		begin
 te = p+1
  begin  emit(:T_SEMICOLON, ts, te)  end
 		end
-	when 15 then
+	when 19 then
 # line 223 "lib/ll/lexer.rl"
 		begin
 te = p+1
  begin  emit(:T_PIPE, ts, te)  end
 		end
-	when 14 then
+	when 13 then
+# line 225 "lib/ll/lexer.rl"
+		begin
+te = p+1
+ begin  emit(:T_PLUS, ts, te)  end
+		end
+	when 12 then
 # line 226 "lib/ll/lexer.rl"
 		begin
+te = p+1
+ begin  emit(:T_STAR, ts, te)  end
+		end
+	when 10 then
+# line 227 "lib/ll/lexer.rl"
+		begin
+te = p+1
+ begin  emit(:T_LPAREN, ts, te)  end
+		end
+	when 11 then
+# line 228 "lib/ll/lexer.rl"
+		begin
+te = p+1
+ begin  emit(:T_RPAREN, ts, te)  end
+		end
+	when 18 then
+# line 230 "lib/ll/lexer.rl"
+		begin
 te = p+1
  begin 
           mark        = ts + 1
@@ -405,26 +429,26 @@ te = p+1
           cs = 25;
          end
 		end
-	when 17 then
+	when 21 then
 # line 213 "lib/ll/lexer.rl"
 		begin
 te = p
 p = p - 1;		end
-	when 16 then
+	when 20 then
 # line 1 "NONE"
 		begin
 	case act
 	when 16 then
 	begin begin p = ((te))-1; end
  emit(:T_EPSILON, ts, te) end
-	when 18 then
+	when 22 then
 	begin begin p = ((te))-1; end
 
         emit(:T_IDENT, ts, te)
       end
 end 
 			end
-	when 19 then
+	when 23 then
 # line 148 "lib/ll/lexer.rl"
 		begin
 
@@ -458,7 +482,7 @@ te = p+1
 		begin
 te = p+1
 		end
-	when 13 then
+	when 17 then
 # line 1 "NONE"
 		begin
 te = p+1
@@ -473,8 +497,8 @@ te = p+1
 		end
 # line 168 "lib/ll/lexer.rl"
 		begin
-act = 18;		end
-# line 477 "lib/ll/lexer.rb"
+act = 22;		end
+# line 501 "lib/ll/lexer.rb"
 	end
 	end
 	end
@@ -484,7 +508,7 @@ act = 18;		end
 # line 1 "NONE"
 		begin
 ts = nil;		end
-# line 487 "lib/ll/lexer.rb"
+# line 511 "lib/ll/lexer.rb"
 	end
 
 	if cs == 0
@@ -576,7 +600,7 @@ end
     end
 
     
-# line 238 "lib/ll/lexer.rl"
+# line 242 "lib/ll/lexer.rl"
 
   end # Lexer
 end # Oga

data/lib/ll/operator.rb

@@ -0,0 +1,26 @@
+module LL
+  ##
+  # Class for operators such as + and *.
+  #
+  class Operator
+    attr_reader :type, :receiver, :source_line
+
+    ##
+    # @param [Symbol] type
+    # @param [LL::Rule] receiver
+    # @param [LL::SourceLine] source_line
+    #
+    def initialize(type, receiver, source_line)
+      @type        = type
+      @receiver    = receiver
+      @source_line = source_line
+    end
+
+    ##
+    # @return [String]
+    #
+    def inspect
+      return "Operator(type: #{type.inspect}, receiver: #{receiver.inspect})"
+    end
+  end # Operator
+end # LL

data/lib/ll/parser.rb

@@ -19,6 +19,10 @@ class Parser < LL::Driver
     :T_PIPE, # 9
     :T_EPSILON, # 10
     :T_SEMICOLON, # 11
+    :T_STAR, # 12
+    :T_PLUS, # 13
+    :T_LPAREN, # 14
+    :T_RPAREN, # 15
   ].freeze
 
   CONFIG.rules = [
@@ -26,7 +30,7 @@ class Parser < LL::Driver
     [3, 1, 2, 0], # 1
     [3, 2, 0, 1, 0, 2], # 2
     [3, 3, 2, 0], # 3
-    [3, 4, 0, 18], # 4
+    [3, 4, 0, 23], # 4
     [3, 5, 0, 3], # 5
     [3, 6, 0, 5], # 6
     [3, 7, 0, 6], # 7
@@ -35,45 +39,59 @@ class Parser < LL::Driver
     [3, 10, 0, 4, 0, 10, 1, 8, 1, 8], # 10
     [3, 11, 2, 0], # 11
     [3, 12, 1, 11, 0, 8, 1, 3], # 12
-    [3, 13, 0, 15, 1, 4], # 13
-    [3, 14, 0, 15, 1, 5], # 14
+    [3, 13, 0, 20, 1, 4], # 13
+    [3, 14, 0, 20, 1, 5], # 14
     [3, 15, 0, 9, 0, 10], # 15
     [3, 16, 0, 8], # 16
     [3, 17, 2, 0], # 17
     [3, 18, 1, 6], # 18
-    [3, 19, 0, 8], # 19
-    [3, 20, 0, 12], # 20
-    [3, 21, 1, 10], # 21
-    [3, 22, 0, 14, 0, 11], # 22
-    [3, 23, 0, 15], # 23
-    [3, 24, 2, 0], # 24
-    [3, 25, 1, 1], # 25
-    [3, 26, 0, 17, 0, 13], # 26
-    [3, 27, 0, 16, 1, 9], # 27
-    [3, 28, 2, 0], # 28
-    [3, 29, 1, 11, 0, 16, 1, 7, 0, 10], # 29
+    [3, 19, 0, 12, 0, 13], # 19
+    [3, 20, 0, 15, 1, 15, 0, 8, 1, 14], # 20
+    [3, 21, 0, 11], # 21
+    [3, 22, 2, 0], # 22
+    [3, 23, 0, 14, 0, 10], # 23
+    [3, 24, 0, 15], # 24
+    [3, 25, 2, 0], # 25
+    [3, 26, 1, 13], # 26
+    [3, 27, 1, 12], # 27
+    [3, 28, 0, 11], # 28
+    [3, 29, 0, 17], # 29
+    [3, 30, 1, 10], # 30
+    [3, 31, 0, 19, 0, 16], # 31
+    [3, 32, 0, 20], # 32
+    [3, 33, 2, 0], # 33
+    [3, 34, 1, 1], # 34
+    [3, 35, 0, 22, 0, 18], # 35
+    [3, 36, 0, 21, 1, 9], # 36
+    [3, 37, 2, 0], # 37
+    [3, 38, 1, 11, 0, 21, 1, 7, 0, 10], # 38
   ].freeze
 
   CONFIG.table = [
-    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], # 0
-    [3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3], # 1
-    [-1, -1, 5, 6, 7, 8, 4, -1, -1, -1, -1, -1], # 2
-    [-1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 3
-    [11, 11, 11, 11, 11, 11, 11, 11, 10, 11, 11, 11], # 4
-    [-1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1], # 5
-    [-1, -1, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1], # 6
-    [-1, -1, -1, -1, -1, 14, -1, -1, -1, -1, -1, -1], # 7
-    [-1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1], # 8
-    [17, 17, 17, 17, 17, 17, 16, 17, 17, 17, 17, 17], # 9
-    [-1, -1, -1, -1, -1, -1, 18, -1, -1, -1, -1, -1], # 10
-    [-1, -1, -1, -1, -1, -1, 19, -1, -1, -1, 20, -1], # 11
-    [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 21, -1], # 12
-    [-1, -1, -1, -1, -1, -1, 22, -1, -1, -1, 22, -1], # 13
-    [24, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24], # 14
-    [-1, 25, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 15
-    [-1, -1, -1, -1, -1, -1, 26, -1, -1, -1, 26, -1], # 16
-    [28, 28, 28, 28, 28, 28, 28, 28, 28, 27, 28, 28], # 17
-    [-1, -1, -1, -1, -1, -1, 29, -1, -1, -1, -1, -1], # 18
+    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], # 0
+    [3, 3, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3], # 1
+    [-1, -1, 5, 6, 7, 8, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 2
+    [-1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 3
+    [11, 11, 11, 11, 11, 11, 11, 11, 10, 11, 11, 11, 11, 11, 11, 11], # 4
+    [-1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 5
+    [-1, -1, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 6
+    [-1, -1, -1, -1, -1, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 7
+    [-1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 8
+    [17, 17, 17, 17, 17, 17, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17], # 9
+    [-1, -1, -1, -1, -1, -1, 18, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 10
+    [-1, -1, -1, -1, -1, -1, 19, -1, -1, -1, -1, -1, -1, -1, 20, -1], # 11
+    [22, 22, 22, 22, 22, 22, 21, 22, 22, 22, 22, 22, 22, 22, 21, 22], # 12
+    [-1, -1, -1, -1, -1, -1, 23, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 13
+    [25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 24, 24, 25, 25], # 14
+    [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, 26, -1, -1], # 15
+    [-1, -1, -1, -1, -1, -1, 28, -1, -1, -1, 29, -1, -1, -1, 28, -1], # 16
+    [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 30, -1, -1, -1, -1, -1], # 17
+    [-1, -1, -1, -1, -1, -1, 31, -1, -1, -1, 31, -1, -1, -1, 31, -1], # 18
+    [33, 32, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33], # 19
+    [-1, 34, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 20
+    [-1, -1, -1, -1, -1, -1, 35, -1, -1, -1, 35, -1, -1, -1, 35, -1], # 21
+    [37, 37, 37, 37, 37, 37, 37, 37, 37, 36, 37, 37, 37, 37, 37, 37], # 22
+    [-1, -1, -1, -1, -1, -1, 38, -1, -1, -1, -1, -1, -1, -1, -1, -1], # 23
   ].freeze
 
   CONFIG.actions = [
@@ -96,17 +114,26 @@ class Parser < LL::Driver
     [:_rule_16, 1], # 16
     [:_rule_17, 0], # 17
     [:_rule_18, 1], # 18
-    [:_rule_19, 1], # 19
-    [:_rule_20, 1], # 20
+    [:_rule_19, 2], # 19
+    [:_rule_20, 4], # 20
     [:_rule_21, 1], # 21
-    [:_rule_22, 2], # 22
-    [:_rule_23, 1], # 23
-    [:_rule_24, 0], # 24
-    [:_rule_25, 1], # 25
-    [:_rule_26, 2], # 26
-    [:_rule_27, 2], # 27
-    [:_rule_28, 0], # 28
-    [:_rule_29, 4], # 29
+    [:_rule_22, 0], # 22
+    [:_rule_23, 2], # 23
+    [:_rule_24, 1], # 24
+    [:_rule_25, 0], # 25
+    [:_rule_26, 1], # 26
+    [:_rule_27, 1], # 27
+    [:_rule_28, 1], # 28
+    [:_rule_29, 1], # 29
+    [:_rule_30, 1], # 30
+    [:_rule_31, 2], # 31
+    [:_rule_32, 1], # 32
+    [:_rule_33, 0], # 33
+    [:_rule_34, 1], # 34
+    [:_rule_35, 2], # 35
+    [:_rule_36, 2], # 36
+    [:_rule_37, 0], # 37
+    [:_rule_38, 4], # 38
   ].freeze
 
   ##
@@ -158,7 +185,7 @@ class Parser < LL::Driver
   end
 
   def _rule_2(val)
-     val[0] + val[1] 
+     [val[0], *val[1]] 
   end
 
   def _rule_3(val)
@@ -166,23 +193,23 @@ class Parser < LL::Driver
   end
 
   def _rule_4(val)
-    val
+    val[0]
   end
 
   def _rule_5(val)
-    val
+    val[0]
   end
 
   def _rule_6(val)
-    val
+    val[0]
   end
 
   def _rule_7(val)
-    val
+    val[0]
   end
 
   def _rule_8(val)
-    val
+    val[0]
   end
 
   def _rule_9(val)
@@ -192,7 +219,7 @@ class Parser < LL::Driver
   end
 
   def _rule_10(val)
-     [val[2]] + val[3] 
+     [val[2], *val[3]] 
   end
 
   def _rule_11(val)
@@ -218,7 +245,7 @@ class Parser < LL::Driver
   end
 
   def _rule_15(val)
-     [val[0]] + val[1] 
+     [val[0], *val[1]] 
   end
 
   def _rule_16(val)
@@ -236,22 +263,62 @@ class Parser < LL::Driver
   end
 
   def _rule_19(val)
+     [val[0], *val[1]] 
+  end
+
+  def _rule_20(val)
+    
+      [s(val[3][0], val[1], :source_line => val[0].source_line)]
+    
+  end
+
+  def _rule_21(val)
+     val[0] 
+  end
+
+  def _rule_22(val)
+    val
+  end
+
+  def _rule_23(val)
+    
+      val[1] ? s(val[1][0], [val[0]], :source_line => val[1][1]) : val[0]
+    
+  end
+
+  def _rule_24(val)
+     val[0] 
+  end
+
+  def _rule_25(val)
+     nil 
+  end
+
+  def _rule_26(val)
+     [:plus, val[0].source_line] 
+  end
+
+  def _rule_27(val)
+     [:star, val[0].source_line] 
+  end
+
+  def _rule_28(val)
     
       s(:steps, val[0], :source_line => val[0][0].source_line)
     
   end
 
-  def _rule_20(val)
+  def _rule_29(val)
     
       s(:steps, [val[0]], :source_line => val[0].source_line)
     
   end
 
-  def _rule_21(val)
+  def _rule_30(val)
      s(:epsilon, [], :source_line => val[0].source_line) 
   end
 
-  def _rule_22(val)
+  def _rule_31(val)
     
       steps = [val[0]]
 
@@ -261,31 +328,31 @@ class Parser < LL::Driver
     
   end
 
-  def _rule_23(val)
+  def _rule_32(val)
      val[0] 
   end
 
-  def _rule_24(val)
+  def _rule_33(val)
      nil 
   end
 
-  def _rule_25(val)
+  def _rule_34(val)
      s(:ruby, [val[0].value], :source_line => val[0].source_line) 
   end
 
-  def _rule_26(val)
-     [val[0]] + val[1] 
+  def _rule_35(val)
+     [val[0], *val[1]] 
   end
 
-  def _rule_27(val)
+  def _rule_36(val)
      val[1] 
   end
 
-  def _rule_28(val)
+  def _rule_37(val)
     val
   end
 
-  def _rule_29(val)
+  def _rule_38(val)
     
       s(:rule, [val[0], *val[2]], :source_line => val[0].source_line)
     

data/lib/ll/version.rb

@@ -1,3 +1,3 @@
 module LL
-  VERSION = '1.1.3'
+  VERSION = '2.0.0'
 end # LL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-ll
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 2.0.0
 platform: ruby
 authors:
 - Yorick Peterse
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-17 00:00:00.000000000 Z
+date: 2015-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
@@ -150,6 +150,7 @@ files:
 - bin/ruby-ll
 - doc/DCO.md
 - doc/css/common.css
+- doc/driver_architecture.md
 - ext/c/driver.c
 - ext/c/driver.h
 - ext/c/driver_config.c
@@ -180,6 +181,7 @@ files:
 - lib/ll/grammar_compiler.rb
 - lib/ll/lexer.rb
 - lib/ll/message.rb
+- lib/ll/operator.rb
 - lib/ll/parser.rb
 - lib/ll/parser_error.rb
 - lib/ll/rule.rb