ruby-ll 1.1.3-java → 2.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5a9dc1d9c35dced64f28eaf2340cd576d990ad5
-  data.tar.gz: 97acb8a53b364b77ef58e4e4060b21366efff1de
+  metadata.gz: 0f307cb5b874f4f151092083942884b5077e5fab
+  data.tar.gz: 1a9dbbcd10bd6898a709aa5759d5f91030272e8f
 SHA512:
-  metadata.gz: 5a2a4f2b163adf8d50072570cd955b412c810d27b1f0a27f5c3f9a138132b012ea6059df8d337e788844194f0450f1661833029c06cdbd42899afcec8a2c260d
-  data.tar.gz: 531c4be54e3d700126c339bd44177c7fcdb1fc2b51c03d135de9fd1d1c6c8162a7628b2a3a9bccabff5159add4a8ab82298acd3032424107f990b7da0d078e63
+  metadata.gz: d397ac3dac1792bb4310508aaec7ee448cfe0f09f9fa1072daa5c53f1b8ea290c2ff342897e8edfcced314d5b9fa0715970dccadea8ac0c9f0b23be1dced959b
+  data.tar.gz: 76771a39c8fdebf2e64f5b0535a019c22ac5aff0907b96f2a8d91ec5d5f054f742aa3fc75dd847cb1c050c1e1a60c79269308a2eb6aa596cd9923479a21b3665

data/README.md

@@ -278,8 +278,30 @@ return specific elements from it:
     numbers = A B { val[0] };
 
 Values returned by code blocks are passed to whatever other rule called it. This
-allows code blocks to be used for building ASTs and the likes. If no explicit
-code block is defined `val` is returned as is.
+allows code blocks to be used for building ASTs and the likes.
+
+If no explicit code block is defined then ruby-ll will generate one for you. If
+a branch consists out of only a single step (e.g. `A = B;`) then only the first
+value is returned, otherwise all values are returned.
+
+This means that in the following example the output will be whatever value "C"
+contains:
+
+    A = B { p val[0] };
+    B = C;
+
+However, here the output would be `[C, D]` as the `B` rule's branch contains
+multiple steps:
+
+    A = B { p val[0] };
+    B = C D;
+
+To summarize (`# =>` denotes the return value):
+
+    A = B;   # => B
+    A = B C; # => [B, C]
+
+You can override this behaviour simply by defining your own code block.
 
 ruby-ll parsers recurse into rules before unwinding, this means that the
 inner-most rule is processed first.
@@ -301,6 +323,50 @@ name as a terminal, as such the following is invalid:
 
 It's also an error to re-define an existing rule.
 
+### Operators
+
+Grammars can use two operators to define a sequence of terminals/non-terminals:
+the star (`*`) and plus (`+`) operators.
+
+The star operator indicates that something should occur 0 or more times. Here
+the "B" identifier could occur 0 times, once, twice or many more times:
+
+    A = B*;
+
+The plus operator indicates that something should occur at least once followed
+by any number of more occurrences. For example, this grammar states that "B"
+should occur at least once but can also occur, say, 10 times:
+
+    A = B+;
+
+Operators can be applied either to a single terminal/rule or a series of
+terminals/rules grouped together using parenthesis. For example, both are
+perfectly valid:
+
+    A = B+;
+    A = (B C)+;
+
+When calling an operator on a single terminal/rule the corresponding entry in
+the `val` array is simply set to the terminal/rule value. For example:
+
+    A = B+ { p val[0] };
+
+For input `B B B` this would output `[B, B, B]`.
+
+However, when grouping multiple terminals/rules using parenthesis every
+occurrence is wrapped in an Array. For example:
+
+    A = (B C)+ { p val[0] };
+
+For input `B C B C` this would output `[[B, C], [B, C]]`. To work around this
+you can simply move the group of identifiers to its own rule and only return
+whatever you need:
+
+    A  = A1+ { p val[0] };
+    A1 = B C { val[0] }; # only return "B"
+
+For input `B C B C` this would output `[B, B]`.
+
 ## Conflicts
 
 LL(1) grammars can have two kinds of conflicts in a rule:

data/doc/driver_architecture.md

@@ -0,0 +1,32 @@
+# Driver Architecture
+
+The actual parsing of input is handled by a so called "driver" represented as
+the class `LL::Driver`. This class is written in either C or Java depending on
+the Ruby platform that's being used. The rationale for this is simple:
+performance. While Ruby is a great language it's sadly not fast enough to handle
+parsing of large inputs in a way that doesn't either require lots of memory,
+time or both.
+
+Both the C and Java drivers try to use native data structures as much as
+possible instead of using Ruby structures. For example, their internal parsing
+stacks are native stacks. In case of Java this is an ArrayDeque, in case of C
+this is a vector created using the [kvec][kvec] library as C doesn't have a
+native vector structure.
+
+The driver operates by iterating over every token supplied by the `each_token`
+method (this method must be defined by a parser itself). For every input token a
+callback function in C/Java is executed that determines what to parse and how to
+parse it.
+
+The parsing process largely operates on integers, only using Ruby objects where
+absolutely required. For example, all steps of a rule's branch are represented
+as integers. Lookup tables are also simply arrays of integers with terminals
+being mapped directly to the indexes of these arrays. See ruby-ll's own parser
+for examples. Note that the integers for the `rules` Array are in reverse order,
+so everything that comes first is processed last.
+
+For more information on the internals its best to refer to the C driver code
+located in `ext/c/driver.c`. The Java code is largely based on this code safe
+for some code comments here and there.
+
+[kvec]: https://github.com/attractivechaos/klib/blob/master/kvec.h

data/ext/c/driver.c

@@ -5,6 +5,10 @@
 #define T_TERMINAL 1
 #define T_EPSILON 2
 #define T_ACTION 3
+#define T_STAR 4
+#define T_PLUS 5
+#define T_ADD_VALUE_STACK 6
+#define T_APPEND_VALUE_STACK 7
 
 ID id_config_const;
 ID id_each_token;
@@ -66,6 +70,8 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
     VALUE method;
     VALUE action_args;
     VALUE action_retval;
+    VALUE operator_buffer;
+    VALUE last_value;
     long num_args;
     long args_i;
 
@@ -113,8 +119,8 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
             }
         }
 
-        /* Rule */
-        if ( stack_type == T_RULE )
+        /* A rule or the "+" operator */
+        if ( stack_type == T_RULE || stack_type == T_PLUS )
         {
             production_i = state->config->table[stack_value][token_id];
 
@@ -132,6 +138,19 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
             }
             else
             {
+                /*
+                Append a "*" operator for all following occurrences as they are
+                optional
+                */
+                if ( stack_type == T_PLUS )
+                {
+                    kv_push(long, state->stack, T_STAR);
+                    kv_push(long, state->stack, stack_value);
+
+                    kv_push(long, state->stack, T_APPEND_VALUE_STACK);
+                    kv_push(long, state->stack, 0);
+                }
+
                 FOR(rule_i, state->config->rule_lengths[production_i])
                 {
                     kv_push(
@@ -142,6 +161,56 @@ VALUE ll_driver_each_token(VALUE token, VALUE self)
                 }
             }
         }
+        /* "*" operator */
+        else if ( stack_type == T_STAR )
+        {
+            production_i = state->config->table[stack_value][token_id];
+
+            if ( production_i != T_EOF )
+            {
+                kv_push(long, state->stack, T_STAR);
+                kv_push(long, state->stack, stack_value);
+
+                kv_push(long, state->stack, T_APPEND_VALUE_STACK);
+                kv_push(long, state->stack, 0);
+
+                FOR(rule_i, state->config->rule_lengths[production_i])
+                {
+                    kv_push(
+                        long,
+                        state->stack,
+                        state->config->rules[production_i][rule_i]
+                    );
+                }
+            }
+        }
+        /*
+        Adds a new array to the value stack that can be used to group operator
+        values together
+        */
+        else if ( stack_type == T_ADD_VALUE_STACK )
+        {
+            operator_buffer = rb_ary_new();
+
+            kv_push(VALUE, state->value_stack, operator_buffer);
+
+            RB_GC_GUARD(operator_buffer);
+        }
+        /*
+        Appends the last value on the value stack to the operator buffer that
+        preceeds it.
+        */
+        else if ( stack_type == T_APPEND_VALUE_STACK )
+        {
+            last_value = kv_pop(state->value_stack);
+
+            operator_buffer = kv_A(
+                state->value_stack,
+                kv_size(state->value_stack) - 1
+            );
+
+            rb_ary_push(operator_buffer, last_value);
+        }
         /* Terminal */
         else if ( stack_type == T_TERMINAL )
         {

data/ext/c/driver_config.c

@@ -166,7 +166,7 @@ VALUE ll_driver_config_set_actions(VALUE self, VALUE array)
 
     Data_Get_Struct(self, DriverConfig, config);
 
-    config->action_names       = ALLOC_N(ID, row_count);
+    config->action_names       = ALLOC_N(VALUE, row_count);
     config->action_arg_amounts = ALLOC_N(long, row_count);
 
     FOR(rindex, row_count)

data/ext/c/driver_config.h

@@ -30,7 +30,7 @@ typedef struct
     long **table;
 
     /* Array containing action names as Symbols */
-    ID *action_names;
+    VALUE *action_names;
 
     /* Array containing the arity for every action */
     long *action_arg_amounts;

data/ext/java/org/libll/Driver.java

@@ -27,11 +27,15 @@ import org.jruby.runtime.builtin.IRubyObject;
 @JRubyClass(name="LL::Driver", parent="Object")
 public class Driver extends RubyObject
 {
-    private static long T_EOF      = -1;
-    private static long T_RULE     = 0;
-    private static long T_TERMINAL = 1;
-    private static long T_EPSILON  = 2;
-    private static long T_ACTION   = 3;
+    private static long T_EOF                = -1;
+    private static long T_RULE               = 0;
+    private static long T_TERMINAL           = 1;
+    private static long T_EPSILON            = 2;
+    private static long T_ACTION             = 3;
+    private static long T_STAR               = 4;
+    private static long T_PLUS               = 5;
+    private static long T_ADD_VALUE_STACK    = 6;
+    private static long T_APPEND_VALUE_STACK = 7;
 
     /**
      * The current Ruby runtime.
@@ -132,8 +136,8 @@ public class Driver extends RubyObject
                         token_id = self.config.terminals.get(type);
                     }
 
-                    // Rule
-                    if ( stack_type == self.T_RULE )
+                    // A rule or the "+" operator
+                    if ( stack_type == self.T_RULE || stack_type == self.T_PLUS )
                     {
                         Long production_i = self.config.table
                             .get(stack_value.intValue())
@@ -152,6 +156,17 @@ public class Driver extends RubyObject
                         }
                         else
                         {
+                            // Append a "*" operator for all following
+                            // occurrences as they are optional
+                            if ( stack_type == self.T_PLUS )
+                            {
+                                stack.push(self.T_STAR);
+                                stack.push(stack_value);
+
+                                stack.push(self.T_APPEND_VALUE_STACK);
+                                stack.push(Long.valueOf(0));
+                            }
+
                             ArrayList<Long> row = self.config.rules
                                 .get(production_i.intValue());
 
@@ -161,6 +176,47 @@ public class Driver extends RubyObject
                             }
                         }
                     }
+                    // "*" operator
+                    else if ( stack_type == self.T_STAR )
+                    {
+                        Long production_i = self.config.table
+                            .get(stack_value.intValue())
+                            .get(token_id.intValue());
+
+                        if ( production_i != self.T_EOF )
+                        {
+                            stack.push(self.T_STAR);
+                            stack.push(stack_value);
+
+                            stack.push(self.T_APPEND_VALUE_STACK);
+                            stack.push(Long.valueOf(0));
+
+                            ArrayList<Long> row = self.config.rules
+                                .get(production_i.intValue());
+
+                            for ( int index = 0; index < row.size(); index++ )
+                            {
+                                stack.push(row.get(index));
+                            }
+                        }
+                    }
+                    // Adds a new array to the value stack that can be used to
+                    // group operator values together
+                    else if ( stack_type == self.T_ADD_VALUE_STACK )
+                    {
+                        RubyArray operator_buffer = self.runtime.newArray();
+
+                        value_stack.push(operator_buffer);
+                    }
+                    // Appends the last value on the value stack to the operator
+                    // buffer that preceeds it.
+                    else if ( stack_type == self.T_APPEND_VALUE_STACK )
+                    {
+                        IRubyObject last_value    = value_stack.pop();
+                        RubyArray operator_buffer = (RubyArray) value_stack.peek();
+
+                        operator_buffer.append(last_value);
+                    }
                     // Terminal
                     else if ( stack_type == self.T_TERMINAL )
                     {

data/lib/ll.rb

@@ -18,6 +18,7 @@ require_relative 'll/rule'
 require_relative 'll/branch'
 require_relative 'll/terminal'
 require_relative 'll/epsilon'
+require_relative 'll/operator'
 require_relative 'll/message'
 require_relative 'll/ast/node'
 require_relative 'll/erb_context'

data/lib/ll/branch.rb

@@ -25,7 +25,13 @@ module LL
     def first_set
       first = steps[0]
 
-      return first.is_a?(Rule) ? first.first_set : [first]
+      if first.is_a?(Rule)
+        return first.first_set
+      elsif first
+        return [first]
+      else
+        return []
+      end
     end
 
     ##

data/lib/ll/configuration_compiler.rb

@@ -8,11 +8,15 @@ module LL
     # @return [Hash]
     #
     TYPES = {
-      :eof      => -1,
-      :rule     => 0,
-      :terminal => 1,
-      :epsilon  => 2,
-      :action   => 3
+      :eof                => -1,
+      :rule               => 0,
+      :terminal           => 1,
+      :epsilon            => 2,
+      :action             => 3,
+      :star               => 4,
+      :plus               => 5,
+      :add_value_stack    => 6,
+      :append_value_stack => 7
     }.freeze
 
     ##
@@ -105,7 +109,21 @@ module LL
 
       grammar.rules.each do |rule|
         rule.branches.each do |branch|
-          bodies[:"_rule_#{index}"] = branch.ruby_code || DEFAULT_RUBY_CODE
+          if branch.ruby_code
+            code = branch.ruby_code
+
+          # If a branch only contains a single, non-epsilon step we can just
+          # return that value as-is. This makes parsing code a little bit
+          # easier.
+          elsif !branch.ruby_code and branch.steps.length == 1 \
+          and !branch.steps[0].is_a?(Epsilon)
+            code = 'val[0]'
+
+          else
+            code = DEFAULT_RUBY_CODE
+          end
+
+          bodies[:"_rule_#{index}"] = code
 
           index += 1
         end
@@ -133,17 +151,24 @@ module LL
           action_index += 1
 
           branch.steps.reverse_each do |step|
-            if step.is_a?(LL::Terminal)
+            if step.is_a?(Terminal)
               row << TYPES[:terminal]
               row << term_indices[step] + 1
 
-            elsif step.is_a?(LL::Rule)
+            elsif step.is_a?(Rule)
               row << TYPES[:rule]
               row << rule_indices[step]
 
-            elsif step.is_a?(LL::Epsilon)
+            elsif step.is_a?(Epsilon)
               row << TYPES[:epsilon]
               row << 0
+
+            elsif step.is_a?(Operator)
+              row << TYPES[step.type]
+              row << rule_indices[step.receiver]
+
+              row << TYPES[:add_value_stack]
+              row << 0
             end
           end