ruby-ll 1.0.0

data/lib/ll/configuration_compiler.rb

@@ -0,0 +1,204 @@
+module LL
+  ##
+  # Compiles an instance of {LL::CompiledConfiguration} which is used by
+  # {LL::CodeGenerator} to actually generate Ruby source code.
+  #
+  class ConfigurationCompiler
+    ##
+    # @return [Hash]
+    #
+    TYPES = {
+      :rule     => 0,
+      :terminal => 1,
+      :epsilon  => 2,
+      :action   => 3
+    }.freeze
+
+    ##
+    # @return [String]
+    #
+    DEFAULT_RUBY_CODE = 'val'.freeze
+
+    ##
+    # @param [LL::CompiledGrammar] grammar
+    # @return [LL::CompiledConfiguration]
+    #
+    def generate(grammar)
+      return CompiledConfiguration.new(
+        :name          => generate_name(grammar),
+        :namespace     => generate_namespace(grammar),
+        :inner         => grammar.inner,
+        :header        => grammar.header,
+        :terminals     => generate_terminals(grammar),
+        :actions       => generate_actions(grammar),
+        :action_bodies => generate_action_bodies(grammar),
+        :rules         => generate_rules(grammar),
+        :table         => generate_table(grammar)
+      )
+    end
+
+    ##
+    # @param [LL::CompiledGrammar] grammar
+    # @return [String]
+    #
+    def generate_name(grammar)
+      return grammar.name.split('::').last
+    end
+
+    ##
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Array]
+    #
+    def generate_namespace(grammar)
+      parts = grammar.name.split('::')
+
+      return parts.length > 1 ? parts[0..-2] : []
+    end
+
+    ##
+    # Returns an Array containing all the terminal names as symbols. The first
+    # terminal is always `:$EOF` to ensure the array has the same amount of rows
+    # as there are columns in the `table` array.
+    #
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Array]
+    #
+    def generate_terminals(grammar)
+      terminals = [:$EOF]
+
+      grammar.terminals.each do |term|
+        terminals << term.name.to_sym
+      end
+
+      return terminals
+    end
+
+    ##
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Array]
+    #
+    def generate_actions(grammar)
+      actions = []
+      index   = 0
+
+      grammar.rules.each do |rule|
+        rule.branches.each do |branch|
+          args = branch.steps.reject { |step| step.is_a?(Epsilon) }.length
+
+          actions << [:"_rule_#{index}", args]
+
+          index += 1
+        end
+      end
+
+      return actions
+    end
+
+    ##
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Hash]
+    #
+    def generate_action_bodies(grammar)
+      bodies = {}
+      index  = 0
+
+      grammar.rules.each do |rule|
+        rule.branches.each do |branch|
+          bodies[:"_rule_#{index}"] = branch.ruby_code || DEFAULT_RUBY_CODE
+
+          index += 1
+        end
+      end
+
+      return bodies
+    end
+
+    ##
+    # Builds the rules table of the parser. Each row is built in reverse order.
+    #
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Array]
+    #
+    def generate_rules(grammar)
+      rules        = []
+      action_index = 0
+      rule_indices = grammar.rule_indices
+      term_indices = grammar.terminal_indices
+
+      grammar.rules.each_with_index do |rule, rule_index|
+        rule.branches.each do |branch|
+          row = [TYPES[:action], action_index]
+
+          action_index += 1
+
+          branch.steps.reverse_each do |step|
+            if step.is_a?(LL::Terminal)
+              row << TYPES[:terminal]
+              row << term_indices[step] + 1
+
+            elsif step.is_a?(LL::Rule)
+              row << TYPES[:rule]
+              row << rule_indices[step]
+
+            elsif step.is_a?(LL::Epsilon)
+              row << TYPES[:epsilon]
+              row << 0
+            end
+          end
+
+          rules << row
+        end
+      end
+
+      return rules
+    end
+
+    ##
+    # Generates the table array for the parser. This array has the following
+    # structure:
+    #
+    #     [
+    #       [EOF, TERMINAL 1, TERMINAL 2, TERMINAL 3, ...]
+    #     ]
+    #
+    # EOF is always the first column and is used when running out of input while
+    # processing a rule.
+    #
+    # @param [LL::CompiledGrammar] grammar
+    # @return [Array]
+    #
+    def generate_table(grammar)
+      branch_index = 0
+      term_indices = grammar.terminal_indices
+      columns      = grammar.terminals.length + 1
+
+      table = Array.new(grammar.rules.length) do
+        Array.new(columns, -1)
+      end
+
+      grammar.rules.each_with_index do |rule, rule_index|
+        rule.branches.each do |branch|
+          branch.first_set.each do |step|
+            # For terminals we'll base the column index on the terminal index.
+            if step.is_a?(Terminal)
+              terminal_index = term_indices[step]
+
+              table[rule_index][terminal_index + 1] = branch_index
+
+            # For the rest (= epsilon) we'll update all columns that haven't
+            # been updated yet.
+            else
+              table[rule_index].each_with_index do |col, col_index|
+                table[rule_index][col_index] = branch_index if col == -1
+              end
+            end
+          end
+
+          branch_index += 1
+        end
+      end
+
+      return table
+    end
+  end # ConfigurationCompiler
+end # LL

data/lib/ll/driver.rb

@@ -0,0 +1,46 @@
+module LL
+  ##
+  # Parser driver for generated parsers.
+  #
+  class Driver
+    ##
+    # Error method that is called when no rule was found for a table index.
+    #
+    # @param [Fixnum] stack_value
+    # @param [Array] token
+    #
+    def stack_input_error(stack_value, token)
+      type = token[0].inspect
+
+      raise ParserError, "Unexpected rule #{stack_value} for #{type}"
+    end
+
+    ##
+    # Error method that is called when the stack has been consumed but there's
+    # still input being sent to the parser.
+    #
+    # @param [Array] token
+    #
+    def unexpected_input_error(token)
+      raise(
+        ParserError,
+        "Received token #{token[0].inspect} but there's nothing left to parse"
+      )
+    end
+
+    ##
+    # Error method that is called when an invalid terminal was specified as the
+    # input.
+    #
+    # @param [Fixnum] got_id The ID of the received terminal.
+    # @param [Fixnum] expected_id The ID of the expected terminal.
+    #
+    def invalid_terminal_error(got_id, expected_id)
+      terminals = self.class::CONFIG.terminals
+      expected  = terminals[expected_id].inspect
+      got       = terminals[got_id].inspect
+
+      raise ParserError, "Invalid terminal #{got}, expected #{expected}"
+    end
+  end # Driver
+end # LL

data/lib/ll/driver_config.rb

@@ -0,0 +1,36 @@
+module LL
+  ##
+  # Class containing C/Java data for a Driver class.
+  #
+  class DriverConfig
+    attr_reader :terminals, :rules, :table, :actions
+
+    ##
+    # @param [Array] array
+    #
+    def terminals=(array)
+      self.terminals_native = @terminals = array
+    end
+
+    ##
+    # @param [Array] array
+    #
+    def rules=(array)
+      self.rules_native = @rules = array
+    end
+
+    ##
+    # @param [Array] array
+    #
+    def table=(array)
+      self.table_native = @table = array
+    end
+
+    ##
+    # @param [Array] array
+    #
+    def actions=(array)
+      self.actions_native = @actions = array
+    end
+  end # DriverConfig
+end # LL

data/lib/ll/driver_template.erb

@@ -0,0 +1,51 @@
+# This file is automatically generated by ruby-ll. Manually changing this file
+# is not recommended as any changes will be lost the next time this parser is
+# re-generated.
+<%- if @add_requires -%>
+require 'll/setup'
+<%- end -%>
+
+<%- @config.namespace.each do |part| -%>
+module <%= part %>
+<%- end -%>
+<%- if @config.header -%>
+<%= @config.header.strip -%>
+
+<%- end -%>
+class <%= @config.name -%> < LL::Driver
+  CONFIG = LL::DriverConfig.new
+
+  CONFIG.terminals = [
+    <%- @config.terminals.each_with_index do |terminal, index| -%>
+    <%= terminal.inspect -%>, # <%= index %>
+    <%- end -%>
+  ].freeze
+
+  CONFIG.rules = [
+    <%- @config.rules.each_with_index do |row, index| -%>
+    <%= row.inspect -%>, # <%= index %>
+    <%- end -%>
+  ].freeze
+
+  CONFIG.table = [
+    <%- @config.table.each_with_index do |row, index| -%>
+    <%= row.inspect -%>, # <%= index %>
+    <%- end -%>
+  ].freeze
+
+  CONFIG.actions = [
+    <%- @config.actions.each_with_index do |row, index| -%>
+    <%= row.inspect -%>, # <%= index %>
+    <%- end -%>
+  ].freeze
+<%= @config.inner.rstrip if @config.inner %>
+  <%- @config.action_bodies.each do |name, body| -%>
+
+  def <%= name %>(val)
+    <%= body %>
+  end
+  <%- end -%>
+end
+<%- @config.namespace.each do |part| -%>
+end
+<%- end -%>

data/lib/ll/epsilon.rb

@@ -0,0 +1,23 @@
+module LL
+  ##
+  # Class used for indicating an epsilon in a grammar. Epsilon objects are
+  # primarily used to break out of recursion.
+  #
+  class Epsilon
+    attr_reader :source_line
+
+    ##
+    # @param [LL::SourceLine] source_line
+    #
+    def initialize(source_line)
+      @source_line = source_line
+    end
+
+    ##
+    # @return [String]
+    #
+    def inspect
+      return 'Epsilon()'
+    end
+  end # Epsilon
+end # LL

data/lib/ll/erb_context.rb

@@ -0,0 +1,23 @@
+module LL
+  ##
+  # A context for a single ERB template, used for storing variables and
+  # retrieving the binding for a template.
+  #
+  class ERBContext
+    ##
+    # @param [Hash] variables
+    #
+    def initialize(variables = {})
+      variables.each do |name, value|
+        instance_variable_set("@#{name}", value)
+      end
+    end
+
+    ##
+    # @return [Binding]
+    #
+    def get_binding
+      return binding
+    end
+  end # ERBContext
+end # LL

data/lib/ll/grammar_compiler.rb

@@ -0,0 +1,359 @@
+module LL
+  ##
+  # The GrammarCompiler class processes an AST (as parsed from an LL(1) grammar)
+  # and returns an {LL::CompiledGrammar} instance.
+  #
+  class GrammarCompiler
+    ##
+    # @param [LL::AST::Node] ast
+    # @return [LL::CompiledGrammar]
+    #
+    def compile(ast)
+      compiled = CompiledGrammar.new
+
+      process(ast, compiled)
+
+      warn_for_unused_terminals(compiled)
+      warn_for_unused_rules(compiled)
+
+      verify_first_first(compiled)
+      verify_first_follow(compiled)
+
+      return compiled
+    end
+
+    ##
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    # @return [LL::CompiledGrammar]
+    #
+    def process(node, compiled_grammar)
+      handler = "on_#{node.type}"
+
+      return send(handler, node, compiled_grammar)
+    end
+
+    ##
+    # Adds warnings for any unused rules. The first defined rule is skipped
+    # since it's the root rule.
+    #
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def warn_for_unused_rules(compiled_grammar)
+      compiled_grammar.rules.each_with_index do |rule, index|
+        next if index == 0 || rule.references > 0
+
+        compiled_grammar.add_warning(
+          "Unused rule #{rule.name.inspect}",
+          rule.source_line
+        )
+      end
+    end
+
+    ##
+    # Adds warnings for any unused terminals.
+    #
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def warn_for_unused_terminals(compiled_grammar)
+      compiled_grammar.terminals.each do |terminal|
+        next if terminal.references > 0
+
+        compiled_grammar.add_warning(
+          "Unused terminal #{terminal.name.inspect}",
+          terminal.source_line
+        )
+      end
+    end
+
+    ##
+    # Verifies all rules to see if they don't have any first/first conflicts.
+    # Errors are added for every rule where this _is_ the case.
+    #
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def verify_first_first(compiled_grammar)
+      compiled_grammar.rules.each do |rule|
+        conflicting = Set.new
+
+        rule.branches.each do |branch|
+          next if conflicting.include?(branch)
+
+          rule.branches.each do |other_branch|
+            next if branch == other_branch || conflicting.include?(other_branch)
+
+            overlapping = branch.first_set & other_branch.first_set
+
+            unless overlapping.empty?
+              conflicting << branch
+              conflicting << other_branch
+            end
+          end
+        end
+
+        unless conflicting.empty?
+          compiled_grammar.add_error(
+            'first/first conflict, multiple branches start with the same terminals',
+            rule.source_line
+          )
+
+          conflicting.each do |branch|
+            labels = branch.first_set.map do |token|
+              token.is_a?(Epsilon) ? 'epsilon' : token.name
+            end
+
+            compiled_grammar.add_error(
+              "branch starts with: #{labels.join(', ')}",
+              branch.source_line
+            )
+          end
+        end
+      end
+    end
+
+    ##
+    # Adds errors for any rules containing first/follow conflicts.
+    #
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def verify_first_follow(compiled_grammar)
+      compiled_grammar.rules.each do |rule|
+        rule.branches.each do |branch|
+          has_epsilon = branch.first_set.find { |step| step.is_a?(Epsilon) }
+
+          if has_epsilon and !branch.follow_set.empty?
+            compiled_grammar.add_error(
+              'first/follow conflict, branch can start with epsilon and is ' \
+                'followed by (non) terminals',
+              branch.source_line
+            )
+
+            compiled_grammar.add_error(
+              'epsilon originates from here',
+              has_epsilon.source_line
+            )
+          end
+        end
+      end
+    end
+
+    ##
+    # Processes the root node of a grammar.
+    #
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def on_grammar(node, compiled_grammar)
+      # Create the prototypes for all rules since rules can be referenced before
+      # they are defined.
+      node.children.each do |child|
+        if child.type == :rule
+          on_rule_prototype(child, compiled_grammar)
+        end
+      end
+
+      node.children.each do |child|
+        process(child, compiled_grammar)
+      end
+    end
+
+    ##
+    # Sets the name of the parser.
+    #
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def on_name(node, compiled_grammar)
+      if compiled_grammar.name
+        compiled_grammar.add_warning(
+          "Overwriting existing parser name #{compiled_grammar.name.inspect}",
+          node.source_line
+        )
+      end
+
+      parts = node.children.map { |child| process(child, compiled_grammar) }
+
+      compiled_grammar.name = parts.join('::')
+    end
+
+    ##
+    # Processes the assignment of terminals.
+    #
+    # @see #process
+    #
+    def on_terminals(node, compiled_grammar)
+      node.children.each do |child|
+        name = process(child, compiled_grammar)
+
+        if compiled_grammar.has_terminal?(name)
+          compiled_grammar.add_error(
+            "The terminal #{name.inspect} has already been defined",
+            child.source_line
+          )
+        else
+          compiled_grammar.add_terminal(name, child.source_line)
+        end
+      end
+    end
+
+    ##
+    # Processes an %inner directive.
+    #
+    # @see #process
+    #
+    def on_inner(node, compiled_grammar)
+      compiled_grammar.inner = process(node.children[0], compiled_grammar)
+    end
+
+    ##
+    # Processes a %header directive.
+    #
+    # @see #process
+    #
+    def on_header(node, compiled_grammar)
+      compiled_grammar.header = process(node.children[0], compiled_grammar)
+    end
+
+    ##
+    # Processes a node containing Ruby source code.
+    #
+    # @see #process
+    # @return [String]
+    #
+    def on_ruby(node, compiled_grammar)
+      return node.children[0]
+    end
+
+    ##
+    # Extracts the name from an identifier.
+    #
+    # @see #process
+    # @return [String]
+    #
+    def on_ident(node, compiled_grammar)
+      return node.children[0]
+    end
+
+    ##
+    # Processes an epsilon.
+    #
+    # @see #process
+    # @return [LL::Epsilon]
+    #
+    def on_epsilon(node, compiled_grammar)
+      return Epsilon.new(node.source_line)
+    end
+
+    ##
+    # Processes the assignment of a rule.
+    #
+    # @see #process
+    #
+    def on_rule(node, compiled_grammar)
+      name = process(node.children[0], compiled_grammar)
+
+      if compiled_grammar.has_terminal?(name)
+        compiled_grammar.add_error(
+          "the rule name #{name.inspect} is already used as a terminal name",
+          node.source_line
+        )
+      end
+
+      if compiled_grammar.has_rule_with_branches?(name)
+        compiled_grammar.add_error(
+          "the rule #{name.inspect} has already been defined",
+          node.source_line
+        )
+
+        return
+      end
+
+      branches = node.children[1..-1].map do |child|
+        process(child, compiled_grammar)
+      end
+
+      rule = compiled_grammar.lookup_rule(name)
+
+      rule.branches.concat(branches)
+    end
+
+    ##
+    # Creates a basic prototype for a rule.
+    #
+    # @see #process
+    #
+    def on_rule_prototype(node, compiled_grammar)
+      name = process(node.children[0], compiled_grammar)
+
+      return if compiled_grammar.has_rule?(name)
+
+      rule = Rule.new(name, node.source_line)
+
+      compiled_grammar.add_rule(rule)
+    end
+
+    ##
+    # Processes a single rule branch.
+    #
+    # @see #process
+    # @return [LL::Branch]
+    #
+    def on_branch(node, compiled_grammar)
+      steps = process(node.children[0], compiled_grammar)
+
+      if node.children[1]
+        code = process(node.children[1], compiled_grammar)
+      else
+        code = nil
+      end
+
+      return Branch.new(steps, node.source_line, code)
+    end
+
+    ##
+    # Processes the steps of a branch.
+    #
+    # @see #process
+    # @return [Array]
+    #
+    def on_steps(node, compiled_grammar)
+      steps = []
+
+      node.children.each do |step_node|
+        retval = process(step_node, compiled_grammar)
+
+        # Literal rule/terminal names.
+        if retval.is_a?(String)
+          step = compiled_grammar.lookup_identifier(retval)
+
+          undefined_identifier!(retval, step_node, compiled_grammar) unless step
+        # Epsilon
+        else
+          step = retval
+        end
+
+        if step
+          step.increment_references if step.respond_to?(:increment_references)
+
+          steps << step
+        end
+      end
+
+      return steps
+    end
+
+    private
+
+    ##
+    # @param [String] name
+    # @param [LL::AST::Node] node
+    # @param [LL::CompiledGrammar] compiled_grammar
+    #
+    def undefined_identifier!(name, node, compiled_grammar)
+      compiled_grammar.add_error(
+        "Undefined terminal or rule #{name.inspect}",
+        node.source_line
+      )
+    end
+  end # Compiler
+end # LL