antelope 0.1.8 → 0.1.9

data/lib/antelope/generator/null.rb

@@ -4,6 +4,11 @@ module Antelope
     # Represents a generator that does not generate anything.
     class Null < Base
 
+      register_as "null"
+
+      has_directive "null.data"
+      has_directive "comment"
+
       # Does nothing.
       #
       # @return [void]

data/lib/antelope/generator/output.rb

@@ -1,7 +1,5 @@
 # encoding: utf-8
 
-require "pp"
-
 module Antelope
   module Generator
 
@@ -11,6 +9,8 @@ module Antelope
 
       register_as "output"
 
+      has_directive "output.show-lookahead", Boolean
+
       # Defines singleton method for every mod that the grammar passed
       # to the generator.
       #
@@ -42,7 +42,7 @@ module Antelope
       #
       # @return [void]
       def generate
-        template "output.erb", "#{file}.output"
+        template "output", "#{file}.output"
       end
     end
   end

data/lib/antelope/generator/ruby.rb

@@ -8,7 +8,10 @@ module Antelope
     # Generates a ruby parser.
     class Ruby < Base
 
-      register_as "ruby"
+      register_as "ruby", "rubby"
+
+      has_directive "panic-mode", Boolean
+      has_directive "ruby.error-class", String
 
       # Creates an action table for the parser.
       #
@@ -36,23 +39,38 @@ module Antelope
           block = if block.empty?
             "proc { |_| _ }"
           else
-            "proc #{production.block}"
+            "proc #{block}"
           end
 
           out << block << "],\n"
         end
 
-        out.chomp!(",\n")
+        out.chomp!(   ",\n")
 
         out << "]"
       end
 
+      def define_own_handler?
+        directives.ruby.error_class? or
+          panic_mode?
+      end
+
+      def panic_mode?
+        directives.panic_mode &&
+          directives.ruby.error_class? &&
+          grammar.contains_error_token?
+      end
+
+      def error_class
+        directives.ruby.error_class
+      end
+
       # Actually performs the generation.  Takes the template from
-      # ruby.erb and outputs it to `<file>_parser.rb`.
+      # ruby.ant and outputs it to `<file>.rb`.
       #
       # @return [void]
       def generate
-        template "ruby.erb", "#{file}.rb" do |body|
+        template "ruby", "#{file}.rb" do |body|
           sprintf(grammar.compiler.body, :write => body)
         end
       end

data/lib/antelope/generator/templates/c_header.ant

@@ -0,0 +1,36 @@
+#ifndef _%{guard_name}_H
+#define _%{guard_name}_H
+
+% if define_stype?
+typedef union {
+  %{union_body}
+} %{stype};
+% end
+
+enum %{prefix}tokentype {
+% starting = 257
+  %{token_prefix}EOF = 0,
+% grammar.terminals.each_with_index do |terminal, i|
+  %{token_prefix}%{terminal.name} = %{starting + i},
+% end
+};
+
+typedef struct {
+  struct %{prefix}stack_element* elements;
+  unsigned int current;
+  unsigned int max;
+  unsigned char free;
+} %{prefix}pstate;
+
+#define %{upper_prefix}PUSH_MORE -1
+#define %{upper_prefix}TERMINALS %{grammar.terminals.size}
+#define %{upper_prefix}STATES %{table.size}
+
+const char* %{prefix}token_string(%{terminal_type} token);
+int %{prefix}lex(%{stype}* lval%{lex_params});
+int %{prefix}parse_push(%{parse_params});
+int %{prefix}parse_pull(%{parse_params});
+%{prefix}pstate* %{prefix}pstate_new();
+void %{prefix}pstate_delete(%{prefix}pstate* stack);
+
+#endif

data/lib/antelope/generator/templates/c_source.ant

@@ -0,0 +1,202 @@
+#include "%{file}.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <limits.h>
+#include <alloca.h>
+#include <string.h>
+
+static const unsigned int %{prefix}states[][%{upper_prefix}TERMINALS] = {
+% table.each do |state|
+  { %{action_for(state)} },
+% end
+};
+
+#ifndef %{upper_prefix}INITSTACK
+#  define %{upper_prefix}INITSTACK 64
+#endif
+
+#ifndef %{upper_prefix}ALLOC
+#  define %{upper_prefix}ALLOC malloc
+#endif
+
+#ifndef %{upper_prefix}FREE
+#  define %{upper_prefix}FREE free
+#endif
+
+struct %{prefix}stack_element {
+  %{stype}* val;
+  unsigned int state;
+};
+
+#define %{upper_prefix}PUSH_STACK(stack, v, s) do { \
+  int err; \
+  if(stack.current + 1 > stack.max) \
+  { \
+    if((err = %{prefix}resize_stack(&stack)) != 0) \
+      return err; \
+  } \
+  stack.elements[stack.current].val = val; \
+  stack.elements[stack.current].state = s; \
+} while(0)
+
+#define %{upper_prefix}POP_STACK(stack, out) do { \
+  out = stack.elements + stack.current; \
+  stack.current -= 1; \
+} while(0)
+
+const char* %{prefix}token_string(%{terminal_type} token)
+{
+  switch(token)
+  {
+% grammar.terminals.each do |terminal|
+  case %{token_prefix}%{terminal.name}:
+    return %{terminal.to_s.inspect};
+% end
+  }
+
+  return "(unknown)";
+}
+
+%{prefix}pstate* %{prefix}state_new()
+{
+  %{prefix}pstate* stack = %{upper_prefix}ALLOC(sizeof(%{prefix}pstate));
+  if(stack != NULL)
+  {
+    stack->max = %{upper_prefix}INITSTACK;
+    stack->elements = %{upper_prefix}ALLOC(sizeof(struct %{prefix}stack_element) * stack->max);
+    stack->current = 0;
+    stack->free = 1;
+  }
+
+  return stack;
+}
+
+void %{prefix}state_delete(%{prefix}pstate* stack)
+{
+  stack->current = 0;
+  stack->max = 0;
+  stack->free = 0;
+
+  if(stack->free)
+  {
+    %{upper_prefix}FREE(stack->elements);
+    %{upper_prefix}FREE(stack);
+  }
+
+  stack->elements = 0;
+}
+
+int %{prefix}resize_stack(%{prefix}pstate* stack)
+{
+  void* body;
+  int new_max;
+
+  if(stack->current + 1 > stack->max)
+  {
+    new_max = stack->max * 2;
+  }
+  else if((stack->current / 2) < stack->max)
+  {
+    new_max = stack->max / 2;
+  }
+  else
+  {
+    return 0;
+  }
+
+
+  body = %{upper_prefix}ALLOC(sizeof(struct %{prefix}stack_element) * new_max);
+
+  if(body == NULL)
+  {
+    return ENOMEM;
+  }
+
+  memcpy(body, stack->elements, sizeof(struct %{prefix}stack_element) * stack->current);
+
+  if(stack->free)
+  {
+    %{upper_prefix}FREE(stack->elements);
+  }
+
+  stack->elements = body;
+  stack->max = new_max;
+
+  return 0;
+}
+
+int %{prefix}parse_push(%{prefix}pstate* stack int token,
+  %{upper_prefix}STYPE* val%{parse_params})
+{
+  unsigned int action;
+  %{prefix}stack_element* current_state;
+
+  current_state = stack.elements + stack.current - 1;
+
+  if(token > %{upper_prefix}TERMINALS || token < 0)
+    %{prefix}abort;
+
+get_action:
+  action = states[current_state->state][token];
+
+  if(action < %{upper_prefix}STATES)
+  {
+    %{upper_prefix}PUSH_STACK(stack, val, action - 1);
+    return %{upper_prefix}PUSH_MORE;
+  }
+
+  switch(action)
+  {
+  case 0:
+    %{prefix}error;
+    break;
+  case %{table.size + 1}:
+    stack.current = 0;
+    return 0;
+% productions.each_with_index do |(label, size, block), i|
+  case %{i + table.size + 2}: { // %{i}
+    %{upper_prefix}STYPE* %{prefix}vals[%{size}];
+    %{upper_prefix}STYPE* %{prefix}out;
+%   size.times do |e|
+    %{upper_prefix}POP_STACK(stack, %{prefix}vals[%{e}]);
+%   end
+    %{prefix}out = %{prefix}vals[0];
+    current_state = stack.elements + stack.current - 1;
+
+    do {
+      {{= cify_block(block) }}
+    } while(0);
+
+    token  = %{symbols[label.name]};
+    action = states[current_state->state][token];
+    %{upper_prefix}PUSH_STACK(stack, %{prefix}out, action);
+    break;
+  }
+% end
+  }
+
+  return %{upper_prefix}PUSH_MORE;
+}
+
+int %{prefix}parse_pull(%{parse_params})
+{
+  %{prefix}pstate stack;
+  int token;
+  unsigned int action;
+  %{upper_prefix}STYPE val;
+  %{prefix}stack_element* current_state;
+
+  stack.max      = %{upper_prefix}INITSTACK;
+  stack.elements = alloca(sizeof(struct %{prefix}stack_element) * stack.max);
+  stack.current  = 0;
+  stack.free     = 0;
+
+  %{upper_prefix}PUSH_STACK(stack, NULL, 1);
+
+  while(stack.current > 0)
+  {
+    token = %{prefix}lex(&val%{params});
+    %{prefix}parse_push(&stack, token, &val%{parse_params});
+  }
+}

data/lib/antelope/generator/templates/output.ant

@@ -0,0 +1,68 @@
+Productions:
+{{
+  len = grammar.all_productions.size.to_s.size
+  productions = grammar.all_productions.
+    map { |x| ["#{x.label}: #{x.items.join(' ')}", x.block] }
+  body = productions.map { |_| _.first.size }.max
+}}
+{{ productions.each_with_index do |prod, i| }}
+  {{=  "%#{len}s" % i }} {{=  "%-#{body}s" % prod[0] }} {{=  prod[1] }}
+{{ end }}
+
+{{ if unused_symbols.any? }}
+Symbols unused in grammar:
+{{   unused_symbols.each do |sym| }}
+  {{=  sym }}
+{{   end }}
+{{ end }}
+
+Precedence:
+  --- highest
+{{ grammar.precedence.each do |pr| }}
+  {{=  "%-8s" % pr.type }} {{=  pr.level }}:
+    {{= "{" << pr.tokens.to_a.join(", ") << "}" }}
+{{ end }}
+  --- lowest
+
+{{
+  len = table.flatten.map(&:keys).flatten.map(&:size).max
+  states = grammar.states.to_a
+}}
+
+{{ table.each_with_index do |v, i| }}
+{{   state = states[i] }}
+  State {{=  i }}:
+{{   state.rules.each do |rule| }}
+    {{=  rule }}
+      {{= "{" << rule.lookahead.to_a.join(", ") << "}" }}
+{{   end }}
+{{
+  transitions = v.each.select { |_, a| a && a[0] == :state }
+  reductions  = v.each.select { |_, a| a && a[0] == :reduce}
+  accepting   = v.each.select { |_, a| a && a[0] == :accept}
+  conflicts   = tableizer.conflicts[i].each
+  thing = [:transitions, :reductions, :accepting]
+  num_type = {
+    transitions: "State",
+    reductions: "Rule",
+    accepting: "Rule"
+  }
+  h = Hash[thing.zip([transitions, reductions, accepting])]
+}}
+{{   h.each do |key, value| }}
+{{     next unless value.any? }}
+    {{=  key }}:
+{{     value.each do |token, (_, name)| }}
+{{       token_value = grammar.terminals.
+            find { |_| _.name == token } || token }}
+      {{=  token_value }}: {{=  num_type[key] }} {{=  name }}
+{{     end }}
+{{   end }}
+{{   if conflicts.any? }}
+    conflicts:
+{{     conflicts.each do |token, (first, second, rule, terminal)| }}
+      {{=  token }}: {{=  first.join(" ") }}/{{=  second.join(" ") }} ({{=  rule }} vs {{=  terminal }})
+{{     end }}
+{{   end }}
+
+{{ end }}

data/lib/antelope/generator/templates/ruby.ant

@@ -0,0 +1,146 @@
+
+# This file assumes that the output of the generator will be placed
+# within a module or a class.  However, the module/class requires a
+# `type` method, which takes a terminal and gives its type, as a
+# symbol.  These types should line up with the terminals that were
+# defined in the original grammar.
+
+# The actions to take during parsing.  In every state, there are a
+# set of acceptable peek tokens; this table tells the parser what
+# to do on each acceptable peek token.  The possible actions include
+# `:accept`, `:reduce`, and `:state`; `:accept` means to accept the
+# input and return the value of the pasing.  `:reduce` means to
+# reduce the top of the stack into a given nonterminal.  `:state`
+# means to transition to another state.
+#
+# @return [Array<Hash<(Symbol, Array<(Symbol, Numeric)>)>>]
+ACTION_TABLE = {{= generate_action_table }}.freeze
+
+# A list of all of the productions.  Only includes the left-hand side,
+# the number of tokens on the right-hand side, and the block to call
+# on reduction.
+#
+# @return [Array<Array<(Symbol, Numeric, Proc)>>]
+PRODUCTIONS  = {{= generate_productions_list }}.freeze
+
+# Runs the parser.
+#
+# @param input [Array<Object>] the input to run the parser over.
+# @return [Object] the result of the accept.
+def parse(input)
+  stack = []
+  stack.push([nil, 0])
+  input = input.dup
+  last  = nil
+
+  until stack.empty? do
+    last = parse_action(stack, input)
+  end
+
+  last
+
+end
+
+# Actually performs the parsing action on the given stack on input.
+# If you want to implement a push parser, than messing with this
+# method is probably the way to go.
+#
+# @param stack [Array<Array<(Object, Numeric)>>] the stack of the
+#   parser.  The actual order of the stack is important.
+# @param input [Array<Object>] the input to run the parser over.
+#   The elements of this may be passed to the `type` method.
+# @return [Object] the result of the last accepting reduction.
+def parse_action(stack, input)
+  last = nil
+  peek_token = if input.empty?
+    :$end
+  else
+    type(input.first)
+  end
+
+  action = ACTION_TABLE[stack.last.last].fetch(peek_token) do
+    ACTION_TABLE[stack.last.last].fetch(:$default)
+  end
+  case action.first
+  when :accept
+    production = PRODUCTIONS[action.last]
+    last       = stack.pop(production[1]).first.first
+    stack.pop
+  when :reduce
+    production = PRODUCTIONS[action.last]
+    removing   = stack.pop(production[1])
+    value = instance_exec(*removing.map(&:first), &production[2])
+    goto  = ACTION_TABLE[stack.last.last][production[0]]
+    stack.push([value, goto.last])
+  when :state
+    stack.push([input.shift, action.last])
+  else
+    raise NotImplementedError, "Unknown action #{action.first}"
+  end
+
+  last
+
+rescue KeyError => e
+  if handle_error(
+    { :stack     => stack,
+      :peek      => peek_token,
+      :remaining => input,
+      :error     => e,
+      :expected  => ACTION_TABLE[stack.last.last].keys
+    })
+    retry
+  end
+end
+
+{{ if define_own_handler? }}
+def handle_error(data, _ = false)
+{{   if panic_mode? }}
+  if _ || data[:peek] == :$end # we can't recover if
+                              # we're at the end
+{{   end }}
+    raise {{= error_class }},
+      "Unexpected token #{data[:peek]}; " \
+      "expected one of #{data[:expected].join(', ')}",
+      data[:error].backtrace
+{{   if panic_mode? }}
+  end
+
+  new_peek = :$error
+  acceptable_state = false
+  state = nil
+
+  until data[:stack].empty? or acceptable_state
+    state = data[:stack].last.last
+
+    if ACTION_TABLE[state].key?(new_peek)
+      acceptable_state = true
+    else
+      data[:stack].pop # discard
+    end
+  end
+
+  return handle_error(data, true) unless acceptable_state
+
+  action = ACTION_TABLE[state][new_peek]
+  lookaheads = nil
+
+  until lookaheads
+    if action[0] == :state
+      lookaheads = ACTION_TABLE[action.last].keys
+    elsif action[0] == :reduce
+      rule   = PRODUCTIONS[action.last]
+      action = ACTION_TABLE[stack[-rule[1]].last][rule[0]]
+    end
+  end
+
+  until data[:remaining].empty? || lookaheads.
+    include?(data[:remaining][0].first)
+    data[:remaining].shift
+  end
+
+  data[:remaining].unshift([new_peek, data[:error]])
+  true
+
+{{   end }}
+end
+{{ end }}

data/lib/antelope/generator.rb

@@ -16,6 +16,15 @@ module Antelope
     def generators
       @_generators ||= Hash.new { |h, k| h[k] = Generator::Null }
     end
+    # Returns a hash of all of the directives that are available in
+    # the generators of this module.
+    #
+    # @see .generators
+    # @return [Hash]
+    def directives
+      generators.values.map(&:directives).
+        inject({}, :merge)
+    end
 
     # Registers a generator with the given names.  If multiple names
     # are given, they are assigned the generator as a value in the
@@ -27,15 +36,14 @@ module Antelope
     # @param name [String, Symbol] a name to associate the generator
     #   with.
     def register_generator(generator, *names)
-      names = [names].flatten
-      raise ArgumentError,
+      names = [names].flatten      raise ArgumentError,
         "Requires at least one name" unless names.any?
       raise ArgumentError,
         "All name values must be a Symbol or string" unless names.
         all? {|_| [Symbol, String].include?(_.class) }
 
       names.each do |name|
-        generators[name] = generator
+        generators[name.to_s.downcase] = generator
       end
     end
 
@@ -48,3 +56,7 @@ require "antelope/generator/base"
 require "antelope/generator/group"
 require "antelope/generator/output"
 require "antelope/generator/ruby"
+require "antelope/generator/null"
+require "antelope/generator/c_header"
+require "antelope/generator/c_source"
+require "antelope/generator/c"

data/lib/antelope/template/compiler.rb

@@ -0,0 +1,78 @@
+module Antelope
+  class Template
+    class Compiler
+
+      attr_reader :buffer
+
+      attr_reader :tokens
+
+      def initialize(tokens, buffer_variable = "_out")
+        @tokens = tokens.dup
+        @buffer = ""
+        @buffer_variable = buffer_variable
+      end
+
+      def compile
+        merge_text_tokens
+
+        @buffer = "#{@buffer_variable} ||= \"\"\n"
+
+        until @tokens.empty?
+          token = @tokens.shift
+          parse_method = "parse_#{token[0]}".intern
+
+          send(parse_method, token[1])
+        end
+
+        @buffer << "#{@buffer_variable}\n"
+
+        @buffer
+
+      rescue NoMethodError => e
+
+        if e.name == parse_method
+          raise NoTokenError, "No token #{token[0]} exists"
+        else
+          raise
+        end
+      end
+
+      private
+
+      def parse_text(value)
+        buffer << "#{@buffer_variable} << #{value.to_s.inspect}\n"
+      end
+
+      def parse_tag(value)
+        value.gsub!(/\A\s*([\s\S]*?)\s*\Z/, "\\1")
+        buffer << "#{value}\n"
+      end
+
+      def parse_output_tag(value)
+        value.gsub!(/\A\s*([\s\S]*?)\s*\Z/, "\\1")
+        buffer << "#{@buffer_variable} << begin\n  " \
+          "#{value}\nend.to_s\n"
+      end
+
+      def parse_newline(_)
+        parse_text("\n")
+      end
+
+      def parse_comment_tag(_)
+      end
+
+      def merge_text_tokens
+        new_tokens = []
+        @tokens.chunk(&:first).each do |type, tokens|
+          if type == :text
+            new_tokens << [:text, tokens.map(&:last).join('')]
+          else
+            new_tokens.push(*tokens)
+          end
+        end
+
+        @tokens = new_tokens
+      end
+    end
+  end
+end

data/lib/antelope/template/errors.rb

@@ -0,0 +1,9 @@
+module Antelope
+  class Template
+    class Error < Antelope::Error; end
+
+    class SyntaxError < Error; end
+
+    class NoTokenError < Error; end
+  end
+end