antelope 0.1.8 → 0.1.9

data/lib/antelope/ace/compiler.rb

@@ -14,8 +14,8 @@ module Antelope
     # - `:directive` (2 arguments)
     # - `:copy` (1 argument)
     # - `:second` (no arguments)
-    # - `:label` (1 argument)
-    # - `:part` (1 argument)
+    # - `:label` (2 arguments)
+    # - `:part` (2 arguments)
     # - `:or` (no arguments)
     # - `:prec` (1 argument)
     # - `:block` (1 argument)
@@ -92,14 +92,21 @@ module Antelope
         @rules    = []
         @current  = nil
         @current_label = nil
-        @options  = { :terminals => [], :prec => [], :extra => {} }
+        @options  = {
+          :terminals    => [],
+          :nonterminals => [],
+          :prec         => [],
+          :type         => nil,
+          :extra        => Hashie::Extensions::IndifferentAccess.
+            inject!({})
+        }
       end
 
       # Pretty inspect.
       #
       # @return [String]
       def inspect
-        "#<#{self.class} state=#{@state.inspect} options=#{@options.inspect}>"
+        "#<#{self.class} state=#{@state.inspect} options=#{options.inspect}>"
       end
 
       # Runs the compiler on the input tokens.  For each token,
@@ -152,19 +159,35 @@ module Antelope
         name = name.intern
         case name
         when :terminal, :token
-          @options[:terminals] << [args[0].intern, args[1]]
+          handle_token(args)
         when :require
           compare_versions(args[0])
         when :left, :right, :nonassoc
-          @options[:prec] << [name, *args.map(&:intern)]
+          options[:prec] << [name, *args.map(&:intern)]
+        when :language, :generator, :"grammar.type"
+          options[:type] = args[0].downcase
         when :type
-          @options[:type] = args[0]
+          raise SyntaxError, "%type directive requires first " \
+            "argument to be caret" unless args[0].caret?
+
+          options[:nonterminals] <<
+            [args[0], args[1..-1].map(&:intern)]
+        when :define
+          compile_extra(args[0], args[1..-1])
         else
-          @options[:extra][name] = args
-          $stderr.puts "Unknown Directive: #{name}"
+          compile_extra(name, args)
         end
       end
 
+      def compile_extra(name, args)
+        matching = Generator.directives[name.to_s]
+
+        raise NoDirectiveError, "no directive named #{name}" \
+          unless matching
+
+        options[:extra][name] = args
+      end
+
       # Compiles a copy token.  A copy token basically copies its
       # argument directly into the body.  Used in both the first
       # and third parts.
@@ -192,16 +215,18 @@ module Antelope
       # @param label [String] the left-hand side of the rule; it
       #   should be a nonterminal.
       # @return [void]
-      def compile_label(label)
+      def compile_label(label, val)
         require_state! :second
         if @current
           @rules << @current
         end
 
-        @current_label = label.intern
+        label = label.intern
+        @current_label = [label, val]
 
         @current = {
-          label: @current_label,
+          label: label,
+          label_id: val,
           set:   [],
           block: "",
           prec:  ""
@@ -213,9 +238,9 @@ module Antelope
       # It adds the first argument to the set of the current rule.
       #
       # @param text [String] the symbol to append to the current rule.
-      def compile_part(text)
+      def compile_part(text, val)
         require_state! :second
-        @current[:set] << text.intern
+        @current[:set] << [text.intern, val]
       end
 
       # Compiles an or.  This should only occur in a rule definition,
@@ -225,7 +250,7 @@ module Antelope
       # @return [void]
       # @see #compile_label
       def compile_or
-        compile_label(@current_label)
+        compile_label(*@current_label)
       end
 
       # Compiles the precedence operator.  This should only occur in a
@@ -265,6 +290,18 @@ module Antelope
 
       private
 
+      def handle_token(args)
+        type = ""
+        if args[0].caret?
+          type = args.shift
+        end
+
+        name = args.shift
+        value = args.shift
+
+        options[:terminals] << [name.intern, type, nil, value]
+      end
+
       # Checks the current state against the given states.
       #
       # @raise [InvalidStateError] if none of the given states match

data/lib/antelope/ace/errors.rb

@@ -37,5 +37,12 @@ module Antelope
     # generator to use for the generation, it raises this.
     class NoTypeError < Error
     end
+
+    # Primarily used in the {Compiler}, it is raised if it encounters
+    # a directive it cannot handle.  This is more to warn the
+    # developer that a directive they wrote may not be accepted by any
+    # generator.
+    class NoDirectiveError < Error
+    end
   end
 end

data/lib/antelope/ace/grammar/generation.rb

@@ -33,14 +33,14 @@ module Antelope
           mods = modifiers.map(&:last).
             map  { |x| x.new(self) }
           mods.each do |mod|
-            puts "Running mod #{mod.class}..."
+            puts "Running mod #{mod.class}..." if options[:verbose]
             mod.call
           end
           hash = Hash[modifiers.map(&:first).zip(mods)]
           # This is when we'd generate
 
           find_generators(generators, options).each do |gen|
-            puts "Running generator #{gen}..."
+            puts "Running generator #{gen}..." if options[:verbose]
             gen.new(self, hash).generate
           end
         end
@@ -71,7 +71,7 @@ module Antelope
 
           generators
 
-        rescue KeyError => e
+        rescue KeyError
           raise NoTypeError, "Undefined type #{type}"
         end
       end

data/lib/antelope/ace/grammar/precedences.rb

@@ -27,14 +27,12 @@ module Antelope
         def precedence_for(token)
           token = token.name if token.is_a?(Token)
 
-          set = Set.new([token, :_])
-
           prec = precedence.
-            select { |pr| set.intersection(pr.tokens).any? }.
-            first
+            select { |pr| pr.tokens.include?(token) }.first
 
-          if token == :modifier
-            p prec
+          unless prec
+            prec = precedence.
+              select { |pr| pr.tokens.include?(:_) }.first
           end
 
           prec
@@ -56,7 +54,7 @@ module Antelope
           end
 
           precedence <<
-            Ace::Precedence.new(:nonassoc, [:"$"].to_set, 0) <<
+            Ace::Precedence.new(:nonassoc, [:$end].to_set, 0) <<
             Ace::Precedence.new(:nonassoc, [:_].to_set, 1)
           precedence.sort_by { |_| _.level }.reverse
         end

data/lib/antelope/ace/grammar/productions.rb

@@ -35,8 +35,10 @@ module Antelope
         # @return [Token]
         def find_token(value)
           value = value.intern
+
           if productions.key?(value)
-            Token::Nonterminal.new(value)
+            typed_nonterminals.find { |term| term.name == value } ||
+              Token::Nonterminal.new(value)
           elsif terminal = terminals.
               find { |term| term.name == value }
             terminal
@@ -45,7 +47,8 @@ module Antelope
           elsif [:nothing, :ε].include?(value)
             Token::Epsilon.new
           else
-            raise UndefinedTokenError, "Could not find a token named #{value.inspect}"
+            raise UndefinedTokenError, "Could not find a token " \
+              "named #{value.inspect}"
           end
         end
 
@@ -88,22 +91,44 @@ module Antelope
         # @param id [Numeric] the id for the production.
         # @return [Production]
         def generate_production_for(rule, id)
-          left  = rule[:label]
-          items = rule[:set].map { |_| find_token(_) }
+          left  = Token::Nonterminal.new(rule[:label])
+          items = rule[:set].map { |_| find_token(_[0]) }
           prec  = if rule[:prec].empty?
             items.select(&:terminal?).last
           else
-            find_token(rule[:prec])
+            rule[:prec].intern
           end
 
-          unless rule[:prec].empty?
-            puts "PREC, #{prec.inspect}"
+          prec  = precedence_for(prec)
+          left.type = type_for(rule[:label])
+          left.id = rule[:label_id]
+
+          rule[:set].each_with_index do |tok, i|
+            items[i] = items[i].dup
+            items[i].id = tok[1]
           end
 
-          prec  = precedence_for(prec)
+          Production.new(left, items, rule[:block], prec, id + 1)
+        end
+
+        # Returns the defined type for the given token name.
+        # Uses the `%type` directive to infer the corresponding types.
+        #
+        # @param token [Symbol] the token to check for
+        #   types.
+        def type_for(token)
+          token = find_token(token) unless token.is_a?(Token)
 
-          Production.new(Token::Nonterminal.new(left), items,
-               rule[:block], prec, id + 1)
+          case token
+          when Token::Nonterminal
+            token.type
+          when Token::Terminal
+            token.type
+          when Token::Epsilon
+            ""
+          when Token::Error
+            ""
+          end
         end
 
         # Creates the default production for the grammar.  The left
@@ -116,7 +141,7 @@ module Antelope
         def default_production
           Production.new(Token::Nonterminal.new(:$start), [
               Token::Nonterminal.new(@compiler.rules.first[:label]),
-              Token::Terminal.new(:"$")
+              Token::Terminal.new(:$end)
             ], "", precedence.last, 0)
         end
       end

data/lib/antelope/ace/grammar/{terminals.rb → symbols.rb}

@@ -4,8 +4,8 @@ module Antelope
   module Ace
     class Grammar
 
-      # Manages a list of the terminals in the grammar.
-      module Terminals
+      # Manages a list of the symbols in the grammar.
+      module Symbols
 
         # A list of all terminals in the grammar.  Checks the compiler
         # options for terminals, and then returns an array of
@@ -28,6 +28,21 @@ module Antelope
           @_nonterminals ||= productions.keys
         end
 
+        # A list of all nonterminals, with types.
+        #
+        # @return [Array<Token::Nonterminal>>]
+        def typed_nonterminals
+          @_typed_nonterminals ||= begin
+            typed = []
+            compiler.options[:nonterminals].each do |data|
+              data[1].each do |nonterm|
+                typed << Token::Nonterminal.new(nonterm, data[0])
+              end
+            end
+            typed
+          end
+        end
+
         # A list of all symbols in the grammar; includes both
         # terminals and nonterminals.
         #
@@ -37,6 +52,14 @@ module Antelope
         def symbols
           @_symbols ||= terminals + nonterminals
         end
+
+        # Checks to see if the grammar uses the `error` terminal
+        # anywhere.
+        #
+        # @return [Boolean]
+        def contains_error_token?
+          all_productions.any? { |_| _.items.any?(&:error?) }
+        end
       end
     end
   end

data/lib/antelope/ace/grammar.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
-require "antelope/ace/grammar/terminals"
+require "hashie"
+require "antelope/ace/grammar/symbols"
 require "antelope/ace/grammar/productions"
 require "antelope/ace/grammar/precedences"
 require "antelope/ace/grammar/loading"
@@ -10,11 +11,11 @@ module Antelope
   module Ace
 
     # Defines a grammar from an Ace file.  This handles setting up
-    # productions, loading from files, terminals, precedence, and
+    # productions, loading from files, symbols, precedence, and
     # generation.
     class Grammar
 
-      include Terminals
+      include Symbols
       include Productions
       include Precedences
       include Loading
@@ -55,6 +56,14 @@ module Antelope
         @output   = Pathname.new(output)
         @compiler = compiler
       end
+
+      # Extra options from the compiler.  This can be used by
+      # generators for output information.
+      #
+      # @return [Hash]
+      def options
+        compiler.options[:extra]
+      end
     end
   end
 end

data/lib/antelope/ace/precedence.rb

@@ -49,6 +49,10 @@ module Antelope
         end
       end
 
+      # Converts the precedence into a representative string, denoting
+      # the type and the level.
+      #
+      # @return [String]
       def to_s
         "#{type.to_s[0]}#{level}"
       end

data/lib/antelope/ace/scanner/argument.rb

@@ -0,0 +1,57 @@
+module Antelope
+  module Ace
+    class Scanner
+
+      # Represents an argument to a directive.  It encapsulates a
+      # string object, which is the value of the argument.
+      class Argument < String
+
+        # Initialize the argument.
+        #
+        # @param type [Symbol] the type of argument it is; it can be
+        #   a `:block`, `:text`, or `:caret`.  The type is defined by
+        #   the encapsulating characters.  If the encapsulating
+        #   characters are `{` and `}`, it's a `:block`; if they are
+        #   `<` and `>`, it's a `:caret`; otherwise, it's a `:text`.
+        # @param value [String] the value of the argument.
+        def initialize(type, value)
+          @type = type
+          super(value)
+        end
+
+        # If this argument is type `:block`.
+        #
+        # @return [Boolean]
+        # @see type?
+        def block?
+          type? :block
+        end
+
+        # If this argument is type `:text`.
+        #
+        # @return [Boolean]
+        # @see type?
+        def text?
+          type? :text
+        end
+
+        # If this argument is type `:caret`.
+        #
+        # @return [Boolean]
+        # @see type?
+        def caret?
+          type? :caret
+        end
+
+        # Checks to see if any of the given arguments match the type
+        # of this argument.
+        #
+        # @param inc [Array<Symbol>]
+        # @return [Boolean]
+        def type?(*inc)
+          inc.include?(@type)
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/scanner/first.rb

@@ -45,17 +45,43 @@ module Antelope
         #
         # @return [Boolean] if it matched.
         def scan_first_directive
-          if @scanner.scan(/%([A-Za-z_-]+) ?/)
+          if @scanner.scan(/%(#{IDENTIFIER}) ?/)
             directive = @scanner[1]
-            arguments = []
-            until @scanner.check(/\n/)
+            arguments = scan_first_directive_arguments
+
+            tokens << [:directive, directive, arguments]
+          end
+        end
+
+        # Scan the arguments for a directive.  It keeps attempting to
+        # scan arguments until the first newline that was not in a
+        # block.  Arguments can be blocks, carets, or text; blocks are
+        # encapsulated with `{` and `}`, carets are encapsulated with
+        # `<` and `>`, and text is encapsulated with quotes or
+        # nothing.
+        #
+        # @return [Array<Argument>]
+        def scan_first_directive_arguments
+          arguments = []
+          until @scanner.check(/\n/)
+            if @scanner.scan(/\{/)
+              argument =
+                Argument.new(:block, _scan_block[1..-2])
+            elsif @scanner.scan(/</)
+              @scanner.scan(/((?:\\>|[^>])*)\>/)
+              argument =
+                Argument.new(:caret, @scanner[1])
+            else
               @scanner.scan(/#{VALUE}/x) or error!
-              arguments.push(@scanner[2] || @scanner[3])
-              @scanner.scan(/ */)
+              argument = Argument.new(:text,
+                @scanner[2] || @scanner[3])
             end
 
-            tokens << [:directive, directive, arguments]
+            arguments.push(argument)
+            @scanner.scan(/ */)
           end
+
+          arguments
         end
       end
     end

data/lib/antelope/ace/scanner/second.rb

@@ -45,7 +45,7 @@ module Antelope
         # @see #scan_second_rule_body
         # @see #error!
         def scan_second_rule
-          if @scanner.check(/([a-z._-]+):/)
+          if @scanner.check(/(#{IDENTIFIER})(\[#{IDENTIFIER}\])?:/)
             scan_second_rule_label or error!
             scan_second_rule_body
             true
@@ -57,8 +57,8 @@ module Antelope
         #
         # @return [Boolean] if it matched.
         def scan_second_rule_label
-          if @scanner.scan(/([a-z._-]+): ?/)
-            tokens << [:label, @scanner[1]]
+          if @scanner.scan(/(#{IDENTIFIER})(?:\[(#{IDENTIFIER})\])?: ?/)
+            tokens << [:label, @scanner[1], @scanner[2]]
           end
         end
 
@@ -88,8 +88,8 @@ module Antelope
         #
         # @return [Boolean] if it matched.
         def scan_second_rule_part
-          if @scanner.scan(/([A-Za-z._-]+)(?!\:|[A-Za-z._-])/)
-            tokens << [:part, @scanner[1]]
+          if @scanner.scan(/(#{IDENTIFIER})(?:\[(#{IDENTIFIER})\])?(?!\:|[A-Za-z._])/)
+            tokens << [:part, @scanner[1], @scanner[2]]
           end
         end
 
@@ -107,7 +107,7 @@ module Antelope
         #
         # @return [Boolean] if it matched.
         def scan_second_rule_prec
-          if @scanner.scan(/%prec ([A-Za-z._-]+)/)
+          if @scanner.scan(/%prec (#{IDENTIFIER})/)
             tokens << [:prec, @scanner[1]]
           end
         end
@@ -139,17 +139,32 @@ module Antelope
         def _scan_block
           brack = 1
           body = "{"
+          scan_for = %r{
+            (
+              (?: " ( \\\\ | \\" | [^"] )* "? )
+            | (?: ' ( \\\\ | \\' | [^'] )* '? )
+            | (?: // .*? \n )
+            | (?: \# .*? \n )
+            | (?: /\* [\s\S]+? \*/ )
+            | (?: \} )
+            | (?: \{ )
+            )
+          }x
 
           until brack.zero?
-            if part = @scanner.scan_until(/(\}|\{)/)
+            if part = @scanner.scan_until(scan_for)
               body << part
 
+
               if @scanner[1] == "}"
                 brack -= 1
-              else
+              elsif @scanner[1] == "{"
                 brack += 1
               end
             else
+              if @scanner.scan(/(.+)/m)
+                @line += @scanner[1].count("\n")
+              end
               error!
             end
           end

data/lib/antelope/ace/scanner.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require "strscan"
+require "antelope/ace/scanner/argument"
 require "antelope/ace/scanner/first"
 require "antelope/ace/scanner/second"
 require "antelope/ace/scanner/third"
@@ -8,16 +9,19 @@ require "antelope/ace/scanner/third"
 module Antelope
   module Ace
 
-    # Scans a given input.  The input should be a properly formatted ACE file;
-    # see the Ace module for more information.  This scanner uses the
-    # StringScanner class internally; see the ruby documentation for more on
-    # that.  This scanner seperates scanning into three seperate stages:
-    # First, Second, and Third, for each section of the file, respectively.
+    # Scans a given input.  The input should be a properly formatted
+    # ACE file; see the Ace module for more information.  This scanner
+    # uses the StringScanner class internally; see the ruby
+    # documentation for more on that.  This scanner seperates scanning
+    # into three seperate stages: First, Second, and Third, for each
+    # section of the file, respectively.
     #
     # @see Ace
     # @see http://ruby-doc.org/stdlib-2.1.2/libdoc/strscan/rdoc/StringScanner.html
     class Scanner
 
+      IDENTIFIER = "[a-zA-Z_.][a-zA-Z0-9_.-]*"
+
       include First
       include Second
       include Third
@@ -32,30 +36,30 @@ module Antelope
       # @return [Array<Array<(Symbol, Object, ...)>>]
       attr_reader :tokens
 
-      # The boundry between each section.  Placed here to be easily modifiable.
-      # **MUST** be a regular expression.
+      # The boundry between each section.  Placed here to be easily.
+      # modifiable. **MUST** be a regular expression.
       #
       # @return [RegExp]
       CONTENT_BOUNDRY = /%%/
 
-      # The value regular expression.  It should match values; for example,
-      # things quoted in strings or word letters without quotes.  Must respond
-      # to #to_s, since it is embedded within other regular expressions.  The
-      # regular expression should place the contents of the value in the
-      # groups 2 or 3.
+      # The value regular expression.  It should match values; for
+      # example, things quoted in strings or word letters without
+      # quotes.  Must respond to #to_s, since it is embedded within
+      # other regular expressions.  The regular expression should
+      # place the contents of the value in the groups 2 or 3.
       #
       # @return [#to_s]
       VALUE = %q{(?:
                     (?:("|')((?:\\\\|\\"|\\'|.)+?)\\1)
-                  | ([[:word:]]+)
+                  | ([A-Za-z0-9_.<>*-]+)
                 )}
 
       # Scans a file.  It returns the tokens resulting from scanning.
       #
-      # @param source [String] the source to scan.  This should be compatible
-      #   with StringScanner.
-      # @param name [String] the name of the source file.  This is primarily
-      #   used in backtrace information.
+      # @param source [String] the source to scan.  This should be
+      #   compatible with StringScanner.
+      # @param name [String] the name of the source file.  This is
+      #   primarilyused in backtrace information.
       # @return [Array<Array<(Symbol, Object, ...)>>]
       # @see #tokens
       def self.scan(source, name = "(ace file)")
@@ -92,8 +96,15 @@ module Antelope
         start = [@scanner.pos - 8, 0].max
         stop  = [@scanner.pos + 8, @scanner.string.length].min
         snip  = @scanner.string[start..stop].strip.inspect
-        char  = @scanner.string[@scanner.pos].inspect
-        new_line = "#{@source}:#{@line}:unexpected #{char} (near #{snip})"
+        char  = @scanner.string[@scanner.pos]
+        char = if char
+          char.inspect
+        else
+          "EOF"
+        end
+
+        new_line = "#{@source}:#{@line}: unexpected #{char} " \
+          "(near #{snip})"
 
         raise e, e.message, [new_line, *e.backtrace]
       end
@@ -111,17 +122,12 @@ module Antelope
 
       private
 
-      # Raises an error; first creates a small snippet to give the developer
-      # some context.
+      # Raises an error.
       #
       # @raise [SyntaxError] always.
       # @return [void]
       def error!
-        start = [@scanner.pos - 8, 0].max
-        stop  = [@scanner.pos + 8, @scanner.string.length].min
-        snip  = @scanner.string[start..stop].strip
-        char  = @scanner.string[@scanner.pos]
-        raise SyntaxError, "invalid syntax"# near `#{snip.inspect}' (#{char.inspect})"
+        raise SyntaxError, "invalid syntax"
       end
     end
   end