antelope 0.0.1

data/lib/antelope/ace/grammar/production.rb

@@ -0,0 +1,47 @@
+module Antelope
+  module Ace
+    class Grammar
+
+      # Defines a production.
+      #
+      # @!attribute [rw] label
+      #   The label (or left-hand side) of the production.  This
+      #   should be a nonterminal.
+      #
+      #   @return [Symbol]
+      # @!attribute [rw] items
+      #   The body (or right-hand side) of the production.  This can
+      #   be array of terminals and nonterminals.
+      #
+      #   @return [Array<Token>]
+      # @!attribute [rw] block
+      #   The block of code to be executed when the production's right
+      #   hand side is reduced.
+      #
+      #   @return [String]
+      # @!attribute [rw] prec
+      #   The presidence declaration for the production.
+      #
+      #   @return [Ace::Presidence]
+      # @!attribute [rw] id
+      #   The ID of the production.  The starting production always
+      #   has an ID of 0.
+      #
+      #   @return [Numeric]
+      class Production < Struct.new(:label, :items, :block, :prec, :id)
+
+        # Creates a new production from a hash.  The hash's keys
+        # correspond to the attributes on this class.
+        #
+        # @param hash [Hash<(Symbol, Object)>]
+        def self.from_hash(hash)
+          new(hash[:label] || hash["label"],
+              hash[:items] || hash["items"],
+              hash[:block] || hash["block"],
+              hash[:prec]  || hash["prec"],
+              hash[:id]    || hash["id"])
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/grammar/productions.rb

@@ -0,0 +1,119 @@
+module Antelope
+  module Ace
+    class Grammar
+
+      # Manages the productions of the grammar.
+      module Productions
+
+        # Returns a hash of all of the productions.  The result is
+        # cached.
+        #
+        # @return [Hash<(Symbol, Array<Production>)>]
+        def productions
+          @_productions || generate_productions
+        end
+
+        # Returns all productions for all nonterminals, sorted by id.
+        #
+        # @return [Array<Production>]
+        def all_productions
+          productions.values.flatten.sort_by(&:id)
+        end
+
+        private
+
+        # Actually generates the productions.  Uses the rules from the
+        # compiler to construct the productions.  Makes two loops over
+        # the compiler's rules; the first to tell the grammar that the
+        # nonterminal does exist, and the second to actually construct
+        # the productions.  The first loop is for {#find_token},
+        # because otherwise it wouldn't be able to return a
+        # nonterminal properly.
+        #
+        # @return [Hash<(Symbol, Array<Production>)>]
+        def generate_productions
+          @_productions = {}
+
+          @compiler.rules.each do |rule|
+            productions[rule[:label]] = []
+          end.each_with_index do |rule, id|
+            productions[rule[:label]] <<
+              generate_production_for(rule, id)
+          end
+
+          productions[:$start] = [default_production]
+
+          productions
+        end
+
+        # Generates a production for a given compiler rule.  Converts
+        # the tokens in the set to their {Token} counterparts,
+        # and then sets the presidence for the production.  If the
+        # presidence declaration from the compiler rule is empty,
+        # then it'll use the last terminal from the set to check for
+        # presidence; otherwise, it'll use the presidence declaration.
+        # This is to make sure that every production has a presidence
+        # declaration.
+        #
+        # @param rule [Hash] the compiler's rule.
+        # @param id [Numeric] the id for the production.
+        # @return [Production]
+        def generate_production_for(rule, id)
+          left  = rule[:label]
+          items = rule[:set].map { |_| find_token(_) }
+          prec  = if rule[:prec].empty?
+            items.select(&:terminal?).last
+          else
+            find_token(rule[:prec])
+          end
+
+          prec  = presidence_for(prec)
+
+          Production.new(Token::Nonterminal.new(left), items,
+               rule[:block], prec, id + 1)
+        end
+
+        # Creates the default production for the grammar.  The left
+        # hand side of the production is the `:$start` symbol, with
+        # the right hand side being the first rule's left-hand side
+        # and the terminal `$`.  This production is automagically
+        # given the last presidence, and an id of 0.
+        #
+        # @return [Production]
+        def default_production
+          Production.new(Token::Nonterminal.new(:$start), [
+              Token::Nonterminal.new(@compiler.rules.first[:label]),
+              Token::Terminal.new(:"$")
+            ], "", presidence.last, 0)
+        end
+
+        # Finds a token based on its corresponding symbol.  First
+        # checks the productions, to see if it's a nonterminal; then,
+        # tries to find it in the terminals; otherwise, if the symbol
+        # is `error`, it returns a {Token::Error}; if the symbol is
+        # `nothing` or `ε`, it returns a {Token::Epsilon}; if it's
+        # none of those, it raises an {UndefiendTokenError}.
+        #
+        # @raise [UndefinedTokenError] if the token doesn't exist.
+        # @param value [String, Symbol, #intern] the token's symbol to
+        #   check.
+        # @return [Token]
+        def find_token(value)
+          value = value.intern
+          if productions.key?(value)
+            Token::Nonterminal.new(value)
+          elsif terminal = terminals.
+              find { |term| term.name == value }
+            terminal
+          elsif value == :error
+            Token::Error.new
+          elsif [:nothing, :ε].include?(value)
+            Token::Epsilon.new
+          else
+            raise UndefinedTokenError, "Could not find a token named #{value.inspect}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/grammar/terminals.rb

@@ -0,0 +1,41 @@
+module Antelope
+  module Ace
+    class Grammar
+
+      # Manages a list of the terminals in the grammar.
+      module Terminals
+
+        # A list of all terminals in the grammar.  Checks the compiler
+        # options for terminals, and then returns an array of
+        # terminals.  Caches the result.
+        #
+        # @return [Array<Token::Terminal>]
+        def terminals
+          @_terminals ||= begin
+            @compiler.options.fetch(:terminals, []).map do |v|
+              Token::Terminal.new(*v)
+            end
+          end
+        end
+
+        # A list of all nonterminals in the grammar.
+        #
+        # @return [Array<Symbol>]
+        # @see #productions
+        def nonterminals
+          @_nonterminals ||= productions.keys
+        end
+
+        # A list of all symbols in the grammar; includes both
+        # terminals and nonterminals.
+        #
+        # @return [Array<Token::Terminal, Symbol>]
+        # @see #terminals
+        # @see #nonterminals
+        def symbols
+          @_symbols ||= terminals + nonterminals
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/grammar.rb

@@ -0,0 +1,59 @@
+require "antelope/ace/grammar/terminals"
+require "antelope/ace/grammar/productions"
+require "antelope/ace/grammar/presidence"
+require "antelope/ace/grammar/loading"
+require "antelope/ace/grammar/generation"
+require "antelope/ace/grammar/production"
+
+module Antelope
+  module Ace
+
+    # Defines a grammar from an Ace file.  This handles setting up
+    # productions, loading from files, terminals, presidence, and
+    # generation.
+    class Grammar
+
+      include Terminals
+      include Productions
+      include Presidence
+      include Loading
+      include Grammar::Generation
+
+      # Used by a generation class; this is all the generated states
+      # of the grammar.
+      #
+      # @return [Set<Generation::Recognizer::State>]
+      # @see Generation::Recognizer
+      attr_accessor :states
+
+      # The name of the grammar.  This is normally assumed from a file
+      # name.
+      #
+      # @return [String]
+      attr_accessor :name
+
+      # The output directory for the grammar.  This is normally the
+      # same directory as the Ace file.
+      #
+      # @return [Pathname]
+      attr_accessor :output
+
+      # The compiler for the Ace file.
+      #
+      # @return [Compiler]
+      attr_reader :compiler
+
+      # Initialize.
+      #
+      # @param name [String]
+      # @param output [String] the output directory.  Automagically
+      #   turned into a Pathname.
+      # @param compiler [Compiler]
+      def initialize(name, output, compiler)
+        @name     = name
+        @output   = Pathname.new(output)
+        @compiler = compiler
+      end
+    end
+  end
+end

data/lib/antelope/ace/presidence.rb

@@ -0,0 +1,51 @@
+module Antelope
+  module Ace
+
+    # Defines a presidence.  A presidence has a type, tokens, and a
+    # level.
+    class Presidence < Struct.new(:type, :tokens, :level)
+
+      # @!attribute [rw] type
+      #   The type of presidence level.  This should be one of
+      #   `:left`, `:right`, or `:nonassoc`.
+      #
+      #   @return [Symbol] the type.
+      # @!attribute [rw] tokens
+      #   An set of tokens that are on this specific presidence
+      #   level.  The tokens are identified as symbols.  The special
+      #   symbol, `:_`, represents any token.
+      #
+      #   @return [Set<Symbol>] the tokens on this level.
+      # @!attribute [rw] level
+      #   The level we're on.  The higher the level, the higher the
+      #   presidence.
+
+      include Comparable
+
+      # Compares the other object to this object.  If the other object
+      # isn't a {Presidence}, it returns nil.  If the other
+      # presidence isn't on the same level as this one, then the
+      # levels are compared and the result of that is returned.  If
+      # it is, however, the type is checked; if this presidence is
+      # left associative, then it returns 1 (it is greater than the
+      # other); if this presidence is right associative, then it
+      # returns -1 (it is less than the other); if this presidence is
+      # nonassociative, it returns 0 (it is equal to the other).
+      #
+      # @param other [Object] the object to compare to this one.
+      # @return [Numeric?]
+      def <=>(other)
+        return nil unless other.is_a? Presidence
+        if level != other.level
+          level <=> other.level
+        elsif type == :left
+          1
+        elsif type == :right
+          -1
+        else
+          0
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/scanner/first.rb

@@ -0,0 +1,61 @@
+module Antelope
+  module Ace
+    class Scanner
+
+      # Scans the first section of the file.  This contains directives and
+      # small blocks that can be copied directly into the body of the output.
+      # The blocks are formatted as `%{ ... %}`; however, the ending tag _must_
+      # be on its own line.  The directive is formatted as `%<name> <value>`,
+      # with `<name>` being the key, and `<value>` being the value.  The value
+      # can be a piece of straight-up text (no quotes), or it can be quoted.
+      # There can be any number of values to a directive.
+      module First
+
+        # Scans until the first content boundry.  If it encounters anything but
+        # a block or a directive (or whitespace), it will raise an error.
+        #
+        # @raise [SyntaxError] if it encounters anything but whitespace, a
+        #   block, or a directive.
+        # @return [void]
+        def scan_first_part
+          until @scanner.check(CONTENT_BOUNDRY)
+            scan_first_copy || scan_first_directive ||
+            scan_whitespace || error!
+          end
+        end
+
+        # Scans for a block.  It is called `copy` instead of `block` because
+        # contents of the block is _copied_ directly into the body.
+        #
+        # @return [Boolean] if it matched.
+        def scan_first_copy
+          if @scanner.scan(/%{([\s\S]+?)\n\s*%}/)
+            tokens << [:copy, @scanner[1]]
+          end
+        end
+
+        # Scans a directive.  A directive has one _name_, and any number of
+        # arguments.  Every argument is a _value_.  The name can be any
+        # combinations of alphabetical characters, underscores, and dashes;
+        # the value can be word characters, or a quote-delimited string.
+        # It emits a `:directive` token with the directive (Sring) as an
+        # argument, and the passed arguments (Array<String>).
+        #
+        # @return [Boolean] if it matched.
+        def scan_first_directive
+          if @scanner.scan(/%([A-Za-z_-]+) ?/)
+            directive = @scanner[1]
+            arguments = []
+            until @scanner.check(/\n/)
+              @scanner.scan(/#{VALUE}/x) or error!
+              arguments.push(@scanner[2] || @scanner[3])
+              @scanner.scan(/ */)
+            end
+
+            tokens << [:directive, directive, arguments]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/scanner/second.rb

@@ -0,0 +1,160 @@
+module Antelope
+  module Ace
+    class Scanner
+
+      # Scans the second part of the file.  The second part of the
+      # file _only_ contains productions (or rules).  Rules have a
+      # label and a body; the label may be any lowercase alphabetical
+      # identifier followed by a colon; the body consists of "parts",
+      # an "or", a "prec", and/or a "block".  The part may consist
+      # of any alphabetical characters.  An or is just a vertical bar
+      # (`|`).  A prec is a presidence declaraction, which is `%prec `
+      # followed by any alphabetical characters.  A block is a `{`,
+      # followed by code, followed by a terminating `}`.  Rules _may_
+      # be terminated by a semicolon, but this is optional.
+      module Second
+
+        # Scans the second part of the file.  This should be from just
+        # before the first content boundry; if the scanner doesn't
+        # find a content boundry, it will error.  It will then check
+        # for a rule.
+        #
+        # @raise [SyntaxError] if no content boundry was found, or if
+        #   the scanner encounters anything but a rule or whitespace.
+        # @return [void]
+        # @see #scan_second_rule
+        # @see #scan_whitespace
+        # @see #error!
+        def scan_second_part
+          scanner.scan(CONTENT_BOUNDRY) or error!
+          tokens << [:second]
+
+          until @scanner.check(CONTENT_BOUNDRY)
+            scan_second_rule || scan_whitespace || error!
+          end
+        end
+
+        # Scans a rule.  A rule consists of a label (the nonterminal
+        # the production is for), a body, and a block; and then,
+        # an optional semicolon.
+        #
+        # @return [Boolean] if it matched
+        # @see #scan_second_rule_label
+        # @see #scan_second_rule_body
+        # @see #error!
+        def scan_second_rule
+          if @scanner.check(/([a-z]+):/)
+            scan_second_rule_label or error!
+            scan_second_rule_body
+            true
+          end
+        end
+
+        # Scans the label for a rule.  It should contain only lower
+        # case letters and a colon.
+        #
+        # @return [Boolean] if it matched.
+        def scan_second_rule_label
+          if @scanner.scan(/([a-z]+): ?/)
+            tokens << [:label, @scanner[1]]
+          end
+        end
+
+        # The body can contain parts, ors, precs, or blocks (or
+        # whitespaces).  Scans all of them, and then attempts to
+        # scan a semicolon.
+        #
+        # @return [void]
+        # @see #scan_second_rule_part
+        # @see #scan_second_rule_or
+        # @see #scan_second_rule_prec
+        # @see #scan_second_rule_block
+        # @see #scan_whitespace
+        def scan_second_rule_body
+          body = true
+          while body
+            scan_second_rule_part || scan_second_rule_or ||
+            scan_second_rule_prec || scan_second_rule_block ||
+            scan_whitespace || (body = false)
+          end
+          @scanner.scan(/;/)
+        end
+
+        # Attempts to scan a "part".  A part is any series of
+        # alphabetical characters that are not followed by a
+        # colon.
+        #
+        # @return [Boolean] if it matched.
+        def scan_second_rule_part
+          if @scanner.scan(/([A-Za-z]+)(?!\:)/)
+            tokens << [:part, @scanner[1]]
+          end
+        end
+
+        # Attempts to scan an "or".  It's just a vertical bar.
+        #
+        # @return [Boolean] if it matched.
+        def scan_second_rule_or
+          if @scanner.scan(/\|/)
+            tokens << [:or]
+          end
+        end
+
+        # Attempts to scan a presidence definition.  A presidence
+        # definition is "%prec " followed by a terminal or nonterminal.
+        #
+        # @return [Boolean] if it matched.
+        def scan_second_rule_prec
+          if @scanner.scan(/%prec ([A-Za-z]+)/)
+            tokens << [:prec, @scanner[1]]
+          end
+        end
+
+        # Attempts to scan a block.  This correctly balances brackets;
+        # however, if a bracket is opened/closed within a string, it
+        # still counts that as a bracket that needs to be balanced.
+        # So, having extensive code within a block is not a good idea.
+        #
+        # @return [Boolean] if it matched.
+        def scan_second_rule_block
+          if @scanner.scan(/\{/)
+            tokens << [:block, _scan_block]
+          end
+        end
+
+        private
+
+        # Scans the block; it scans until it encounters enough closing
+        # brackets to match the opening brackets.  If it encounters
+        # an opening brackets, it increments the bracket counter by
+        # one; if it encounters a closing bracket, it decrements by
+        # one.  It will error if it reaches the end before the
+        # brackets are fully closed.
+        #
+        # @return [String] the block's body.
+        # @raise [SyntaxError] if it reaches the end before the final
+        #   bracket is closed.
+        def _scan_block
+          brack = 1
+          body = "{"
+
+          until brack.zero?
+            if part = @scanner.scan_until(/(\}|\{)/)
+              body << part
+
+              if @scanner[1] == "}"
+                brack -= 1
+              else
+                brack += 1
+              end
+            else
+              error!
+            end
+          end
+
+          body
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/scanner/third.rb

@@ -0,0 +1,25 @@
+module Antelope
+  module Ace
+    class Scanner
+
+      # Scans the third part.  Everything after the content
+      # boundry is copied directly into the output.
+      module Third
+
+        # Scans the third part.  It should start with a content
+        # boundry; raises an error if it does not.  It then scans
+        # until the end of the file.
+        #
+        # @raise [SyntaxError] if somehow there is no content
+        #   boundry.
+        # @return [void]
+        def scan_third_part
+          @scanner.scan(CONTENT_BOUNDRY) or error!
+
+          tokens << [:third]
+          tokens << [:copy, @scanner.scan(/[\s\S]+/m) || ""]
+        end
+      end
+    end
+  end
+end

data/lib/antelope/ace/scanner.rb

@@ -0,0 +1,110 @@
+require "strscan"
+require "antelope/ace/scanner/first"
+require "antelope/ace/scanner/second"
+require "antelope/ace/scanner/third"
+
+module Antelope
+  module Ace
+
+    # Scans a given input.  The input should be a properly formatted ACE file;
+    # see the Ace module for more information.  This scanner uses the
+    # StringScanner class internally; see the ruby documentation for more on
+    # that.  This scanner seperates scanning into three seperate stages:
+    # First, Second, and Third, for each section of the file, respectively.
+    #
+    # @see Ace
+    # @see http://ruby-doc.org/stdlib-2.1.2/libdoc/strscan/rdoc/StringScanner.html
+    class Scanner
+
+      include First
+      include Second
+      include Third
+
+      # The string scanner that we're using to scan the string with.
+      #
+      # @return [StringScanner]
+      attr_reader :scanner
+
+      # An array of the tokens that the scanner scanned.
+      #
+      # @return [Array<Array<(Symbol, Object, ...)>>]
+      attr_reader :tokens
+
+      # The boundry between each section.  Placed here to be easily modifiable.
+      # **MUST** be a regular expression.
+      #
+      # @return [RegExp]
+      CONTENT_BOUNDRY = /%%/
+
+      # The value regular expression.  It should match values; for example,
+      # things quoted in strings or word letters without quotes.  Must respond
+      # to #to_s, since it is embedded within other regular expressions.  The
+      # regular expression should place the contents of the value in the
+      # groups 2 or 3.
+      #
+      # @return [#to_s]
+      VALUE = %q{(?:
+                    (?:("|')((?:\\\\|\\"|\\'|.)+?)\\1)
+                  | ([[:word:]]+)
+                )}
+
+      # Scans a file.  It returns the tokens resulting from scanning.
+      #
+      # @param source [String] the source to scan.  This should be compatible
+      #   with StringScanner.
+      # @return [Array<Array<(Symbol, Object, ...)>>]
+      # @see #tokens
+      def self.scan(source)
+        new(source).scan_file
+      end
+
+      # Initialize the scanner with the input.
+      #
+      # @param input [String] The source to scan.
+      def initialize(input)
+        @scanner = StringScanner.new(input)
+        @tokens  = []
+      end
+
+      # Scans the file in parts.
+      #
+      # @raise [SyntaxError] if the source is malformed in some way.
+      # @return [Array<Array<(Symbol, Object, ...)>>] the tokens that
+      #   were scanned in this file.
+      # @see #scan_first_part
+      # @see #scan_second_part
+      # @see #scan_third_part
+      # @see #tokens
+      def scan_file
+        scan_first_part
+        scan_second_part
+        scan_third_part
+        tokens
+      end
+
+      # Scans for whitespace.  If the next character is whitespace, it
+      # will consume all whitespace until the next non-whitespace
+      # character.
+      #
+      # @return [Boolean] if any whitespace was matched.
+      def scan_whitespace
+        @scanner.scan(/\s+/)
+      end
+
+      private
+
+      # Raises an error; first creates a small snippet to give the developer
+      # some context.
+      #
+      # @raise [SyntaxError] always.
+      # @return [void]
+      def error!
+        start = [@scanner.pos - 8, 0].max
+        stop  = [@scanner.pos + 8, @scanner.string.length].min
+        snip  = @scanner.string[start..stop].strip
+        char  = @scanner.string[@scanner.pos]
+        raise SyntaxError, "invalid syntax near `#{snip.inspect}' (#{char.inspect})"
+      end
+    end
+  end
+end

data/lib/antelope/ace/token/epsilon.rb

@@ -0,0 +1,22 @@
+module Antelope
+  module Ace
+    class Token
+
+      # Defines an epsilon token.  An epsilon token represents
+      # nothing.  This is used to say that a nonterminal can
+      # reduce to nothing.
+      class Epsilon < Token
+        # Initialize.  Technically takes no arguments.  Sets
+        # the name of the token to be `:epsilon`.
+        def initialize(*)
+          super :epsilon
+        end
+
+        # (see Token#epsilon?)
+        def epsilon?
+          true
+        end
+      end
+    end
+  end
+end