llip 0.1.0

data/examples/ariteval/exp.rb

@@ -0,0 +1,104 @@
+require 'visitable'
+
+class NumExp
+
+  include Visitable
+  
+  attr_reader :value
+
+  def initialize(value)
+    @value = value
+  end
+
+  def to_s
+    @value.to_s
+  end
+end
+
+class IdentExp
+  include Visitable
+  
+  attr_reader :value
+
+  def initialize(value)
+    @value = value
+  end
+	
+  def to_s
+    @value.to_s
+  end
+end
+
+class AssignIdentExp
+	
+  include Visitable
+
+  attr_reader :name
+  attr_reader :value
+
+  def initialize(name,value)
+    @name = name
+    @value = value
+  end
+
+  def to_s
+    "( #{@name} = #{@value} )"
+  end
+
+end
+
+class OpExp
+
+  include Visitable
+
+  attr_reader :op
+  attr_reader :left
+  attr_reader :right
+
+  def initialize(left,right)
+    @left = left
+    @right = right
+  end
+
+  def to_s
+    "( #{left.to_s} #{op} #{right.to_s} )"
+  end
+
+  def ==(other)
+    return false if other.class != self.class
+
+    left == other.left and right == other.right
+  end
+end
+
+class PlusExp < OpExp
+  def initialize(left,right)
+    super
+    @op = "+"
+  end
+end
+
+class MinusExp < OpExp
+  def initialize(left,right)
+    super
+    @op = "-"
+  end
+end
+
+class MulExp < OpExp
+  def initialize(left,right)
+    super
+    @op = "*"
+  end
+end
+
+class DivExp < OpExp
+  def initialize(left,right)
+    super
+    @op = "/"
+  end
+end
+
+
+
+

data/lib/llip.rb

@@ -0,0 +1,6 @@
+# :include: README.txt
+module LLIP
+  VERSION = "0.1.0"
+end
+
+require File.dirname(__FILE__) + '/llip/parser'

data/lib/llip/abstract_parser.rb

@@ -0,0 +1,170 @@
+require File.dirname(__FILE__) + '/production_specification'
+require File.dirname(__FILE__) + '/production_compiler'
+require File.dirname(__FILE__) + '/recursive_production_compiler'
+require File.dirname(__FILE__) + '/llip_error'
+
+module LLIP
+    
+  # This class hide all the complexity of generating an building a parser. 
+  # Ater subclassing it, it's possible to use all the methods defined in 
+  # AbstractParser::ClassMethods to specify the productions.
+  class AbstractParser
+    
+    def self.inherited(other)
+      other.extend(ClassMethods)
+    end
+    
+    def initialize
+      @hash = {}
+    end
+    
+    def productions
+      self.class.productions
+    end
+    
+    # Parse the token generated from the scanner until it reaches the end.
+    # See AbstractScanner to know how to develop a scanner.
+    def parse(scanner)
+      raise "This method hasn't been compiled yet."
+    end
+    
+    def [](key)
+      @hash[key]
+    end
+    
+    def []=(key,value)
+      @hash[key] = value
+    end
+    
+    # It raises a ParserError instead of a RuntimeError if no exception is given.
+    #
+    # It's public so it's important to call it from the production definitions, to have the exception set to ParserError.
+    def raise(*args) 
+      if args.first.respond_to? :exception or not @scanner.respond_to? :current or @scanner.current == nil
+        super(*args)
+      else
+        error = ParserError.new(@scanner.current,args.shift)
+        backtrace = args.shift
+        backtrace ||= caller(1)
+        error.set_backtrace(backtrace)
+        super error
+      end
+    end
+    
+    module ClassMethods
+      
+      # Contains the evaluated code, it's useful for debugging.
+      attr_reader :code
+      
+      # :call-seq:
+      # 	autocompile(true)
+      # 	autocompile(false)
+      #
+      # Set the autocompile flag true or false. The default is *true*.
+      # If this flag is turned on every production is automatically evaulated and converted into code. 
+      # Otherwise you can compile it using AbstractParser::ClassMethods#compile.
+      def autocompile(autocompile=nil)
+        if not autocompile.nil?
+          @autocompile = autocompile 
+        else
+          @autocompile = true if @autocompile.nil?
+        end
+        init_compile if @autocompile
+        @autocompile
+      end
+      
+      # Add a production to the parser, the block must accept an argument which is
+      # a new ProductionSpecification. 
+      # The ProductionSpecficiation name is set to the first parameter and its mode to the second if exists.
+      # A ProductionSpecification is compiled to a method named +parse_name+
+      def production(name,mode=nil) # :yields: production_specification
+        productions[name.to_sym] ||= LLIP::ProductionSpecification.new(name.to_sym)
+        productions[name.to_sym].mode = mode if mode
+        yield productions[name.to_sym]
+        compile_production(productions[name.to_sym]) if autocompile
+        name
+      end
+      
+      # Return an hash containing all the specified productions
+      def productions
+        @productions ||= {}
+      end
+      
+      # Return/set the scope, which is the first production to be called.
+      # The scope is mandatory to generate the parse method.
+      def scope(name=nil)
+        if name
+          raise ArgumentError.new("The scope must be a not empty string") if name == ""
+          @scope = name
+          compile_scope if autocompile
+        end
+        @scope
+      end
+      
+      # Compile all the productions and sets the code attribute correctly.
+      def compile
+        
+        init_compile
+        
+        #first check the scope
+        if @scope.nil? or not @productions.has_key? @scope.to_sym
+          raise "You must give a legal scope"
+        end
+        
+        compile_scope
+        
+        #compile and eval all the productions
+        @productions.values.each { |prod| compile_production(prod) }
+        
+        class_eval(@code)
+        @compiled = true
+      end
+      
+      # Returns a boolean which specify if the parser has been compiled
+      def compiled
+        @compiled ||= false
+      end
+      
+      private
+      def compile_scope
+        scope_code = <<-CODE
+				def parse(scanner)
+					@scanner = scanner
+					@scanner.next
+					result = parse_#{@scope}
+					raise "The parsing terminating without processing all tokens, the exceeding token is '\#{@scanner.current}'" unless @scanner.current.nil?
+					result
+				end
+			CODE
+        
+        class_eval(scope_code) if autocompile
+        @code << scope_code
+      end
+      
+      def compile_production(prod)
+        if prod.mode == :single
+          compiler = @single_compiler
+        elsif prod.mode == :recursive
+          compiler = @recursive_compiler
+        else
+          raise "Unknow compile mode(#{prod.mode})for production #{prod.name}"
+        end
+        
+        compiler.compile(prod)
+        @code << "\n\n"
+        @code << compiler.code
+        
+        class_eval(compiler.code) if autocompile
+        compiler.reset
+      end
+      
+      def init_compile
+        unless @code
+          @code = "" 
+          @single_compiler = LLIP::ProductionCompiler.new
+          @recursive_compiler = LLIP::RecursiveProductionCompiler.new
+        end
+      end
+    end
+  end
+end

data/lib/llip/abstract_scanner.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/token'
+require 'stringio'
+
+module LLIP
+  
+  #It's the base class of all the scanners. It handles common issues like read a multibyte char or setting up correctly an IO to read from.
+  #This class is +Abstract+ because it lacks the +next+ method, which a subclass must add.
+  class AbstractScanner
+    
+    # It contains the data scanned by the scanner. It can be nil or an IO.
+    # To set it, use scan.
+    attr_reader :source
+    
+    # It contains the last token generated by the scanner
+    attr_reader :current
+    
+    # It contains the current line of the input file.
+    # It's default is -1, and it's incremented every $-0 the AbstractScanner reads.
+    attr_reader :current_line
+    
+    # It contains the current char of the input file, it's automatically incremented for every read char.
+    # It's default is -1.
+    # For every $-0 read, it's re-initialized to -1.
+    attr_reader :current_char
+    
+    def initialize(source=nil)
+      @current_line = -1
+      @current_char = -1
+      scan(source) if source
+      @current = Token.new
+    end
+    
+    # :call-seq:
+    # 	scan(IO)
+    # 	scan(String)
+    #
+    # It initializes the scanner to scan an IO. If a String is given, it is automatically converted into a StringIO.
+    # It memorize the IO in the source attribute.
+    def scan(source)
+      @source = source
+      @source = StringIO.new(source) unless @source.respond_to? :readchar and @source.respond_to? :eof?
+      @current_line += 1
+      @current_char = -1
+      read_next 
+      self
+    end
+    
+    #	This method is abstract and it must be implemented by a subclass. 
+    # This method must read the chars to produce a Token, set it to current and return it. If a char doesn't match the first char of a token, this method should raise LLIPError. 
+    # If a char matches the first char of a token, but the next ones don't, this method should raise an UnvalidTokenError.
+    def next
+      raise NotImplementedError.new
+    end
+    
+    protected
+    
+    # It's used to recognize if a char is multibyte
+    UTF8_MB_PATTERN = /[\xc0-\xdf]$|[\xe0-\xef]$|[\xe0-\xef][\x80-\xbf]$/ #it's a little hack
+    
+    # It reads one char from the source and memorize it in the internal variabile @next_char. It handles multibyte chars correctly.
+    # If EOF is reached, it will return nil.
+    def read_next
+      unless @source.eof?
+        @next_char = @source.readchar.chr	
+        while @next_char =~ UTF8_MB_PATTERN
+          @next_char << @source.readchar.chr	
+        end
+        
+        if @next_char == $-0
+          @current_line += 1
+          @current_char = -1	
+        else
+          @current_char += 1
+        end
+        
+        @next_char
+      else
+        @next_char = nil
+      end
+    end
+    
+  end
+end

data/lib/llip/buffer.rb

@@ -0,0 +1,35 @@
+
+module LLIP
+  class Buffer
+    
+    attr_accessor :scanner
+    attr_reader :current
+    
+    def initialize(scanner)
+      @scanner = scanner
+      @current = nil
+      @buffer = nil
+    end
+    
+    def scan(text)
+      @scanner.scan(text)
+      self
+    end
+    
+    def next
+      return @current = @scanner.next unless @buffer
+      
+      @current = @buffer.shift
+      @buffer = nil if @buffer.size == 0
+      @current
+    end
+    
+    def lookahead(n)
+      @buffer ||= []
+      while @buffer.size < n	
+        @buffer << @scanner.next
+      end
+      @buffer[n-1]
+    end
+  end
+end

data/lib/llip/llip_error.rb

@@ -0,0 +1,43 @@
+
+module LLIP
+  
+  # It's the base Exception for all the exception of LLIP.
+  # It adds a header to all the messages with the line and the char of the token
+  # that caused the exception.
+  #
+  # To subclass it for a class-specific message, pass it to the constructor or
+  # override the :message method.
+  #
+  class LLIPError < StandardError
+    
+    # The token that caused the exception
+    attr_reader :token
+    
+    def initialize(token,msg=nil)
+      super msg
+      @token = token
+    end
+    
+    alias :internal_message :to_s
+    
+    def to_s
+      "At line #{token.line} char #{token.char} a #{self.class.name} occurred: #{internal_message}"
+    end
+  end
+  
+  class UnvalidTokenError < LLIPError
+    
+    def initialize(token)
+      super token, "the current token '#{token.value}' doesn't match with the regular expression #{token.name}."
+    end
+  end
+  
+  class ParserError < LLIPError
+  end
+  
+  class NotAllowedTokenError < ParserError
+    def initialize(token,production)
+      super token, "the token '#{token.value}' matched by the regexp '#{token.name}' isn't allowed in production #{production}."
+    end
+  end
+end

data/lib/llip/parser.rb

@@ -0,0 +1,93 @@
+require File.dirname(__FILE__) + '/abstract_parser'
+require File.dirname(__FILE__) + '/regexp_abstract_scanner'
+require File.dirname(__FILE__) + '/regexp_parser'
+require File.dirname(__FILE__) + '/regexp_scanner'
+require File.dirname(__FILE__) + '/buffer'
+require 'forwardable'
+
+module LLIP
+  
+  # It's a +facade+ of the LLIP library.
+  #
+  # To use it subclass it and then use the methods: production, scope and token to build the parser and its scanner.
+  class Parser
+    
+    def self.inherited(other)
+      other.extend(ClassMethods)
+      other.send(:init_parser)
+    end
+    
+    # The parser of the Parser subclass. It's created from the class returned from LLIP::Parser::ClassMethods.parser.
+    attr_reader :parser
+    
+    # The scanner of the Parser subclass. It's created from the class returned from LLIP::Parser::ClassMethods.scanner.
+    attr_reader :scanner
+    
+    def initialize
+      @parser = self.class.parser.new
+      @scanner = self.class.scanner.new
+      @scanner = Buffer.new(@scanner) if self.class.lookahead
+    end
+    
+    # Parse the source using the parser and the scanner.
+    #
+    # See AbstractScanner#scan to know what is a valid source.
+    def parse(source)
+      @parser.parse(@scanner.scan(source))
+    end
+    
+    module ClassMethods
+      
+      # A class descending from AbstractParser which will contain all the productions.
+      # The messages :production and :scope are redirected to it.
+      # See AbstractParser::ClassMethods#production, AbstractParser::ClassMethods#scope and ProductionSpecification.
+      attr_reader :parser
+      
+      # A class desceding from RegexpAbstractScanner which will contain all the token definitions.
+      # To add it in a simple way use token.
+      attr_reader :scanner
+      
+      # It's a RegexpParser
+      attr_reader :regexp_parser
+      
+      # It's a RegexpScanner
+      attr_reader :regexp_scanner
+      
+      extend Forwardable
+      
+      def_delegators :@parser, :production, :scope
+      
+      # It use _regexp_parser_ and _regexp_scanner_ to compile a correct regular expression string in a RegexpSpecification.
+      # A correct regular expression string must follow the grammar specified in RegexpParser.
+      #
+      # The first argument is the name with which all the Token derived by this regular expression will be marked. It must be a symbol.
+      def token(name,string)
+        regexp = @regexp_parser.parse(@regexp_scanner.scan(string))
+        regexp.name = name
+        @scanner.add_regexp(regexp)
+        self
+      end
+      
+      # :call-seq:
+      # 	lookahead
+      # 	lookahead(true)
+      #
+      # It allows to set the lookahead behaviour. If the lookahead is set to true, a Buffer will be used during parsing.
+      def lookahead(lookahead = nil) 
+        @lookahead = lookahead unless lookahead.nil?
+        @lookahead
+      end
+      
+      private
+      def init_parser # :nodoc:
+        @parser = Class.new(AbstractParser)
+        @scanner = Class.new(RegexpAbstractScanner)
+        
+        @regexp_scanner = RegexpScanner.new
+        @regexp_parser = RegexpParser.new
+        
+        @lookahead = false
+      end
+    end
+  end
+end