ast_ast 0.0.0 → 0.1.0

data/lib/ast_ast/token.rb

@@ -8,28 +8,71 @@ module Ast
     end
     
     # Check whether an array given is valid, ie. it has a symbol
-    # then an object only.
+    # then one or no objects only.
     #
+    # @param arr [Array, Token] 
     # @example
     #
     #   Ast::Token.valid? [:type, 'val'] #=> true
     #   Ast::Token.valid? ['wrong', 'val'] #=> false
     #   Ast::Token.valid? ['too', 'long', 1] #=> false
+    #   Ast::Token.valid? [:single] #=> true
     #
     def self.valid?(arr)
       if arr.is_a? Array
-        if arr.nil? || arr.size != 2 
-          return false
+        if arr.nil? || arr.size > 2 || arr.size == 0
+          false
         elsif !arr[0].is_a?(Symbol)
-          return false
+          false
         else
-          return true
+          true
         end
-      elsif arr.is_a? Ast::Token
-        return true
+      elsif arr.is_a? Token
+        true
       else
-        return false
+        false
       end
     end
+    
+    # Turn the Token to a String, similar to an array.
+    #
+    # @example
+    #
+    #   Ast::Token.new(:test, "str").to_s
+    #   #=> <:test "str">
+    #
+    # @return [String]
+    #
+    def to_s
+      if @value.nil?
+        "<:#{@type}>"
+      else
+        "<:#{@type}, #{@value.inspect}>"
+      end
+    end
+    
+    # Turn the Token to an Array.
+    #
+    # @example 
+    #
+    #   Ast::Token.new(:test, "str").to_a
+    #   #=> [:test, "str"]
+    #
+    # @return [Array]
+    #
+    def to_a
+      if @value.nil?
+        [@type]
+      else
+        [@type, @value]
+      end
+    end
+    
+    # Make #inspect show something a bit prettier
+    def inspect
+      self.to_s
+    end
+  
   end
+  
 end

data/lib/ast_ast/tokeniser.rb

@@ -1,17 +1,19 @@
 # @abstract
 module Ast
   class Tokeniser
-    attr_accessor :rules, :scanner
     
-    # Describes a single rule created within the Ast::Tokeniser subclass
     class Rule  
       attr_accessor :name, :regex, :block
       
       # Creates a new Rule instance
       # 
-      # @param [Symbol] name name of the token to be created
-      # @param [Regexp] regex regular expression to be matched
-      # @param [Proc] block optional block to be executed with match(es)
+      # @param name [Symbol]
+      #   Name of the token to be created.
+      # @param regex [Regexp] 
+      #    Regular expression to be matched
+      # @param block [Proc]  
+      #    Optional block to be executed with match(es)
+      #
       def initialize(name, regex, &block)
         @name = name
         @regex = regex
@@ -45,36 +47,54 @@ module Ast
       #   end
       #
       #   Klass.tokenise("split up")
-      #   #=> [[:letter, "s"], [:letter, "p"], [:letter, "l"], [:letter, "i"], [:letter, "t"], [:letter, "u"], [:letter, "p"]]
+      #   #=> [[:letter, "s"], [:letter, "p"], [:letter, "l"], [:letter, "i"], 
+      #   #     [:letter, "t"], [:letter, "u"], [:letter, "p"]]
       #
       #
       def run(val)
         arr = val.match(@regex).to_a
         val = arr unless arr.empty?
         val = arr[0] if arr.size == 1
+        val = arr[0] if arr[0] == arr[1] # this happens with /(a|b|c)/ regexs
         @block.call val
       end
     end
     
     # Creates a new Rule and adds to the +@rules+ list.
-    # @see Ast::Tokeniser::Rule#initialize
+    # @see Rule#initialize
+    #
+    # @param name [Symbol]
+    # @param regex [Regexp]
+    #
     def self.rule(name, regex, &block)
       @rules ||= []
+      # make rules with same name overwrite first rule
+      @rules.delete_if {|i| i.name == name} 
       @rules << Rule.new(name, regex, &block)
     end
     
+    # @return [Array]
+    #   Rules that have been defined.
+    #
+    def self.rules; @rules; end
+    
     # Takes the input and uses the rules that were created to scan it.
     #
-    # @param [String] input string to scan
-    # @return [Array]
+    # @param [String] 
+    #   Input string to scan.
+    #
+    # @return [Tokens]
+    #
     def self.tokenise(input)
       @scanner = StringScanner.new(input)
       
-      result = []
+      result = Tokens.new
       until @scanner.eos?
+        m = false # keep track of matches
         @rules.each do |i|
           a = @scanner.scan(i.regex)
           unless a.nil?
+            m = true # match happened
             ran = i.run(a)
             # split array into separate tokens, *not* values
             if ran.is_a? Array
@@ -84,9 +104,11 @@ module Ast
             end
           end
         end
-        # obviously no rule matches this so ignore it
-        # could add verbose mode where this throws an exception!
-        @scanner.pos += 1 unless @scanner.eos?
+        unless m # if no match happened
+          # obviously no rule matches this so ignore it
+          # could add verbose mode?
+          @scanner.pos += 1 unless @scanner.eos?
+        end
       end
       result
     end

data/lib/ast_ast/tokens.rb

@@ -1,22 +1,273 @@
 module Ast
+
+  # An Array of Token instances basically, but with added methods
+  # which add StringScanner type capabilities.
   class Tokens < Array
+    attr_accessor :prev_pos, :pos
+  
+    class Error < StandardError; end
     
+    # Creates tokens for each item given if not already and sets 
+    # pointer.
+    def initialize(args=[])
+      @pos = 0
+      return self if args == []
+      if args[0].is_a? Token
+        args.each_token do |i|
+          self << i
+        end
+      else
+        args.each do |i|
+          if i.size > 0
+            self << Token.new(i[0], i[1])
+          else
+            self << Token.new(i[0], nil)
+          end
+        end
+      end
+      self
+    end
+    
+    # Adds +val+ to self, if a Token is given it is added as expected.
+    # If an Array is given and it is valid, it will be converted to a
+    # Token and added, if invalid an error is raised.
+    #
+    # @param val [Token, .valid?]
+    # @return [Tokens]
+    #
     def <<(val)
-      raise "value given #{val} is invalid" unless Ast::Token.valid?(val)
+      raise "value given #{val} is invalid" unless Token.valid?(val)
       if val.is_a? Array
-        self << Ast::Token.new(val[0], val[1])
+        if val.size > 0
+          self << Token.new(val[0], val[1])
+        else
+          self << Token.new(val[0], nil)
+        end
       else
         super
       end
     end
     
+    # Turns the Tokens, and Token instances inside into arrays.
+    #
+    # @return [Array]
+    #
+    def to_a
+      self.collect {|i| i.to_a }
+    end
+    
+    def inspect
+      "#< #{@pos}/#{self.size-1} #{self.to_s[1..-2]} >"
+    end
+    
+    # @group Scanning Tokens
+      
+      # @return [Token] the current token being 'pointed' to
+      def pointer
+        self[@pos]
+      end
+      alias_method :curr_item, :pointer
+      
+      # Increment the pointer unless at end of tokens.
+      #
+      # @return [Integer, nil]
+      #   New position
+      #
+      def inc
+        @pos += 1 unless eot?
+      end
+      
+      # Decrement the pointer unless at first token.
+      #
+      # @return [Integer, nil]
+      #   New position
+      #
+      def dec
+        @pos -= 1 unless @pos == 0
+      end
+      
+      # Checks whether the pointer is at a token with type +type+
+      #
+      # @return [true, false]
+      #
+      def pointing_at?(type)
+        pointing_at == type
+      end
+      
+      # Gets the type of the current token.
+      #
+      # @return [Symbol]
+      #
+      def pointing_at
+        pointer.type
+      end
+      
+      # Gets a list of tokens +len+ from current position, without
+      # advancing pointer.
+      #
+      # @param len [Integer]
+      # @return [Tokens]
+      #
+      def peek(len)
+        self[@pos..(@pos+len-1)]
+      end
+      
+      # Reads the current token and advances the pointer. If a type is
+      # given it will throw an error if types do not match.
+      #
+      # @param type [Symbol] 
+      # @return [Token]
+      #
+      # @raise [Error]
+      #
+      def scan(type=nil)
+        @prev_pos = @pos
+        a = check(type)
+        inc
+        a
+      end
+      
+      # Reads the current token, but does not advance pointer. If a type
+      # is given it will throw an error if types do not match.
+      #
+      # @param type [Symbol]
+      # @return [Token]
+      #
+      # @raise [Error]
+      #
+      def check(type=nil)
+        if type.nil?
+          pointer
+        else
+          if pointing_at?(type)
+            pointer
+          else
+            raise Error, "wrong type: #{type} for #{self.pointer}"
+          end
+        end
+      end
+      
+      # Attempts to skip the current token. If type is given will only skip
+      # a token of that type, will raise error for anything else.
+      #
+      # @param type [Symbol] 
+      # @return [Integer] 
+      #   The new pointer position
+      #
+      # @raise [Error] if type of next token does not match +type+
+      #
+      def skip(type=nil)
+        @prev_pos = @pos
+        if type.nil?
+          inc
+        else
+          if pointing_at?(type)
+            inc
+          else
+            raise Error, "wrong type: #{type} for #{self.pointer}"
+          end
+        end
+      end
+      
+      # @return [boolean] whether at end of tokens
+      def eot?
+        @pos >= self.size-1
+      end
+      
+      # Scans the tokens until a token of +type+ is found. Returns tokens
+      # upto and including the matched token.
+      
+      # Reads the tokens until a token of +type+ is found. Return tokens
+      # upto and including the matched token, also advances pointer.
+      #
+      # @see #scan
+      #
+      # @param type [Symbol] 
+      # @return [Tokens]
+      #
+      def scan_until(type)
+        @prev_pos = @pos
+        r = Tokens.new
+        until pointing_at?(type) || self.eot?
+          r << scan
+        end
+        r << scan
+        r
+      end
+      
+      # Reads the tokens until a token of +type+ is found. Returns tokens
+      # upto and including the matched token, but does not advance the 
+      # pointer.
+      #
+      # @see #check
+      #
+      # @param type [Symbol]
+      # @return [Tokens]
+      #
+      def check_until(type)
+        r = Tokens.new
+        a = 0
+        until pointing_at?(type) || self.eot?
+          r << scan
+          a += 1
+        end
+        r << scan
+        @pos -= a + 1
+        r
+      end
+      
+      # Advances the pointer until token of +type+ is found.
+      #
+      # @param type [Symbol]
+      # @return [Integer] 
+      #   Number of tokens advanced, including match
+      #
+      def skip_until(type)
+        @prev_pos = @pos
+        r = 0
+        until pointing_at?(type) || self.eot?
+          inc
+          r += 1
+        end
+        inc
+        r += 1
+        r
+      end
+      
+      # @return [Tokens]
+      #   All tokens after the current token.
+      #
+      def rest
+        self[pos..-1]
+      end
+      
+      # Set the scan pointer to the end of the tokens.
+      #
+      def clear
+        @pos = self.size-1
+      end
+      
+      # Sets the pointer to the previous remembered position. Only one
+      # previous position is remembered, which is updated every scan or
+      # skip.
+      #
+      def unscan
+        if @prev_pos
+          @pos = @prev_pos
+          @prev_pos = nil
+        end
+      end
+      alias_method :unskip, :unscan
+    
+    # @endgroup
+    
     # @group Enumeration
     
       alias_method :_each, :each
+      
       # Loops through the types and contents of each tag separately, passing them
       # to the block given.
       #
-      # @return [Ast::Tokens] returns self
       # @yield [Symbol, Object] gives the type and content of each block in turn
       #
       # @example
@@ -39,16 +290,48 @@ module Ast
         self
       end
       
-      # Evalute block given for the type of each token
+      # Loops through the types of each tag, passing them to block given.
+      #
+      # @yield [Symbol]
       # @see #each
+      # 
+      # @example 
+      #  
+      #   tokens = Ast::Tokens.new
+      #   tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]
+      #
+      #   sa.each_type do |t|
+      #     puts t
+      #   end
+      #   #=> a
+      #   #=> b
+      #   #=> c
+      #   #=> d
+      #  
       def each_type(&blck)
         self._each do |i|
           yield(i.type)
         end
       end
       
-      # Evaluate block given for the value of each token
-      # @see each
+      # Loops through the values of each tag, passing them to block given.
+      #
+      # @yield [Object]
+      # @see #each
+      #
+      # @example 
+      #  
+      #   tokens = Ast::Tokens.new
+      #   tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]
+      #
+      #   sa.each_value do |v|
+      #     puts v
+      #   end
+      #   #=> 1
+      #   #=> 2
+      #   #=> 3
+      #   #=> 4
+      #  
       def each_value(&blck)
         self._each do |i|
           yield(i.value)
@@ -57,6 +340,25 @@ module Ast
       
       # Evaluate block given for each token instance
       # @see each
+      
+      # Loops through the tokens, passing them to block given.
+      #
+      # @yield [Token]
+      # @see #each
+      #
+      # @example 
+      #  
+      #   tokens = Ast::Tokens.new
+      #   tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]
+      #
+      #   sa.each_token do |token|
+      #     puts token
+      #   end
+      #   #=> <:a, 1>
+      #   #=> <:b, 2>
+      #   #=> <:c, 3>
+      #   #=> <:d, 4>
+      #  
       def each_token(&blck)
         self._each do |i|
           yield(i)