ast_ast 0.2.0 → 0.2.1

data/Rakefile

@@ -39,6 +39,7 @@ namespace :release do
       end
       system "git tag v#{Ast::VERSION}"
       system 'git push origin --tags'
+      system 'git push origin master'
     else
       raise "Unstaged changes still waiting to be commited"
     end

data/lib/ast_ast/token.rb

@@ -3,7 +3,7 @@ module Ast
     attr_accessor :type, :value
     
     def initialize(type, value)
-      @type = type.to_sym
+      @type = type
       @value = value
     end
     
@@ -45,9 +45,9 @@ module Ast
     #
     def to_s
       if @value.nil?
-        "<:#{@type}>"
+        "<#{@type.inspect}>"
       else
-        "<:#{@type}, #{@value.inspect}>"
+        "<#{@type.inspect}, #{@value.inspect}>"
       end
     end
     

data/lib/ast_ast/tokeniser.rb

@@ -114,6 +114,20 @@ module Ast
       @rules << Rule.new(nil, regex, block)
     end
     
+    # Define a block to run when no match is found, as with +.token+ the block
+    # should return a token instance. The block will only be passed a single 
+    # character at a time.
+    #
+    # @example
+    #
+    #   missing do |i|
+    #     Ast::Token.new(i, i)
+    #   end
+    #
+    def self.missing(&block)
+      @missing ||= block
+    end
+    
     # Takes the input and uses the rules that were created to scan it.
     #
     # @param [String] 
@@ -122,6 +136,7 @@ module Ast
     # @return [Tokens]
     #
     def self.tokenise(input)
+      @rules ||= []
       @scanner = StringScanner.new(input)
       
       result = Tokens.new
@@ -143,9 +158,11 @@ module Ast
           end
         end
         unless m # if no match happened
-          # obviously no rule matches this so ignore it
-          # could add verbose mode?
-          @scanner.pos += 1 unless @scanner.eos?
+          # obviously no rule matches this so invoke missing if it exists
+          ch = @scanner.getch # this advances pointer as well
+          if @missing
+            result << @missing.call(ch)
+          end
         end
       end
       result

data/lib/ast_ast/version.rb

@@ -1,3 +1,3 @@
 module Ast
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/spec/ast_ast/tokeniser_spec.rb

@@ -35,18 +35,18 @@ describe Ast::Tokeniser do
 
   describe ".rule" do
     
-    class Klass1 < Ast::Tokeniser
+    class KlassRule < Ast::Tokeniser
       rule :over, /b/
     end
      
     it "adds a new rule to list" do
-      Klass1.rule(:test, /c/)
-      Klass1.rules.map {|i| i.name}.should include :test
+      KlassRule.rule(:test, /c/)
+      KlassRule.rules.map {|i| i.name}.should include :test
     end
     
     it "overwrites existing rules with same name" do
-      Klass1.rule(:over, /a/)
-      Klass1.rules.find_all {|i| i.name == :over}.size.should == 1
+      KlassRule.rule(:over, /a/)
+      KlassRule.rules.find_all {|i| i.name == :over}.size.should == 1
     end
   end
   
@@ -68,9 +68,27 @@ describe Ast::Tokeniser do
     
   end
   
+  describe ".missing" do
+  
+    class KlassMissing < Ast::Tokeniser
+      missing do |i|
+        Ast::Token.new(i, i)
+      end
+    end
+    
+    it "creates a proc" do
+      KlassMissing.missing.should be_kind_of Proc
+    end
+    
+    it "invokes the proc when a match is not found" do
+      KlassMissing.tokenise("abc").to_a.should == [["a", "a"], ["b", "b"], ["c", "c"]]
+    end
+  
+  end
+  
   describe ".tokenise" do
   
-    class Klass2 < Ast::Tokeniser
+    class KlassTokenise < Ast::Tokeniser
       
       commands = %w(git commit status)
     
@@ -92,10 +110,10 @@ describe Ast::Tokeniser do
       
     end
     
-    specify { Klass2.tokenise("").should be_kind_of Ast::Tokens }
+    specify { KlassTokenise.tokenise("").should be_kind_of Ast::Tokens }
     
     it "retuns the correct tokens" do
-      r = Klass2.tokenise("git --along -sh aword")
+      r = KlassTokenise.tokenise("git --along -sh aword")
       r.to_a.should == [[:command, "git"], [:long, "along"], [:short, "s"], [:short, "h"], [:word, "aword"]]
     end
     

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - Joshua Hawxwell