ast_ast 0.0.0 → 0.1.0

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009 Joshua Hawxwell
+Copyright (c) 2009-2010 Joshua Hawxwell
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -1,50 +1,120 @@
 # AstAst
 
 
-                    sSSSSs
-              saaAAA     Tttttts
-             sa   tT  t  TT    tt
-      saaaaaaA  t tT  t  TT     Ts
-     sa  tt  T    tT  t  TT     Ts   
-     AaaaaaaAaaaaAAt     TsssssTs
-       tT      t tSTSsssSTt tt
-               t tt       t tt
-              /t tt      /t tt  
-             ( t tt     ( t tt  
-              \t tt      \t tt  
-               t tt       t tt
-               t tt\      t tts
-               S tS.`.    S tS ss
-              tsssst-'   tsssstSSS
+                          sSSSSs
+                    saaAAA     Tttttts
+                   sa   tT  t  TT    tt
+            saaaaaaA  t tT  t  TT     Ts
+           sa  tt  T    tT  t  TT     Ts   
+     - -  AaaaaaaAaaaaAAt     TsssssTs
+             tT      t tSTSsssSTt tt
+                     t tt       t tt
+                    st tt      st tt  
+                   S t tt     S t tt  
+                    st tt      st tt  
+                     t tt       t tt
+                     t tts      t tts
+                     S tS s     S tS ss
+                    tsssstss   tsssstSSS
+  
 
 
+## How To 
+### String -> Ast::Tokens
 
+So you have a string, eg:
 
-__VERY IMPORTANT:__ it is probably a very bad idea to use this in something that relies on it. It will change without warning!
+    an example String, lorem!
+
+And you want to turn it into a set of tokens, for some reason, but can't be bothered messing around with `strscan` so instead use `Ast::Tokeniser`
+
+    string = "an example String, lorem!"
+    
+    class StringTokens < Ast::Tokeniser
+    
+      # A rule uses a regular expression to match against the string given
+      # if it matches a token is created with the name given, eg. +:article+
+      rule :article, /an|a|the/
+      rule :word,    /[a-z]+/
+      rule :punct,   /,|\.|!/
+      
+      # A rule can be passed a block that then modifies the match and returns
+      # something new in it's place, here we are removing the capital.
+      rule :pronoun, /[A-Z][a-z]+/ do |i|
+        i.downcase
+      end
+    end
+    
+    StringTokens.tokenise(string)
+    #=> #< [0] <:article, "an">, <:word, "example">, <:pronoun, "string">, <:punct, ",">, <:word, "lorem">, <:punct, "!"> >
+    
+
+### Ast::Tokens -> Ast::Tree
+
+Later.
 
 ## Goals/Ideas
 
-Crazy simple string -> token converting, using regular expression rules and optional blocks. Some of the finer points of this still need working out, mainly should you be able to affect the name of the token within the block.
+Now that it is possible to take a string and turn it into a set of tokens, I want to be able to take the tokens and turn them into a tree structure. This should be easy to write using a similar DSL to Tokeniser. See below for an idea on how this might be done, though of course when I start writing it, it will change _a lot_.
+ 
+### Ast::Ast
+
+Imagine we have a string:
 
-    class MyTokeniser < Ast::Tokeniser
-      rule :long, /--[a-zA-Z0-9]+/
-      rule :short, /-[a-zA-Z0-9]+/
-      rule :word, /[a-zA-Z0-9]+/
+    string = <<EOS
+    def method
+      print 'hi'
     end
-    input = "--along -sh aword"
-    MyTokeniser.tokenise(input)
-    #=> #<Ast::Tokens [[:long, "--along"], [:short, "-sh"], [:word, "aword"]]>
+    EOS
+
+Which becomes these tokens:
+
+    tokens #=> [:defn], [:id, 'method'], [:id, 'print'], [:string, 'Hi'], [:end]
+
+We're looking for a tree like this:
+  
+    tree #=> [:defn, 'method', [
+      [:id, 'print', [
+        [:string, 'Hi']
+      ]]
+    ]]
+
+Then the class could look something like (something being the keyword):
+
+    class MyAst < Ast::Ast
     
-    # Use blocks to change results, passes matches
-    class MyTokeniser < Ast::Tokeniser
-      rule :long, /--([a-zA-Z0-9]+)/ {|i| i[1]}
-      rule :short, /-([a-zA-Z0-9]+)/ {|i| i[1].split} # creates an array so splits into multiple tokens
-      rule :word, /[a-zA-Z0-9]+/
+      # create a defn token
+      token :defn do
+        [
+          # start with :defn
+          :defn,
+          
+          # get the name of method by reading next :id
+          read_next(:id), # if not :id throw error
+          
+          # read rest of block, until the matching :end
+          [read_until(:end)]
+        ]
+      end
+      
+      # allows you to use the name given in place of a list of token names
+      group :literal, [:string, :integer, :float]
+      # really just creates an array which responds to name given
+      group :defined, [:print, :puts, :putc, :gets, :getc]
+      
+      token :id do |t, v|
+        case v
+        when 'method'
+          v
+        when :defined
+          [:call, v, [read_next(:literal)]]
+        else
+          [t, v]
+        end
+      end
+      
+      token(:string) {|i| i } # not really necessary
     end
-    input = "--along -sh aword"
-    MyTokeniser.tokenise(input)
-    #=> #<Ast::Tokens [[:long, "along"], [:short, "s"], [:short, "h"], [:word, "aword"]]>
-    
 
 ## Copyright
 

data/Rakefile

@@ -1,53 +1,10 @@
-require 'rubygems'
 require 'rake'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gem|
-    gem.name = "ast_ast"
-    gem.summary = %Q{String -> Tokens -> AST}
-    gem.description = %Q{Easily convert strings to Tokens and then on to an Abstract Syntax Tree easily. (Very far from finished!)}
-    gem.email = "m@hawx.me"
-    gem.homepage = "http://github.com/hawx/ast_ast"
-    gem.authors = ["Joshua Hawxwell"]
-    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
-    gem.add_development_dependency "yard", ">= 0"
-    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
-end
-
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
-task :test => :check_dependencies
-
-task :default => :test
+require File.expand_path('../lib/ast_ast/version', __FILE__)
 
-begin
-  require 'yard'
-  YARD::Rake::YardocTask.new
-rescue LoadError
-  task :yardoc do
-    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+namespace :release do
+  task :tag do
+    system("git tag v#{Ast::VERSION}")
+    system('git push origin --tags')
   end
 end

data/lib/ast_ast.rb

@@ -1,10 +1,10 @@
-# Loads everything
-$:.unshift File.dirname(__FILE__)
+$: << File.dirname(__FILE__)
 
 require 'strscan'
 
-require 'ast_ast/ast'
-require 'ast_ast/tree'
+# require 'ast_ast/ast'
+# require 'ast_ast/tree'
+
 require 'ast_ast/tokeniser'
-require 'ast_ast/tokens'
-require 'ast_ast/token'
+require 'ast_ast/token'
+require 'ast_ast/tokens'

data/lib/ast_ast/ast.rb

@@ -1,5 +1,166 @@
 module Ast
   class Ast
+    attr_accessor :tokens, :token_descs, :block_descs, :groups
+    
+    # @see Ast::Ast#token
+    class TokenDesc
+      attr_accessor :name, :block
+    
+      def initialize(name, &block)
+        @name = name
+        @block = block || Proc.new {|t| t}
+      end
+    end
+    
+    # @see Ast::Ast#group
+    class Group
+      attr_accessor :name, :items
+      
+      def initialize(name, items)
+        @name = name
+        @items = items
+      end
+      
+      # @see Array#include?
+      def include?(arg)
+        @items.include?(arg)
+      end
+    end
+    
+    # @see Ast::Ast#block
+    class BlockDesc
+      attr_accessor :open, :close, :block
+      
+      def initialize(open, close, &block)
+        @open, @close = open, close
+        @block = block || Proc.new {|b| b}
+      end
+    end
+    
+    # Creates a new token within the subclass. The block is executed
+    # when the token is found during the execution of #astify.
+    #
+    # @example
+    #
+    #   class TestAst < Ast::Ast
+    #     token :test do
+    #       p 'test'
+    #     end
+    #   end
+    #
+    def self.token(name, &block)
+      @token_descs ||= []
+      @token_descs << TokenDesc.new(name, &block)
+    end
+    
+    # Creates a new group of token types, this allows you to refer 
+    # to multiple tokens easily.
+    #
+    # @example
+    #
+    #     group :names, [:john, :dave, :josh]
+    #
+    def self.group(name, items)
+      @groups ||= []
+      @groups << Group.new(name, items)
+    end
+    
+    # Creates a block which begins with a certain token and ends with 
+    # different token.
+    #
+    # @example
+    #
+    #     block :begin => :end do |r|
+    #       ...
+    #     end
+    #
+    def self.block(t, &block)
+      @block_descs ||= []
+      @block_descs << BlockDesc.new(t.keys[0], t.values[0], &block)
+    end
+    
+    
+    def self.astify(tokens)
+      @tokens = tokens
+      t = find_block
+      t = run_tokens(t, @token_descs)
+    end
+    
+    def self.run_tokens(tok, descs)
+      r = []
+      @curr_tree = tok
+      
+      until tok.eot?
+        i = tok.scan
+        case i
+        when Token
+          # run the token
+          _desc = descs.find_all{|j| j.name == i.type}[0]
+          if _desc
+            r << _desc.block.call(i)
+          else
+            r << i
+          end
+        when Tree
+          # run the whole branch
+          r << run_tokens(i, descs)
+        end
+      end
+
+      r
+    end
+    
+    def self.find_block(curr_desc=nil)
+      body = Tree.new
+      
+      until @tokens.eot?
+        # Check if closes current search
+        if curr_desc && curr_desc.close == @tokens.curr_item.type
+          @tokens.inc
+          return body
+        
+        # Check if close token in wrong place
+        elsif @block_descs.map(&:close).include?(@tokens.curr_item.type)
+          raise "close found before open: #{@tokens.curr_item}"
+        
+        # Check if open token
+        elsif @block_descs.map(&:open).include?(@tokens.curr_item.type)
+          _desc = @block_descs.find_all {|i| i.open == @tokens.curr_item.type }[0]
+          @tokens.inc
+          found = find_block(_desc)
+          body << Tree.new(_desc.block.call(found))
+        
+        # Otherwise add to body, and start with next token
+        else
+          body << @tokens.curr_item
+          @tokens.inc
+        end
+      end
+      
+      body
+    end
+    
+    # @group For #token block
+    
+      # @see Tokens#scan
+      def self.scan(type=nil)
+        @curr_tree.scan(type)
+      end
+      
+      # @see Tokens#check
+      def self.check(type=nil)
+        @curr_tree.check(type)
+      end
+      
+      # @see Tokens#check
+      def self.scan_until(type)
+        @curr_tree.scan_until(type)
+      end
+      
+      # @return [Ast::Tree] current tree being read
+      def self.curr_tree
+        @curr_tree
+      end
   
   end
-end
+end

data/lib/ast_ast/bnf.rb

@@ -0,0 +1,187 @@
+$: << File.dirname(__FILE__)
+require 'token'
+
+module Ast
+
+  # Allows you to describe the tree using BNF style syntax.
+  #
+  # In normal BNF you would write something like:
+  #
+  #   <LETTER> ::= a|b|c|d|...|X|Y|Z
+  #   <WORD>   ::= <WORD><LETTER>|<LETTER>
+  #   <QUOTE>  ::= ' 
+  #   <STRING> ::= <QUOTE><WORD><QUOTE>
+  #
+  # With Ast::BNF, assuming you have the correct tokens, it would
+  # become:
+  #
+  #   define "Word",  ["Word", :letter], :letter
+  #   define "String",  [:quote, "Word", :quote]
+  #
+  class BNF
+    attr_accessor :tokens, :defs
+  
+    class Definition
+      attr_accessor :name, :rules
+      
+      def initialize(name, rules, klass)
+        @name = name
+        @rules = rules.map {|i| i.is_a?(Array) ? i : [i] }
+        @klass = klass
+      end
+      
+      # Gets the order of the Definition, this does require
+      # access to the other definitions. Here's why:
+      #
+      # The order of a definition is basically how many (max) 
+      # times would you have to loop thorugh to get to a 
+      # terminal rule. So from the example below,
+      #
+      #   <LETTER> ::= a|b|c|d|...|X|Y|Z       #=> terminal
+      #   <WORD>   ::= <WORD><LETTER>|<LETTER> #=> 1st order
+      #   <STRING> ::= '<WORD>'                #=> 2nd order
+      #
+      # Here it is easy to see that <LETTER> is terminal, no 
+      # other rule will have to be looked at to determine if 
+      # something is a <LETTER>. For a <WORD> you have to look
+      # at the <LETTER> definition, so this is 1st order. And 
+      # for <STRING>, you need to look at <WORD> which in turn
+      # looks at <LETTER>, so you are going back 2 steps.
+      #
+      # @return [Integer] order of definition
+      #
+      def order
+        if terminal?
+          0
+        elsif self_referential?
+          1
+        else
+          r = 0
+          @rules.each do |rule|
+            # Only interested in rule with recursion
+            if rule.size > 1
+              rule.each do |elem|
+                # Only interested in references
+                if elem.is_a? String
+                  b = @klass.defs.find_all {|i| i.name == elem}[0].order + 1
+                  r = b if b > r # swap if higher
+                end
+              end
+            end
+          end
+          r
+        end
+      end
+      
+      # A terminal defintion does not reference any other 
+      # definitions. This is largely irrelevent as Ast::Tokeniser
+      # should take care of this but it may be useful in some 
+      # cases.
+      #
+      # @return [Boolean] whether contains just terminal elements
+      #
+      def terminal?
+        @rules.each do |r|
+          if r.is_a? Array
+            r.each do |i|
+             return false if i.is_a? String
+            end
+          end
+        end
+        true
+      end
+      
+      # A Definition is self referential if the only refernce to
+      # another rule is to itself or if the other references are
+      # to terminal rule.
+      #
+      # This is not a perfect definition of what "self referential"
+      # really means but it does help when finding the order!
+      #
+      # @return [Boolean] whether the definition is self referential 
+      #
+      def self_referential?
+        r = false
+        @rules.each do |rule|
+          rule.each do |elem|
+            if elem == @name
+              r = true
+            else
+              k = @klass.defs.find_all{|i| i.name == elem}[0]
+              if k && k.terminal?
+                r = true
+              else
+                return false  
+              end
+            end
+          end
+        end
+        r
+      end
+      
+      def inspect; "#<Ast::BNF::Definition #{@name}>"; end
+      
+    end
+    
+    def initialize(name, &block)
+      @block = block
+    end
+    
+    def to_tree(tokens)
+      self.instance_eval(&@block)
+      
+      # get matrix of defs in order by order
+      defs_orders = @defs.collect {|i| [i.order, i]}
+      ordered_defs = []
+      defs_orders.each do |i|
+        ordered_defs[i[0]] ||= []
+        ordered_defs[i[0]] << i[1]
+      end
+        
+      result = []
+      ordered_defs.each do |order|
+        
+        order.each do |definition|
+          c = tokens.scan
+          
+          definition.rules.each do |rule|
+            list = tokens.peek(rule.size)
+            
+            res = []
+            rule.zip(list) do |(a, b)|
+              next if b.nil?
+              if a == b.type
+                res << b.value
+              end
+            end
+            next if res.size != rule.size
+            p [definition.name, res.join('')]
+          end
+        end
+      end
+
+      tokens
+    end
+    
+    def define(name, *args)
+      @defs ||= []
+      @defs << Definition.new(name, args, self)
+    end
+    
+  end
+end
+
+
+# This is here for testing only! Better name is required
+def bnf_definition(name, &block)
+  Ast::BNF.new(name, &block)
+end
+
+test = bnf_definition('hello') do
+  define "Digit",   :number
+  define "Letter",  :letter
+  define "Number",  ["Number", "Digit"], "Digit"
+  define "Word",    ["Word", "Letter"], "Letter"
+  define "String",  [:quote, "Word", :quote]
+end
+p test.to_tree Ast::Tokens.new([[:letter, 'a'], [:letter, 'b'], [:number, '5'], [:number, '9']])