ast_ast 0.0.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,24 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC
+doc/
+cov/
+.yardoc/

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Joshua Hawxwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# AstAst
+
+
+                    sSSSSs
+              saaAAA     Tttttts
+             sa   tT  t  TT    tt
+      saaaaaaA  t tT  t  TT     Ts
+     sa  tt  T    tT  t  TT     Ts   
+     AaaaaaaAaaaaAAt     TsssssTs
+       tT      t tSTSsssSTt tt
+               t tt       t tt
+              /t tt      /t tt  
+             ( t tt     ( t tt  
+              \t tt      \t tt  
+               t tt       t tt
+               t tt\      t tts
+               S tS.`.    S tS ss
+              tsssst-'   tsssstSSS
+
+
+
+
+__VERY IMPORTANT:__ it is probably a very bad idea to use this in something that relies on it. It will change without warning!
+
+## Goals/Ideas
+
+Crazy simple string -> token converting, using regular expression rules and optional blocks. Some of the finer points of this still need working out, mainly should you be able to affect the name of the token within the block.
+
+    class MyTokeniser < Ast::Tokeniser
+      rule :long, /--[a-zA-Z0-9]+/
+      rule :short, /-[a-zA-Z0-9]+/
+      rule :word, /[a-zA-Z0-9]+/
+    end
+    input = "--along -sh aword"
+    MyTokeniser.tokenise(input)
+    #=> #<Ast::Tokens [[:long, "--along"], [:short, "-sh"], [:word, "aword"]]>
+    
+    # Use blocks to change results, passes matches
+    class MyTokeniser < Ast::Tokeniser
+      rule :long, /--([a-zA-Z0-9]+)/ {|i| i[1]}
+      rule :short, /-([a-zA-Z0-9]+)/ {|i| i[1].split} # creates an array so splits into multiple tokens
+      rule :word, /[a-zA-Z0-9]+/
+    end
+    input = "--along -sh aword"
+    MyTokeniser.tokenise(input)
+    #=> #<Ast::Tokens [[:long, "along"], [:short, "s"], [:short, "h"], [:word, "aword"]]>
+    
+
+## Copyright
+
+Copyright (c) 2010 Joshua Hawxwell. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "ast_ast"
+    gem.summary = %Q{String -> Tokens -> AST}
+    gem.description = %Q{Easily convert strings to Tokens and then on to an Abstract Syntax Tree easily. (Very far from finished!)}
+    gem.email = "m@hawx.me"
+    gem.homepage = "http://github.com/hawx/ast_ast"
+    gem.authors = ["Joshua Hawxwell"]
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "yard", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.0

data/lib/ast_ast.rb

@@ -0,0 +1,10 @@
+# Loads everything
+$:.unshift File.dirname(__FILE__)
+
+require 'strscan'
+
+require 'ast_ast/ast'
+require 'ast_ast/tree'
+require 'ast_ast/tokeniser'
+require 'ast_ast/tokens'
+require 'ast_ast/token'

data/lib/ast_ast/ast.rb

@@ -0,0 +1,5 @@
+module Ast
+  class Ast
+  
+  end
+end

data/lib/ast_ast/token.rb

@@ -0,0 +1,35 @@
+module Ast
+  class Token
+    attr_accessor :type, :value
+    
+    def initialize(type, value)
+      @type = type.to_sym
+      @value = value
+    end
+    
+    # Check whether an array given is valid, ie. it has a symbol
+    # then an object only.
+    #
+    # @example
+    #
+    #   Ast::Token.valid? [:type, 'val'] #=> true
+    #   Ast::Token.valid? ['wrong', 'val'] #=> false
+    #   Ast::Token.valid? ['too', 'long', 1] #=> false
+    #
+    def self.valid?(arr)
+      if arr.is_a? Array
+        if arr.nil? || arr.size != 2 
+          return false
+        elsif !arr[0].is_a?(Symbol)
+          return false
+        else
+          return true
+        end
+      elsif arr.is_a? Ast::Token
+        return true
+      else
+        return false
+      end
+    end
+  end
+end

data/lib/ast_ast/tokeniser.rb

@@ -0,0 +1,95 @@
+# @abstract
+module Ast
+  class Tokeniser
+    attr_accessor :rules, :scanner
+    
+    # Describes a single rule created within the Ast::Tokeniser subclass
+    class Rule  
+      attr_accessor :name, :regex, :block
+      
+      # Creates a new Rule instance
+      # 
+      # @param [Symbol] name name of the token to be created
+      # @param [Regexp] regex regular expression to be matched
+      # @param [Proc] block optional block to be executed with match(es)
+      def initialize(name, regex, &block)
+        @name = name
+        @regex = regex
+        @block = block || Proc.new {|i| i}
+      end
+      
+      # Runs the block that was given using either, the full match if there
+      # were no captures in the regex, or an array of the captures.
+      #
+      # If a String is returned, which will be the case most of the time, a
+      # single token is created with the value that is returned. But if an 
+      # Array is returned then multiple tokens will be created for each item 
+      # in the Array. See the examples for a better explanation.
+      #
+      # @param [String] val the string that was matched to +@regex+
+      # @return [String, Array]
+      #
+      # @example Single tokens created (returns String)
+      #
+      #   class Klass < Ast::Tokeniser
+      #     rule(:word, /[a-z]+/) {|i| i.reverse}
+      #   end
+      #
+      #   Klass.tokenise("backwards sdrawrof")
+      #   #=> [[:word, "sdrawkcab"], [:word, "forwards"]]
+      #
+      # @example Multiple tokens created (returns Array)
+      #
+      #   class Klass < Ast::Tokeniser
+      #     rule(:letter, /[a-z]+/) {|i| i.split('')}
+      #   end
+      #
+      #   Klass.tokenise("split up")
+      #   #=> [[:letter, "s"], [:letter, "p"], [:letter, "l"], [:letter, "i"], [:letter, "t"], [:letter, "u"], [:letter, "p"]]
+      #
+      #
+      def run(val)
+        arr = val.match(@regex).to_a
+        val = arr unless arr.empty?
+        val = arr[0] if arr.size == 1
+        @block.call val
+      end
+    end
+    
+    # Creates a new Rule and adds to the +@rules+ list.
+    # @see Ast::Tokeniser::Rule#initialize
+    def self.rule(name, regex, &block)
+      @rules ||= []
+      @rules << Rule.new(name, regex, &block)
+    end
+    
+    # Takes the input and uses the rules that were created to scan it.
+    #
+    # @param [String] input string to scan
+    # @return [Array]
+    def self.tokenise(input)
+      @scanner = StringScanner.new(input)
+      
+      result = []
+      until @scanner.eos?
+        @rules.each do |i|
+          a = @scanner.scan(i.regex)
+          unless a.nil?
+            ran = i.run(a)
+            # split array into separate tokens, *not* values
+            if ran.is_a? Array
+              ran.each {|a| result << [i.name, a]}
+            else
+              result << [i.name, ran]
+            end
+          end
+        end
+        # obviously no rule matches this so ignore it
+        # could add verbose mode where this throws an exception!
+        @scanner.pos += 1 unless @scanner.eos?
+      end
+      result
+    end
+
+  end
+end

data/lib/ast_ast/tokens.rb

@@ -0,0 +1,69 @@
+module Ast
+  class Tokens < Array
+    
+    def <<(val)
+      raise "value given #{val} is invalid" unless Ast::Token.valid?(val)
+      if val.is_a? Array
+        self << Ast::Token.new(val[0], val[1])
+      else
+        super
+      end
+    end
+    
+    # @group Enumeration
+    
+      alias_method :_each, :each
+      # Loops through the types and contents of each tag separately, passing them
+      # to the block given.
+      #
+      # @return [Ast::Tokens] returns self
+      # @yield [Symbol, Object] gives the type and content of each block in turn
+      #
+      # @example
+      #  
+      #   tokens = Ast::Tokens.new
+      #   tokens << [:a, 1] << [:b, 2] << [:c, 3] << [:d, 4]
+      #
+      #   sa.each do |t, v|
+      #     puts "#{t} -> #{v}"
+      #   end
+      #   #=> a -> 1
+      #   #=> b -> 2
+      #   #=> c -> 3
+      #   #=> d -> 4
+      #  
+      def each(&blck)
+        self._each do |i|
+          yield(i.type, i.value)
+        end
+        self
+      end
+      
+      # Evalute block given for the type of each token
+      # @see #each
+      def each_type(&blck)
+        self._each do |i|
+          yield(i.type)
+        end
+      end
+      
+      # Evaluate block given for the value of each token
+      # @see each
+      def each_value(&blck)
+        self._each do |i|
+          yield(i.value)
+        end
+      end
+      
+      # Evaluate block given for each token instance
+      # @see each
+      def each_token(&blck)
+        self._each do |i|
+          yield(i)
+        end
+      end
+    
+    # @endgroup
+    
+  end
+end

data/lib/ast_ast/tree.rb

@@ -0,0 +1,5 @@
+module Ast
+  class Tree
+  
+  end
+end

data/lib/ast_tokens.rb

@@ -0,0 +1,8 @@
+# Loads only token stuff
+$:.unshift File.dirname(__FILE__)
+
+require 'strscan'
+
+require 'ast_ast/tokeniser'
+require 'ast_ast/tokens'
+require 'ast_ast/token'

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'ast_ast'
+
+class Test::Unit::TestCase
+end

data/test/test_ast_ast.rb

@@ -0,0 +1,5 @@
+require 'helper'
+
+class TestAstAst < Test::Unit::TestCase
+
+end

data/test/test_tokeniser.rb

@@ -0,0 +1,13 @@
+require 'helper'
+
+class TestTokeniser < Test::Unit::TestCase
+  context "When trying examples" do
+    should "run first" do
+      class Klass < Ast::Tokeniser
+        rule(:word, /[a-z]+/) {|i| i.reverse}
+      end
+      result = [[:word, "sdrawkcab"], [:word, "forwards"]]
+      assert_equal result, Klass.tokenise("backwards sdrawrof")
+    end
+  end
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification 
+name: ast_ast
+version: !ruby/object:Gem::Version 
+  hash: 31
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 0
+  version: 0.0.0
+platform: ruby
+authors: 
+- Joshua Hawxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-09-28 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: yard
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: Easily convert strings to Tokens and then on to an Abstract Syntax Tree easily. (Very far from finished!)
+email: m@hawx.me
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.md
+- Rakefile
+- VERSION
+- lib/ast_ast.rb
+- lib/ast_ast/ast.rb
+- lib/ast_ast/token.rb
+- lib/ast_ast/tokeniser.rb
+- lib/ast_ast/tokens.rb
+- lib/ast_ast/tree.rb
+- lib/ast_tokens.rb
+- test/helper.rb
+- test/test_ast_ast.rb
+- test/test_tokeniser.rb
+has_rdoc: true
+homepage: http://github.com/hawx/ast_ast
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: String -> Tokens -> AST
+test_files: 
+- test/helper.rb
+- test/test_ast_ast.rb
+- test/test_tokeniser.rb