toy_lang 0.0.1 → 0.0.2

data/README.md

@@ -1,6 +1,16 @@
 # ToyLang
 
-TODO: Write a gem description
+Parser for a simple languange. An example program would be:
+
+    def fibbo(number) {
+      if number == 0 { return 0 }
+      if number == 1 { return 1 }
+      return fibbo(number-1) + fibbo(number-2)
+    }
+
+    fibbo(5)
+
+This program should output 8
 
 ## Installation
 
@@ -17,8 +27,16 @@ Or install it yourself as:
     $ gem install toy_lang
 
 ## Usage
-
-TODO: Write usage instructions here
+  
+    # Create a parser
+    @parser = ToyLang::Parser.new
+    # Set the program
+    @parser.program = "methodname(1,3)"
+    # Generate the AST
+    puts @parser.program
+
+TODO: Find better names to avoid collision between 'program =' and
+program
 
 ## Contributing
 

data/Rakefile

@@ -1,2 +1,7 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new('spec')
+
+task :default => :spec

data/lib/toy_lang/parser.rb

@@ -0,0 +1,171 @@
+module ToyLang
+
+  # This it the class that parses the toy language
+  # grammatical rules are lower cased (e.g. statement)
+  # tokens are upper case (e.g. COMMA)
+  # optional rules are surrounded by parentheses
+  #
+  # The toy language grammar is as follows
+  #
+  #   program =>
+  #     statement*
+  #   statement =>
+  #     function_definition |
+  #     conditional_expression |
+  #     function_call |
+  #     return_statement
+  #   function_definition =>
+  #     function_header OPEN_BLOCK expression* CLOSE_BLOCK
+  #   function_header =>
+  #     DEF IDENTIFIER OPEN_PARENTHESES argument_list CLOSE_PARENTHESES
+  #   argument_list =>
+  #     (IDENTIFIER ( COMMA IDENTIFIER)*)
+  #   conditional_expression =>
+  #     IF condition OPEN_BLOCK expression* CLOSE_BLOCK
+  #   expression =>
+  #     additive_expression
+  #   additive_expression =>
+  #     substraction_expression PLUS substraction_expression
+  #   substraction_expression =>
+  #     primary_expresion MINUS primary_expresion
+  #   primary_expresion =>
+  #     NUMBER
+  #   function_call =>
+  #     IDENTIFIER OPEN_PARENTHESES parameter_list CLOSE_PARENTHESES
+  #   parameter_list =>
+  #     (expression ( COMMA expression)*)
+  #   return_statement =>
+  #     RETURN expression
+  #
+  # An example program would be
+  # def fibbo(number) {
+  #   if number == 0 { return 0 }
+  #   if number == 1 { return 1 }
+  #   return fibbo(number-1) + fibbo(number-2)
+  # }
+  # fibbo(5)
+  #
+  # This program should output 8
+  class Parser
+
+    def program=(program)
+      @scanner = Scanner.new
+      @scanner.set_program(program)
+    end
+
+    # statement =>
+    #   function_definition |
+    #   conditional_expression |
+    #   function_call |
+    #   return_statement
+    def statement
+      # ast => Abstract Syntax Tree
+      if ((ast = function_definition) != nil)
+        return ast
+      elsif ((ast = conditional_expression) != nil)
+        return ast
+      elsif ((ast = function_call) != nil)
+        return ast
+      elsif ((ast = return_statement) != nil)
+        return ast
+      end
+      throw :parser_exception
+    end
+
+    # function_definition =>
+    #   function_header OPEN_BLOCK expression* CLOSE_BLOCK
+    def function_definition
+      return nil
+    end
+
+    # conditional_expression =>
+    #   IF condition OPEN_BLOCK expression* CLOSE_BLOCK
+    def conditional_expression
+      return nil
+    end
+
+    # function_call =>
+    #   IDENTIFIER OPEN_PARENTHESES parameter_list CLOSE_PARENTHESES
+    def function_call
+      unless tokens_are?(:id, :open_parentheses)
+        return nil
+      end
+
+      method_name = @scanner.get_next_token.content
+      @scanner.get_next_token # open parentheses
+      params = parameter_list()
+
+      # Verify close parentheses
+      if token_is_not? :close_parentheses
+        throw :parser_exception
+      end
+
+      @scanner.get_next_token # close parentheses
+
+      return { function_call: method_name, params: params }
+    end
+
+    # parameter_list =>
+    #   (expression ( COMMA expression)*)
+    def parameter_list
+      expression_list = []
+      expr = expression()
+      return [] if expr == nil
+
+      expression_list << expr
+
+      while (token_is? :comma)
+        @scanner.get_next_token # the comma
+        expr = expression()
+        expression_list << expr if expr != nil
+      end
+
+      expression_list
+    end
+
+    # return_statement =>
+    #   RETURN expression
+    def return_statement
+      unless token_is? :return
+        return nil
+      end
+
+      @scanner.get_next_token
+      return {return: expression()}
+    end
+
+    # expression =>
+    #   ....
+    # !!! INCOMPLETE IMPLEMENTATION !!!
+    # To get going, expression can only be a number
+    # TODO: Do it for real
+    def expression
+      if token_is_not? :number
+        nil
+      end
+
+      token = @scanner.get_next_token
+      return { number: token.content }
+    end
+
+    private
+
+    def token_is?(token)
+      tokens_are?(token)
+    end
+
+    def token_is_not?(token)
+      not token_is? token
+    end
+
+    def tokens_are?(*tokens)
+      look_ahead_index = 1
+      tokens.each do |token|
+        return false if @scanner.look_ahead(look_ahead_index).is_not? token
+        look_ahead_index += 1
+      end
+      return true
+    end
+
+  end
+end

data/lib/toy_lang/scanner.rb

@@ -0,0 +1,89 @@
+module ToyLang
+  class Scanner
+
+    # Tokens the scanner generates
+    # :return => for 'return' tokens
+    # :def => for 'def' tokens
+    # :number => for regexp '\d+'
+    # :id => for '[a-z]+'
+    # :open_block => for '{'
+    # :close_block => for '}'
+    # :eof => for end of file
+
+    IDENTIFIER = /\A[a-z]+/
+    WHITESPACE = /\A\s+/
+
+    LANGUAGE_TOKENS = {
+      number: /\A\d+/,
+      open_block: /\A\{/,
+      close_block: /\A\}/,
+      open_parentheses: /\A\(/,
+      close_parentheses: /\A\)/,
+      comma: /\A,/
+    }
+
+    RESERVED_WORDS = %w[return def]
+
+    def set_program(program)
+      @program = program
+      @token_list =[] # used to keep tokens in look_aheads
+    end
+
+    def get_next_token
+      if @token_list.empty?
+        consume_token
+      else
+        @token_list.shift
+      end
+    end
+
+    def look_ahead(number_of_tokens = 1)
+      end_of_file_met = false
+      while @token_list.size < number_of_tokens
+        throw :scanner_exception if end_of_file_met
+        token = consume_token
+        @token_list << token
+        end_of_file_met = token.is? :eof
+      end
+      @token_list[number_of_tokens - 1]
+    end
+
+    private
+
+    def identifier
+      ident = consume(IDENTIFIER)
+      # Check if the token is part of the reserved words
+      return Token.new(ident.to_sym, ident) if RESERVED_WORDS.include? ident
+      return Token.new(:id,ident)
+    end
+
+    def consume_token
+      clear_whitespace
+      if @program.size == 0
+        return Token.new(:eof)
+      elsif @program =~ IDENTIFIER
+        return identifier
+      end
+
+      # Check for language symbols
+      LANGUAGE_TOKENS.each do |symbol, reg_exp|
+        if @program =~ reg_exp
+          return Token.new(symbol, consume(reg_exp))
+        end
+      end
+
+      throw :scanner_exception # Unrecognized token
+    end
+
+    def clear_whitespace
+      consume(WHITESPACE)
+    end
+
+    def consume(regexp)
+      content = @program[regexp]
+      @program.gsub!(regexp,"")
+      content
+    end
+
+  end
+end

data/lib/toy_lang/token.rb

@@ -0,0 +1,18 @@
+module ToyLang
+  class Token
+    attr_reader :symbol, :content
+
+    def initialize(symbol, content = nil)
+      @symbol = symbol
+      @content = content
+    end
+
+    def is?(symbol)
+      @symbol == symbol
+    end
+
+    def is_not?(symbol)
+      not is? symbol
+    end
+  end
+end

data/lib/toy_lang/version.rb

@@ -1,3 +1,3 @@
 module ToyLang
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/toy_lang.rb

@@ -1,4 +1,7 @@
 require "toy_lang/version"
+require "toy_lang/scanner"
+require "toy_lang/token"
+require "toy_lang/parser"
 
 module ToyLang
   def self.description

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require 'rspec'
+require 'toy_lang'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end
+

data/spec/toy_lang/parser_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+describe ToyLang::Parser do
+
+  before(:each) do
+    @parser = ToyLang::Parser.new
+  end
+
+  describe "return statement" do
+    it "parses" do
+      @parser.program = "return 2"
+      @parser.statement.should == {return: { number: "2" }}
+    end
+  end
+
+  describe "function call" do
+    it "parses function" do
+      @parser.program = "methodname(1,3)"
+      @parser.statement.should == { function_call: 'methodname',
+                                    params: [ {number: "1"}, {number: "3"} ]}
+    end
+
+    it "throws parser_exception when no closing parentheses" do
+      @parser.program = "methodname(1,3"
+      expect { @parser.statement }.to throw_symbol :parser_exception
+    end
+
+    it "throws parser_exception when no further expression after comma" do
+      @parser.program = "methodname(1,"
+      expect { @parser.statement }.to throw_symbol :parser_exception
+    end
+
+    it "throws parser_exception when first expression empty" do
+      @parser.program = "methodname(,3)"
+      expect { @parser.statement }.to throw_symbol :parser_exception
+    end
+
+    it "throws parser_exception when middle expression empty" do
+      @parser.program = "methodname(1,,3)"
+      expect { @parser.statement }.to throw_symbol :parser_exception
+    end
+  end
+end

data/spec/toy_lang/scanner_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+describe ToyLang::Scanner do
+
+  before(:each) do
+    @scanner = ToyLang::Scanner.new
+  end
+
+  it "returns :eof when no tokens left" do
+    @scanner.set_program("")
+    assert_token_is :eof
+  end
+
+  it "clears white spaces" do
+    @scanner.set_program("  \n\t")
+    assert_token_is :eof
+  end
+
+  it "returns :return when token is 'return'" do
+    @scanner.set_program("return")
+    assert_token_is :return
+  end
+
+  it "returns :def when token is 'def'" do
+    @scanner.set_program("def")
+    assert_token_is :def
+  end
+
+  it "returns :id when token is not a reserved word" do
+    @scanner.set_program("classic")
+    assert_token_is :id
+  end
+
+  it "returns token content when token is not a reserved word" do
+    @scanner.set_program("classic")
+    assert_token_content_is "classic"
+  end
+
+  it "returns :number when token is digits" do
+    @scanner.set_program("9823")
+    assert_token_is :number
+  end
+
+  it "returns content when token is digits" do
+    @scanner.set_program("9823")
+    assert_token_content_is "9823"
+  end
+
+  it "returns :open_block when token is '{'" do
+    @scanner.set_program("{")
+    assert_token_is :open_block
+  end
+
+  it "returns :close_block when token is '}'" do
+    @scanner.set_program("}")
+    assert_token_is :close_block
+  end
+
+  it "returns :open_parentheses when token is '('" do
+    @scanner.set_program("(")
+    assert_token_is :open_parentheses
+  end
+
+  it "returns :close_parentheses when token is ')'" do
+    @scanner.set_program(")")
+    assert_token_is :close_parentheses
+  end
+
+  it "scans small program" do
+    @scanner.set_program """
+      def method {
+        return 9
+      }
+      """
+    assert_token_is :def
+    assert_token_and_content_is :id, "method"
+    assert_token_is :open_block
+    assert_token_is :return
+    assert_token_and_content_is :number, "9"
+    assert_token_is :close_block
+    assert_token_is :eof
+  end
+
+  describe "look_ahead" do
+    it "without parameters looks one ahead" do
+      @scanner.set_program("token")
+      @scanner.look_ahead.content.should == "token"
+    end
+
+    it "with parameter looks ahead 'n' tokens" do
+      @scanner.set_program("def method")
+      @scanner.look_ahead(2).content.should == "method"
+    end
+
+    it "does not consume token (e.g. get_next_token gets the next token)" do
+      @scanner.set_program("token")
+      @scanner.look_ahead
+      @scanner.get_next_token.content.should == "token"
+    end
+
+    it "looking ahead of :eof throws exception" do
+      @scanner.set_program("")
+      expect { @scanner.look_ahead(2) }.to throw_symbol :scanner_exception
+    end
+  end
+
+  def assert_token_content_is(content)
+    @scanner.get_next_token.content.should == content
+  end
+
+  def assert_token_is(symbol)
+    @scanner.get_next_token.symbol.should be symbol
+  end
+
+  def assert_token_and_content_is(symbol, content)
+    token = @scanner.get_next_token
+    token.symbol.should be symbol
+    token.content.should == content
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: toy_lang
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-22 00:00:00.000000000 Z
+date: 2012-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -56,7 +56,13 @@ files:
 - README.md
 - Rakefile
 - lib/toy_lang.rb
+- lib/toy_lang/parser.rb
+- lib/toy_lang/scanner.rb
+- lib/toy_lang/token.rb
 - lib/toy_lang/version.rb
+- spec/spec_helper.rb
+- spec/toy_lang/parser_spec.rb
+- spec/toy_lang/scanner_spec.rb
 - toy_lang.gemspec
 homepage: ''
 licenses: []
@@ -72,7 +78,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -795914555458179842
+      hash: -3501641450620113713
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -81,11 +87,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -795914555458179842
+      hash: -3501641450620113713
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Toy Language parser and scanner to play with language compilation
-test_files: []
+test_files:
+- spec/spec_helper.rb
+- spec/toy_lang/parser_spec.rb
+- spec/toy_lang/scanner_spec.rb