layo 1.0.0

data/lib/layo/unknown_token_error.rb

@@ -0,0 +1,7 @@
+module Layo
+  class UnknownTokenError < SyntaxError
+    def initialize(lexeme)
+      super lexeme[1], lexeme[2], "Unknown token '#{lexeme[0]}'"
+    end
+  end
+end

data/spec/interpreter_spec.rb

@@ -0,0 +1,52 @@
+require 'minitest/autorun'
+require 'layo'
+
+include Layo
+
+describe Interpreter do
+  before do
+    @interpreter = Interpreter.new
+    @interpreter.output = StringIO.new
+  end
+
+  it "should correctly interpret test programs" do
+    mask = File.join(File.dirname(__FILE__), 'source', '**', '*.lol')
+    Dir.glob(mask).each do |source_filename|
+      lexer = Lexer.new(File.new(source_filename))
+      parser = Parser.new(Tokenizer.new(lexer))
+      @interpreter.output.string = ''
+
+      # Supply input stream if provided
+      if File.exist?(infile = source_filename[0..-4] + 'in')
+        infile = File.new(infile)
+        @interpreter.input = infile
+      end
+
+      # Get contents of output file (if provided) to assert later
+      if File.exist?(outfile = source_filename[0..-4] + 'out')
+        expected_output = File.open(outfile) do |file|
+          file.read
+        end
+      else
+        expected_output = nil
+      end
+
+      # Execute the program
+      begin
+        @interpreter.interpret(parser.parse)
+      rescue RuntimeError, SyntaxError => e
+        puts "Error interpreting #{source_filename}"
+        puts e.message
+      end
+
+      # Assertions
+      if expected_output
+        assert_equal expected_output, @interpreter.output.string, "File: #{source_filename}"
+      end
+
+      # Cleanup
+      lexer.input.close
+      infile.close if infile.instance_of?(File)
+    end
+  end
+end

data/spec/lexer_spec.rb

@@ -0,0 +1,176 @@
+# encoding: UTF-8
+
+require 'stringio'
+require 'minitest/autorun'
+require 'layo'
+
+include Layo
+
+describe Lexer do
+  before do
+    @lexer = Lexer.new
+  end
+
+  it 'should transform all line-ending characters to \n' do
+    str = "\n \r\n \r "
+    @lexer.input = StringIO.new(str)
+    @lexer.next.must_equal ["\n", 1, 1]
+    @lexer.next.must_equal ["\n", 2, 2]
+    @lexer.next.must_equal ["\n", 3, 2]
+  end
+
+  it 'should recognize lexemes separated by whitespaces' do
+    @lexer.input = StringIO.new("abc  def   \t\tghi")
+    @lexer.next.must_equal ['abc', 1, 1]
+    @lexer.next.must_equal ['def', 1, 6]
+    @lexer.next.must_equal ['ghi', 1, 14]
+  end
+
+  it 'should recognize lexemes separated by newlines' do
+    @lexer.input = StringIO.new("abc\rdef\nghi")
+    @lexer.next.must_equal ['abc', 1, 1]
+    @lexer.next.must_equal ["\n", 1, 4]
+    @lexer.next.must_equal ['def', 2, 1]
+    @lexer.next.must_equal ["\n", 2, 4]
+    @lexer.next.must_equal ['ghi', 3, 1]
+  end
+
+  it 'should recognize special character lexemes' do
+    @lexer.input = StringIO.new("abc! ,def ,")
+    @lexer.next.must_equal ['abc', 1, 1]
+    @lexer.next.must_equal ['!', 1, 4]
+    # Comma acts as a virtual newline or a soft-command-break
+    @lexer.next.must_equal ["\n", 1, 6]
+    @lexer.next.must_equal ['def', 1, 7]
+    @lexer.next.must_equal ["\n", 1, 11]
+  end
+
+  describe 'when sees line ending with triple dots' do
+    it 'should join subsequent non-empty line' do
+      @lexer.input = StringIO.new("abc...\ndef…\nghi")
+      @lexer.next.must_equal ['abc', 1, 1]
+      @lexer.next.must_equal ['def', 2, 1]
+      @lexer.next.must_equal ['ghi', 3, 1]
+    end
+
+    it 'should raise a syntax error when the subsequent line is empty' do
+      @lexer.input = StringIO.new("abc...\n  \n")
+      @lexer.next
+      lambda { @lexer.next }.must_raise Layo::SyntaxError
+    end
+
+    it 'should raise a syntax error when there is not subsequent line' do
+      @lexer.input = StringIO.new("abc...\n")
+      @lexer.next
+      lambda { @lexer.next }.must_raise Layo::SyntaxError
+    end
+  end
+
+  describe 'when sees BTW' do
+    it 'should treat everything till the end of line as a comment' do
+      @lexer.input = StringIO.new("abc BTW it's comment")
+      @lexer.next.must_equal ['abc', 1, 1]
+      # Newline should be added
+      @lexer.next[0].must_equal "\n"
+      @lexer.next[0].must_be_nil
+    end
+  end
+
+  describe 'when sees OBTW' do
+    it 'should treat all lines until TLDR as a comment' do
+      @lexer.input = StringIO.new("ABC
+OBTW this is a long comment block
+      see, i have more comments here
+      and here
+TLDR
+DEF")
+      @lexer.next.must_equal ['ABC', 1, 1]
+      @lexer.next.must_equal ["\n", 1, 4]
+      @lexer.next.must_equal ["\n", 5, 5]
+      @lexer.next.must_equal ['DEF', 6, 1]
+    end
+
+    it 'should recognize commands before OBTW and after TLDR' do
+      @lexer.input = StringIO.new("ABC, OBTW
+        this is comment
+        valid comment
+TLDR, DEF")
+      @lexer.next.must_equal ['ABC', 1, 1]
+      @lexer.next.must_equal ["\n", 1, 4]
+      @lexer.next.must_equal ['DEF', 4, 7]
+    end
+
+    it 'should raise a syntax error when there is not TLDR' do
+      @lexer.input = StringIO.new('OBTW
+        comment. no tldr
+      ')
+      lambda { @lexer.next }.must_raise Layo::SyntaxError
+    end
+  end
+
+  describe 'when sees double quotation marks (")' do
+    it 'should treat everything till another " character as a string lexeme' do
+      @lexer.input = StringIO.new('"hello world"')
+      @lexer.next.must_equal ['"hello world"', 1, 1]
+    end
+
+    it 'should handle empty string' do
+      @lexer.input = StringIO.new('""')
+      @lexer.next.must_equal ['""', 1, 1]
+    end
+
+    it "should handle escape characters" do
+      @lexer.input = StringIO.new('"Special :) :::" :> :o chars ::"')
+      @lexer.next[0].must_equal %["Special \n :" \t \a chars :"]
+    end
+
+    it 'should raise a syntax error if string is unterminated' do
+      @lexer.input = StringIO.new(' "bla bla bla ')
+      lambda { @lexer.next }.must_raise Layo::SyntaxError
+    end
+
+    it 'should raise a syntax error if string terminator is not followed by allowed delimiter' do
+      @lexer.input = StringIO.new('"a","b" "c"!"d"...
+"e"…
+"f"
+"g"bla')
+      # OK, since "a" is followed by a ','
+      @lexer.next.must_equal ['"a"', 1, 1]
+      @lexer.next.must_equal ["\n", 1, 4]
+      # OK, since "b" is followed by a space
+      @lexer.next.must_equal ['"b"', 1, 5]
+      # OK, since "c" is followed by a '!'
+      @lexer.next.must_equal ['"c"', 1, 9]
+      @lexer.next.must_equal ['!', 1, 12]
+      # OK, since "d" is followed by a '...'
+      @lexer.next.must_equal ['"d"', 1, 13]
+      # OK, since "e" is followed by a '…'
+      @lexer.next.must_equal ['"e"', 2, 1]
+      # OK, since "f" is followed by a newline
+      @lexer.next.must_equal ['"f"', 3, 1]
+      @lexer.next.must_equal ["\n", 3, 4]
+      # Error, since "g" is not followed by any allowed delimiter
+      lambda { @lexer.next }.must_raise Layo::SyntaxError
+    end
+  end
+
+  describe '#escape_string' do
+    it "should handle escape characters" do
+      str = 'Example :) usage :> of :" special :o chars ::'
+      result = @lexer.escape_string(str)
+      result.must_equal %[Example \n usage \t of " special \a chars :]
+    end
+
+    it "should resolve :(<hex>) into corresponding Unicode code point" do
+      str = 'Kazakh symbol :(049b), pound sign :(0A3)'
+      result = @lexer.escape_string(str)
+      result.must_equal 'Kazakh symbol қ, pound sign £'
+    end
+
+    it "should resolve :[<char name>] into corresponding Unicode normative name" do
+      str = ':[CYRILLIC SMALL LETTER KA WITH DESCENDER] and :[COMMERCIAL AT]'
+      result = @lexer.escape_string(str)
+      result.must_equal 'қ and @'
+    end
+  end
+end

data/spec/parser_spec.rb

@@ -0,0 +1,373 @@
+require 'minitest/autorun'
+require 'mocha'
+require 'layo'
+
+include Layo
+include Layo::Ast
+
+describe Parser do
+  before do
+    @tokenizer = Tokenizer.new(Lexer.new)
+    @parser = Parser.new(@tokenizer)
+  end
+
+  it "should restore peek index after statement lookaheads" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :identifier, :data => 'abc'},
+      {:type => :is_now_a}, {:type => :newline}
+    )
+    @parser.next_statement.must_equal 'cast'
+    @tokenizer.peek.must_equal :type => :identifier, :data => 'abc'
+  end
+
+  it "should restore peek index after expression lookaheads" do
+    @tokenizer.stubs(:next_item).returns({:type => :string, :data => 'abc'})
+    @parser.next_expression.must_equal 'constant'
+    @tokenizer.peek.must_equal :type => :string, :data => 'abc'
+  end
+
+  it 'should parse program' do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :hai}, {:type => :float, :data => 1.2},
+      {:type => :newline}, {:type => :kthxbye}, {:type => :eof}
+    )
+    block = mock
+    @parser.expects(:parse_block).returns(block)
+    node = @parser.parse_program
+    node.must_be_instance_of Program
+    node.version.must_equal 1.2
+    node.block.must_be_same_as block
+  end
+
+  it "should parse block" do
+    statements = [mock]
+    @parser.stubs(:skip_newlines)
+    @parser.stubs(:parse_statement).returns(*statements)
+    @parser.stubs(:next_statement).returns('print', nil)
+    node = @parser.parse_block
+    node.must_be_instance_of Block
+    node.statement_list.must_equal statements
+  end
+
+  it "should parse cast statement" do
+    @tokenizer.stubs(:next_item).returns(
+      { type: :identifier, data: 'abc' },
+      { type: :is_now_a}, { type: :troof }
+    )
+    node = @parser.parse_cast_statement
+    node.type.must_equal 'cast'
+    node.identifier.must_equal 'abc'
+    node.to.must_equal :troof
+  end
+
+  describe "#parse_print_statement" do
+    it "should parse print statement with exclamation mark" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :visible}, {:type => :exclamation}
+      )
+      expr = mock
+      @parser.expects(:parse_expression).returns(expr)
+      @parser.stubs(:next_expression).returns(nil)
+      node = @parser.parse_print_statement
+      node.type.must_equal 'print'
+      node.expressions.size.must_equal 1
+      node.expressions.first.must_be_same_as expr
+      node.suppress.must_equal true
+    end
+
+    it "should parse print statement without exclamation mark" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :visible}
+      )
+      exprs = [mock, mock]
+      @parser.stubs(:parse_expression).returns(*exprs)
+      @parser.stubs(:next_expression).returns('constant', nil)
+      node = @parser.parse_print_statement
+      node.type.must_equal 'print'
+      node.expressions.must_equal exprs
+      node.suppress.must_equal false
+    end
+  end
+
+  it "should parse input statement" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :gimmeh}, {:type => :identifier, :data => 'var'},
+    )
+    node = @parser.parse_input_statement
+    node.type.must_equal 'input'
+    node.identifier.must_equal 'var'
+  end
+
+  it "should parse assignment statement" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :identifier, :data => 'abc'}, {:type => :r}
+    )
+    expr = mock
+    @parser.expects(:parse_expression).returns(expr)
+    node = @parser.parse_assignment_statement
+    node.type.must_equal 'assignment'
+    node.identifier.must_equal 'abc'
+    node.expression.must_be_same_as expr
+  end
+
+  describe "#parse_declaration_statement" do
+    it "should parse declaration statement without initialization" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :i_has_a}, {:type => :identifier, :data => 'abc'}
+      )
+      node = @parser.parse_declaration_statement
+      node.type.must_equal 'declaration'
+      node.identifier.must_equal 'abc'
+      node.initialization.must_be_nil
+    end
+
+    it "should parse declaration statement with initialization" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :i_has_a}, {:type => :identifier, :data => 'abc'},
+        {:type => :itz}
+      )
+      init = mock
+      @parser.expects(:parse_expression).returns(init)
+      node = @parser.parse_declaration_statement
+      node.type.must_equal 'declaration'
+      node.identifier.must_equal 'abc'
+      node.initialization.must_be_same_as init
+    end
+  end
+
+  describe "#parse_condition_statement" do
+    before do
+      @tokenizer_expectation = @tokenizer.stubs(:next_item).returns(
+        {:type => :o_rly?}, {:type => :newline},
+        {:type => :ya_rly}, {:type => :newline}
+      )
+      @block = mock
+      @parser_expectation = @parser.stubs(:parse_block).returns(@block)
+    end
+
+    it "should parse conditional statement without else's" do
+      @tokenizer_expectation.then.returns({:type => :oic})
+      node = @parser.parse_condition_statement
+      node.type.must_equal 'condition'
+      node.then.must_be_same_as @block
+      node.elseif.must_be_empty
+    end
+
+    it "should parse conditional statement with else and elseif's" do
+      @tokenizer_expectation.then.returns(
+        { type: :mebbe }, { type: :newline },
+        {:type => :no_wai}, {:type => :newline},
+        {:type => :oic}
+      )
+      elseif_condition = mock
+      elseif_block = mock
+      else_block = mock
+      @parser.stubs(:parse_expression).returns(elseif_condition)
+      @parser_expectation.then.returns(elseif_block, else_block)
+
+      node = @parser.parse_condition_statement
+
+      node.type.must_equal 'condition'
+      node.then.must_be_same_as @block
+      node.elseif.first[:condition].must_be_same_as elseif_condition
+      node.elseif.first[:block].must_be_same_as elseif_block
+      node.else.must_be_same_as else_block
+    end
+  end
+
+  it "should parse switch statement" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :wtf?}, {:type => :newline},
+      # One case
+      { type: :omg }, { type: :newline },
+      # Default case
+      {:type => :omgwtf}, {:type => :newline},
+      {:type => :oic}
+    )
+    kase_expr, kase, default = mock, mock, mock
+    @parser.expects(:parse_expression).returns(kase_expr)
+    @parser.stubs(:parse_block).returns(kase, default)
+
+    node = @parser.parse_switch_statement
+
+    node.type.must_equal 'switch'
+    node.cases.first[:expression].must_be_same_as kase_expr
+    node.cases.first[:block].must_be_same_as kase
+    node.default.must_be_same_as default
+  end
+
+  it "should parse break statement" do
+    @tokenizer.stubs(:next_item).returns({:type => :gtfo})
+    node = @parser.parse_break_statement
+    node.type.must_equal 'break'
+  end
+
+  it "should parse return statement" do
+    @tokenizer.stubs(:next_item).returns({:type => :found_yr})
+    expr = mock
+    @parser.expects(:parse_expression).returns(expr)
+    node = @parser.parse_return_statement
+    node.type.must_equal 'return'
+    node.expression.must_be_same_as expr
+  end
+
+  describe "#parse_loop_statement" do
+    it "should parse loop statement" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :im_in_yr}, {:type => :identifier, :data => 'abc'},
+        # Loop operation
+        { type: :uppin }, { type: :yr }, { type: :identifier, data: 'i' },
+        # Loop condition
+        { type: :wile },
+        {:type => :im_outta_yr}, {:type => :identifier, :data => 'abc'}
+      )
+      expr, block = mock, mock
+      @parser.expects(:parse_expression).returns(expr)
+      @parser.expects(:parse_block).returns(block)
+
+      node = @parser.parse_loop_statement
+
+      node.type.must_equal 'loop'
+      node.label.must_equal 'abc'
+      node.op.must_equal :uppin
+      node.counter.must_equal 'i'
+      node.guard[:type].must_equal :wile
+      node.guard[:expression].must_be_same_as expr
+      node.block.must_be_same_as block
+    end
+
+    it "should raise exception if labels are not equal" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :im_in_yr, :line => 1, :pos => 1},
+        {:type => :identifier, :data => 'foo'},
+        {:type => :newline}, {:type => :im_outta_yr},
+        {:type => :identifier, :data => 'bar'}
+      )
+      lambda { @parser.parse_loop_statement }.must_raise Layo::SyntaxError
+    end
+  end
+
+  it "should parse function statement" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :how_duz_i}, {:type => :identifier, :data => 'hello'},
+      {:type => :newline}, {:type => :if_u_say_so}
+    )
+    block = mock
+    @parser.expects(:parse_block).returns(block)
+    node = @parser.parse_function_statement
+    node.type.must_equal 'function'
+    node.name.must_equal 'hello'
+    node.block.must_be_same_as block
+    node.args.must_equal []
+  end
+
+  it "should parse expression statement" do
+    expr = mock
+    @parser.expects(:parse_expression).returns(expr)
+    node = @parser.parse_expression_statement
+    node.type.must_equal 'expression'
+    node.expression.must_be_same_as expr
+  end
+
+  it "should parse cast expression" do
+    @tokenizer.stubs(:next_item).returns(
+      {:type => :maek}, {:type => :a}, {:type => :troof}
+    )
+    expr = mock
+    @parser.expects(:parse_expression).returns(expr)
+    node = @parser.parse_cast_expression
+    node.type.must_equal 'cast'
+    node.being_casted.must_be_same_as expr
+    node.to.must_equal :troof
+  end
+
+  describe "#parse_constant_expressionession" do
+    it "should parse boolean values" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :boolean, :data => true},
+        {:type => :boolean, :data => false}
+      )
+      node = @parser.parse_constant_expression
+      node.type.must_equal 'constant'
+      node.vtype.must_equal :boolean
+      node.value.must_equal true
+
+      node = @parser.parse_constant_expression
+      node.value.must_equal false
+    end
+
+    it "should parse integer values" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :integer, :data => 567}
+      )
+      node = @parser.parse_constant_expression
+      node.value.must_equal 567
+    end
+
+    it "should parse float values" do
+      @tokenizer.stubs(:next_item).returns(
+        {:type => :float, :data => -5.234}
+      )
+      node = @parser.parse_constant_expression
+      node.value.must_equal -5.234
+    end
+
+    it "should parse string values" do
+      @tokenizer.stubs(:next_item).returns :type => :string, :data => 'some value'
+      node = @parser.parse_constant_expression
+      node.value.must_equal 'some value'
+    end
+  end
+
+  describe '#parse_identifier_expression' do
+    it "should parse variable expression" do
+      @tokenizer.stubs(:next_item).returns({:type => :identifier, :data => 'var'})
+      node = @parser.parse_identifier_expression
+      node.type.must_equal 'variable'
+      node.name.must_equal 'var'
+    end
+
+    it "should parse function expression" do
+      @tokenizer.stubs(:next_item).returns({:type => :identifier, :data => 'foo'})
+      exprs = [mock, mock]
+      @parser.stubs(:functions).returns({'foo' => ['arg1', 'arg2']})
+      @parser.stubs(:next_expression).returns('cast', 'constant')
+      @parser.stubs(:parse_expression).returns(*exprs)
+      node = @parser.parse_identifier_expression
+      node.type.must_equal 'function'
+      node.name.must_equal 'foo'
+      node.parameters.must_equal exprs
+    end
+  end
+
+  it "should parse unary expr" do
+    @tokenizer.stubs(:next_item).returns({:type => :not})
+    expr = mock
+    @parser.expects(:parse_expression).returns(expr)
+    node = @parser.parse_unary_expression
+    node.type.must_equal 'unary'
+    node.expression.must_be_same_as expr
+  end
+
+  it "should parse binary expr" do
+    @tokenizer.stubs(:next_item).returns({:type => :both_of})
+    expr1, expr2 = mock, mock
+    @parser.stubs(:parse_expression).returns(expr1, expr2)
+    node = @parser.parse_binary_expression
+    node.type.must_equal 'binary'
+    node.operator.must_equal :both_of
+    node.left.must_be_same_as expr1
+    node.right.must_be_same_as expr2
+  end
+
+  it "should parse nary expr" do
+    @tokenizer.stubs(:next_item).returns({:type => :any_of}, {:type => :mkay})
+    expressions = [mock, mock, mock]
+    @parser.stubs(:parse_expression).returns(*expressions)
+    @parser.stubs(:next_expression).returns('constant', 'constant', nil)
+    node = @parser.parse_nary_expression
+    node.type.must_equal 'nary'
+    node.operator.must_equal :any_of
+    node.expressions.must_equal expressions
+  end
+end