nasl 0.0.5 → 0.0.6

data/lib/nasl/parser/while.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/lib/nasl/test.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/lib/nasl/token.rb

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/lib/nasl/tokenizer.rb

@@ -1,5 +1,5 @@
 ###############################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -116,6 +116,12 @@ module Nasl
       ["}",    :RBRACE]
     ]
 
+    @@annotated = [
+      :EXPORT,
+      :FUNCTION,
+      :GLOBAL
+    ]
+
     def initialize!
       return if @@initialized
 
@@ -155,6 +161,11 @@ module Nasl
     end
 
     def reset
+      # We need to remember the last token so we only emit comments significant
+      # to nasldoc.
+      @previous = nil
+      @deferred = nil
+
       # Set tokenizer to initial state, ready to tokenize the code from the
       # start.
       @point = 0
@@ -167,7 +178,7 @@ module Nasl
 
     def skip
       while true do
-        whitespace = @line[/^(\s+|\s*#.*$)/]
+        whitespace = @line[/^\s+/]
         return if whitespace.nil?
         consume(whitespace.length)
       end
@@ -275,6 +286,25 @@ module Nasl
       return [type, contents]
     end
 
+    def get_comment
+      # Remember the column the comment begins in.
+      col = @ctx.col(@point)
+
+      # Consume all of the comments in the block.
+      block = []
+      begin
+        prev = @ctx.row(@point)
+        comment = @line[/^#.*$/]
+        break if comment.nil?
+        block << comment
+        consume(comment.length)
+        skip
+        cur = @ctx.row(@point)
+      end while @ctx.col(@point) == col && cur == prev + 1
+
+      return [:COMMENT, block.join("\n")]
+    end
+
     def get_operator
       # These are all of the operators defined in NASL. Their order is vitally
       # important.
@@ -288,6 +318,13 @@ module Nasl
     end
 
     def get_token
+      # If we deferred a token, emit it now.
+      unless @deferred.nil?
+        token = @deferred
+        @deferred = nil
+        return token
+      end
+
       # Make sure we're not at the end of the file.
       return [false, Token.new(:EOF, "$", @point...@point, @ctx)] if @eof
 
@@ -302,6 +339,8 @@ module Nasl
         get_string
       elsif @char =~ /[0-9]/
         get_integer
+      elsif @char == '#'
+        get_comment
       else
         get_operator
       end
@@ -309,17 +348,39 @@ module Nasl
       # Everything in the language is enumerated by the above functions, so if
       # we get here without a token parsed, the input file is invalid.
       die("Invalid character ('#@char')") if token.nil?
+
+      # Consume all whitespace after the token, and create an object with
+      # context.
       skip
+      token = [token.first, Token.new(*token, @mark...@point, @ctx)]
+
+      # If a comment is the first token in a file, or is followed by certain
+      # tokens, then it is considered significant. Such tokens will appear in
+      # the grammar so that it can be made visible to nasldoc.
+      if token.first == :COMMENT
+        if @previous.nil?
+          @previous = [:DUMMY, ""]
+        else
+          @previous = token
+          token = get_token
+        end
+      elsif !@previous.nil? && @previous.first == :COMMENT && @@annotated.include?(token.first)
+        @deferred = token
+        token = @previous
+        @previous = @deferred       
+      else
+        @previous = token
+      end
 
-      return [token.first, Token.new(*token, @mark...@point, @ctx)]
+      return token
     end
 
     def get_tokens
       tokens = []
 
-      until @eof
+      begin
         tokens << get_token
-      end
+      end while not tokens.last.last.type == :EOF
 
       return tokens
     end

data/lib/nasl/version.rb

@@ -1,3 +1,3 @@
 module Nasl
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

data/test/unit/parser/test_assignment.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_blank.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_block.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_call.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_comment.rb

@@ -0,0 +1,140 @@
+################################################################################
+# Copyright (c) 2011-2012, Mak Kolybabi
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+################################################################################
+
+class TestComment < Test::Unit::TestCase
+  include Nasl::Test
+
+  def test_standalone
+    tree = parse(
+      <<-EOF
+      # Standalone
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Standalone", comm.text.body)
+    assert_nil(comm.next)
+  end
+
+  def test_unattached
+    tree = parse(
+      <<-EOF
+      # Unattached
+      ;
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Unattached", comm.text.body)
+    assert_nil(comm.next)
+
+    tree = parse(
+      <<-EOF
+      # Unattached
+      global_var foo;
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Unattached", comm.text.body)
+    assert_nil(comm.next)
+  end
+
+  def test_export
+    tree = parse(
+      <<-EOF
+      foo = TRUE;
+
+      # Export
+      export function foo() {}
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Export", comm.text.body)
+    assert_equal(Nasl::Export, comm.next.class)
+  end
+
+  def test_function
+    tree = parse(
+      <<-EOF
+      foo = TRUE;
+
+      # Function
+      function foo() {}
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Function", comm.text.body)
+    assert_equal(Nasl::Function, comm.next.class)
+  end
+
+  def test_global
+    tree = parse(
+      <<-EOF
+      foo = TRUE;
+
+      # Global
+      global_var bar;
+      EOF
+    )
+    assert_not_nil(tree)
+
+    comms = tree.all(:Comment)
+    assert_not_nil(comms)
+    assert_equal(1, comms.length)
+
+    comm = comms.first
+    assert_equal("# Global", comm.text.body)
+    assert_equal(Nasl::Global, comm.next.class)
+  end
+end

data/test/unit/parser/test_constant.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_empty.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_expressions.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_function.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_if.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_include.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_incr_decr.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_ip.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_return.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_string.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/parser/test_whitespace.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/test_context.rb

@@ -1,5 +1,5 @@
 ################################################################################
-# Copyright (c) 2011, Mak Kolybabi
+# Copyright (c) 2011-2012, Mak Kolybabi
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

data/test/unit/tokenizer/test_comment.rb

@@ -0,0 +1,218 @@
+################################################################################
+# Copyright (c) 2011-2012, Mak Kolybabi
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this
+#    list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+#    this list of conditions and the following disclaimer in the documentation
+#    and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+################################################################################
+
+class TestTokenizerComment < Test::Unit::TestCase
+  include Nasl::Test
+
+  def verify(code, expected)
+    tkz = tokenize(code)
+    received = tkz.get_tokens
+
+    expected << [:EOF, "$"]
+
+    0.upto(received.length - 1) do |i|
+      tok = received[i].last
+
+      # Handle more tokens than expected.
+      if expected[i].nil?
+        assert_equal([nil, nil], [tok.type, tok.body])
+        break
+      end
+
+      assert_equal(expected[i].first, tok.type, tok.context)
+      assert_equal(expected[i].last, tok.body, tok.context)
+    end
+
+    # Handle less tokens than expected.
+    if received.length < expected.length
+      assert_equal(expected[received.length], [nil, nil])
+    end
+
+    assert_equal(expected.length, received.length)
+  end
+
+  def test_empty
+    # Tokenize empty comment.
+    0.upto(3).each do |i|
+      pad = ' ' * i
+      verify(pad + '#', [[:COMMENT, '#']])
+    end
+  end
+
+  def test_whitespace
+    # Tokenize whitespace comment.
+    0.upto(3).each do |i|
+      pad = ' ' * i
+      verify('#' + pad, [[:COMMENT, '#' + pad]])
+    end
+  end
+
+  def test_block
+    code = <<-EOF
+      # Line 1
+      # Line 2
+      # Line 3
+    EOF
+
+    verify(code, [[:COMMENT, "# Line 1\n# Line 2\n# Line 3"]])
+  end
+
+  def test_unaligned
+    code = <<-EOF
+      # Comment 1
+       # Comment 2
+    EOF
+
+    verify(code, [[:COMMENT, "# Comment 1"]])
+  end
+
+  def test_disjoint
+    code = <<-EOF
+      # Comment 1
+
+      # Comment 2
+    EOF
+
+    verify(code, [[:COMMENT, "# Comment 1"]])
+  end
+
+  def test_hidden_before
+    code = <<-EOF
+      foo = TRUE;
+      # Comment
+      foo = TRUE;
+    EOF
+
+    assign = [
+      [:IDENT, "foo"],
+      [:ASS_EQ, "="],
+      [:TRUE, "TRUE"],
+      [:SEMICOLON, ";"],
+    ]
+
+    verify(code, assign + assign) 
+  end
+
+  def test_hidden_aften
+    code = <<-EOF
+      foo = TRUE;
+
+      # Comment
+    EOF
+
+    verify(
+      code,
+      [
+        [:IDENT, "foo"],
+        [:ASS_EQ, "="],
+        [:TRUE, "TRUE"],
+        [:SEMICOLON, ";"],
+      ]
+    )
+  end
+
+  def test_export
+    code = <<-EOF
+      foo = TRUE;
+
+      # Export
+      export function bar() {}
+    EOF
+
+    verify(
+      code,
+      [
+        [:IDENT, "foo"],
+        [:ASS_EQ, "="],
+        [:TRUE, "TRUE"],
+        [:SEMICOLON, ";"],
+
+        [:COMMENT, "# Export"],
+
+        [:EXPORT, "export"],
+        [:FUNCTION, "function"],
+        [:IDENT, "bar"],
+        [:LPAREN, "("],
+        [:RPAREN, ")"],
+        [:LBRACE, "{"],
+        [:RBRACE, "}"]
+      ]
+    )
+  end
+
+  def test_function
+    code = <<-EOF
+      foo = TRUE;
+
+      # Function
+      function bar() {}
+    EOF
+
+    verify(
+      code,
+      [
+        [:IDENT, "foo"],
+        [:ASS_EQ, "="],
+        [:TRUE, "TRUE"],
+        [:SEMICOLON, ";"],
+
+        [:COMMENT, "# Function"],
+
+        [:FUNCTION, "function"],
+        [:IDENT, "bar"],
+        [:LPAREN, "("],
+        [:RPAREN, ")"],
+        [:LBRACE, "{"],
+        [:RBRACE, "}"]
+      ]
+    )
+  end
+
+  def test_global
+    code = <<-EOF
+      foo = TRUE;
+
+      # Global
+      global_var bar;
+    EOF
+
+    verify(
+      code,
+      [
+        [:IDENT, "foo"],
+        [:ASS_EQ, "="],
+        [:TRUE, "TRUE"],
+        [:SEMICOLON, ";"],
+
+        [:COMMENT, "# Global"],
+
+        [:GLOBAL, "global_var"],
+        [:IDENT, "bar"],
+        [:SEMICOLON, ";"]
+      ]
+    )
+  end
+end