gherkin 1.0.30-i386-mingw32 → 2.0.0-i386-mingw32

data/spec/gherkin/parser/tag_expression_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec_helper'
 require 'gherkin/parser/tag_expression'
 
 module Gherkin
@@ -6,7 +6,7 @@ module Gherkin
     describe TagExpression do
       context "no tags" do
         before(:each) do
-          @e = TagExpression.new([])
+          @e = Gherkin::Parser::TagExpression.new([])
         end
 
         it "should match @foo" do
@@ -20,7 +20,7 @@ module Gherkin
 
       context "@foo" do
         before(:each) do
-          @e = TagExpression.new(['@foo'])
+          @e = Gherkin::Parser::TagExpression.new(['@foo'])
         end
 
         it "should match @foo" do
@@ -38,7 +38,7 @@ module Gherkin
 
       context "!@foo" do
         before(:each) do
-          @e = TagExpression.new(['~@foo'])
+          @e = Gherkin::Parser::TagExpression.new(['~@foo'])
         end
 
         it "should match @bar" do
@@ -52,7 +52,7 @@ module Gherkin
 
       context "@foo || @bar" do
         before(:each) do
-          @e = TagExpression.new(['@foo,@bar'])
+          @e = Gherkin::Parser::TagExpression.new(['@foo,@bar'])
         end
 
         it "should match @foo" do
@@ -70,7 +70,7 @@ module Gherkin
 
       context "(@foo || @bar) && !@zap" do
         before(:each) do
-          @e = TagExpression.new(['@foo,@bar', '~@zap'])
+          @e = Gherkin::Parser::TagExpression.new(['@foo,@bar', '~@zap'])
         end
 
         it "should match @foo" do
@@ -84,7 +84,7 @@ module Gherkin
 
       context "(@foo:3 || !@bar:4) && @zap:5" do
         before(:each) do
-          @e = TagExpression.new(['@foo:3,~@bar','@zap:5'])
+          @e = Gherkin::Parser::TagExpression.new(['@foo:3,~@bar','@zap:5'])
         end
 
         it "should count tags for positive tags" do
@@ -98,7 +98,7 @@ module Gherkin
 
       context "Parsing '@foo:3,~@bar', '@zap:5'" do
         before(:each) do
-          @e = TagExpression.new([' @foo:3 , ~@bar ', ' @zap:5 '])
+          @e = Gherkin::Parser::TagExpression.new([' @foo:3 , ~@bar ', ' @zap:5 '])
         end
 
         unless defined?(JRUBY_VERSION)

data/spec/gherkin/rb_lexer_spec.rb

@@ -1,5 +1,5 @@
 #encoding: utf-8
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'spec_helper'
 require 'gherkin/rb_lexer/en'
 
 module Gherkin

data/spec/gherkin/sexp_recorder.rb

@@ -1,5 +1,10 @@
+require 'gherkin/rubify'
+require 'gherkin/parser/row'
+
 module Gherkin
   class SexpRecorder
+    include Rubify
+    
     def initialize
       @sexps = []
     end
@@ -8,7 +13,8 @@ module Gherkin
       event = :scenario_outline if event == :scenarioOutline # Special Java Lexer handling
       event = :py_string if event == :pyString # Special Java Lexer handling
       event = :syntax_error if event == :syntaxError # Special Java Lexer handling
-      args[0] = args[0].to_a if event == :row # Special JRuby handling
+      args  = rubify(args)
+      args  = sexpify(args)
       @sexps << [event] + args
     end
 
@@ -28,5 +34,19 @@ module Gherkin
     def line(number)
       @sexps.find { |sexp| sexp.last == number }
     end
+
+    def sexpify(o)
+      if (defined?(JRUBY_VERSION) && Java.java.util.Collection === o) || Array === o
+        o.map{|e| sexpify(e)}
+      elsif(Parser::Row === o)
+        {
+          "cells" => rubify(o.cells),
+          "comments" => rubify(o.comments),
+          "line" => o.line,
+        }
+      else
+        o
+      end
+    end
   end
 end

data/spec/gherkin/shared/{lexer_spec.rb → lexer_group.rb}

@@ -1,22 +1,27 @@
 #encoding: utf-8
-require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'spec_helper'
 
 module Gherkin
   module Lexer
     shared_examples_for "a Gherkin lexer" do
+      def scan(gherkin)
+        @lexer.scan(gherkin, "test.feature", 0)
+      end
 
       describe "Comments" do
         it "should parse a one line comment" do
-          @lexer.scan("# My comment\n")
+          scan("# My comment\n")
           @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
             [:comment, "# My comment", 1],
             [:eof]
           ]
         end
 
         it "should parse a multiline comment" do
-          @lexer.scan("# Hello\n\n# World\n")
+          scan("# Hello\n\n# World\n")
           @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
             [:comment, "# Hello", 1],
             [:comment, "# World", 3],
             [:eof]
@@ -24,18 +29,20 @@ module Gherkin
         end
 
         it "should not consume comments as part of a multiline name" do
-          @lexer.scan("Scenario: test\n#hello\n Scenario: another")
+          scan("Scenario: test\n#hello\n Scenario: another")
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "test", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "test", "", 1],
             [:comment, "#hello", 2],
-            [:scenario, "Scenario", "another", 3],
+            [:scenario, "Scenario", "another", "", 3],
             [:eof]
           ]
         end
 
         it "should allow empty comment lines" do 
-          @lexer.scan("#\n   # A comment\n   #\n")
+          scan("#\n   # A comment\n   #\n")
           @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
             [:comment, "#", 1],
             [:comment, "# A comment", 2],
             [:comment, "#", 3],
@@ -45,52 +52,66 @@ module Gherkin
         
         it "should not allow comments within the Feature description" do
           lambda { 
-            @lexer.scan("Feature: something\nAs a something\n# Comment\nI want something") 
+            scan("Feature: something\nAs a something\n# Comment\nI want something") 
             }.should raise_error(/Lexing error on line 4/)
         end
       end
 
       describe "Tags" do
         it "should not take the tags as part of a multiline name feature element" do
-          @lexer.scan("Feature: hi\n Scenario: test\n\n@hello\n Scenario: another")
+          scan("Feature: hi\n Scenario: test\n\n@hello\n Scenario: another")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "hi", 1],
-            [:scenario, "Scenario", "test", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "hi", "", 1],
+            [:scenario, "Scenario", "test", "", 2],
             [:tag, "@hello", 4],
-            [:scenario, "Scenario", "another", 5],
+            [:scenario, "Scenario", "another", "", 5],
             [:eof]
           ]
         end  
       end
 
       describe "Background" do
+        it "should allow an empty background name and description" do
+          scan("Background:\nGiven I am a step\n")
+          @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
+            [:background, "Background", "", "", 1],
+            [:step, "Given ", "I am a step", 2],
+            [:eof]
+          ]
+        end
+
         it "should allow an empty background description" do
-          @lexer.scan("Background:\nGiven I am a step\n")
+          scan("Background: Yeah\nGiven I am a step\n")
           @listener.to_sexp.should == [
-            [:background, "Background", "", 1],
+            [:location, 'test.feature', 0],
+            [:background, "Background", "Yeah", "", 1],
             [:step, "Given ", "I am a step", 2],
             [:eof]
           ]
         end
         
         it "should allow multiline names ending at eof" do
-          @lexer.scan("Background: I have several\n   Lines to look at\n None starting with Given")
+          scan("Background: I have several\n   Lines to look at\n None starting with Given")
           @listener.to_sexp.should == [
-            [:background, "Background", "I have several\nLines to look at\nNone starting with Given", 1],
+            [:location, 'test.feature', 0],
+            [:background, "Background", "I have several", "Lines to look at\nNone starting with Given", 1],
             [:eof]
           ]
         end
          
         it "should allow multiline names" do
-          @lexer.scan(%{Feature: Hi
+          scan(%{Feature: Hi
 Background: It is my ambition to say 
             in ten sentences
             what others say 
             in a whole book.
 Given I am a step})
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Hi", 1],
-            [:background, "Background", "It is my ambition to say\nin ten sentences\nwhat others say\nin a whole book.",2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Hi", "", 1],
+            [:background, "Background", "It is my ambition to say", "in ten sentences\nwhat others say\nin a whole book.",2],
             [:step, "Given ", "I am a step", 6],
             [:eof]
           ]
@@ -99,68 +120,74 @@ Given I am a step})
 
       describe "Scenarios" do
         it "should be parsed" do
-          @lexer.scan("Scenario: Hello\n")
+          scan("Scenario: Hello\n")
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "Hello", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "Hello", "", 1],
             [:eof]
           ]
         end
  
         it "should allow whitespace lines after the Scenario line" do
-          @lexer.scan(%{Scenario: bar
+          scan(%{Scenario: bar
 
                           Given baz
                           })
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "bar", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "bar", "", 1],
             [:step, "Given ", "baz", 3],
             [:eof]
           ]
         end
         
         it "should allow multiline names" do
-          @lexer.scan(%{Scenario: It is my ambition to say
+          scan(%{Scenario: It is my ambition to say
                           in ten sentences
                           what others say 
                           in a whole book.
                           Given I am a step
                           })
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "It is my ambition to say\nin ten sentences\nwhat others say\nin a whole book.", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "It is my ambition to say", "in ten sentences\nwhat others say\nin a whole book.", 1],
             [:step, "Given ", "I am a step", 5],
             [:eof]
           ]
         end
 
         it "should allow multiline names ending at eof" do
-          @lexer.scan("Scenario: I have several\n       Lines to look at\n None starting with Given")
+          scan("Scenario: I have several\n       Lines to look at\n None starting with Given")
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "I have several\nLines to look at\nNone starting with Given", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "I have several", "Lines to look at\nNone starting with Given", 1],
             [:eof]
           ]
         end
   
         it "should ignore gherkin keywords embedded in other words" do
-          @lexer.scan(%{Scenario: I have a Button
+          scan(%{Scenario: I have a Button
           Buttons are great
   Given I have some
   But I might not because I am a Charles Dickens character
 })
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "I have a Button\nButtons are great", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "I have a Button", "Buttons are great", 1],
             [:step, "Given ", "I have some", 3],
             [:step, "But ", "I might not because I am a Charles Dickens character", 4],
             [:eof]
           ]
         end
         
-        it "should allow step names in Scenario descriptions" do
-          @lexer.scan(%{Scenario: When I have when in scenario
+        it "should allow step keywords in Scenario names" do
+          scan(%{Scenario: When I have when in scenario
           I should be fine
 Given I am a step
 })
           @listener.to_sexp.should == [
-            [:scenario, "Scenario", "When I have when in scenario\nI should be fine", 1],
+            [:location, 'test.feature', 0],
+            [:scenario, "Scenario", "When I have when in scenario", "I should be fine", 1],
             [:step, "Given ", "I am a step", 3],
             [:eof]
           ]
@@ -169,36 +196,41 @@ Given I am a step
 
       describe "Scenario Outlines" do
         it "should be parsed" do
-          @lexer.scan(%{Scenario Outline: Hello
+          scan(%{Scenario Outline: Hello
+                          With a description
                           Given a <what> cucumber
-                          Examples:
+                          Examples: With a name
+                          and a description
                           |what|
                           |green|
                           })
           @listener.to_sexp.should == [
-            [:scenario_outline, "Scenario Outline", "Hello", 1],
-            [:step, "Given ", "a <what> cucumber", 2],
-            [:examples, "Examples", "", 3],
-            [:row, ["what"], 4],
-            [:row, ["green"], 5],
+            [:location, 'test.feature', 0],
+            [:scenario_outline, "Scenario Outline", "Hello", "With a description", 1],
+            [:step, "Given ", "a <what> cucumber", 3],
+            [:examples, "Examples", "With a name", "and a description", 4],
+            [:row, ["what"], 6],
+            [:row, ["green"], 7],
             [:eof]
           ]
         end
 
+
         it "should parse with no steps or examples" do
-          @lexer.scan(%{Scenario Outline: Hello
+          scan(%{Scenario Outline: Hello
 
                           Scenario: My Scenario
                           })
           @listener.to_sexp.should == [
-            [:scenario_outline, "Scenario Outline", "Hello", 1],
-            [:scenario, "Scenario", "My Scenario", 3],
+            [:location, 'test.feature', 0],
+            [:scenario_outline, "Scenario Outline", "Hello", "", 1],
+            [:scenario, "Scenario", "My Scenario", "", 3],
             [:eof]
           ]
         end
 
-        it "should allow multiline names" do
-          @lexer.scan(%{Scenario Outline: It is my ambition to say 
+        it "should allow multiline description" do
+          scan(%{Scenario Outline: It is my ambition to say 
                           in ten sentences
                           what others say 
                           in a whole book.
@@ -206,7 +238,8 @@ Given I am a step
 
                           })
           @listener.to_sexp.should == [
-            [:scenario_outline, "Scenario Outline", "It is my ambition to say\nin ten sentences\nwhat others say\nin a whole book.", 1],
+            [:location, 'test.feature', 0],
+            [:scenario_outline, "Scenario Outline", "It is my ambition to say", "in ten sentences\nwhat others say\nin a whole book.", 1],
             [:step, "Given ", "I am a step", 5],
             [:eof]
           ]
@@ -215,12 +248,13 @@ Given I am a step
 
       describe "Examples" do
         it "should be parsed" do
-          @lexer.scan(%{Examples:
+          scan(%{Examples:
                           |x|y|
                           |5|6|
                           })
           @listener.to_sexp.should == [
-            [:examples, "Examples", "", 1],
+            [:location, 'test.feature', 0],
+            [:examples, "Examples", "", "", 1],
             [:row, ["x","y"], 2],
             [:row, ["5","6"], 3],
             [:eof]
@@ -228,14 +262,15 @@ Given I am a step
         end
         
         it "should parse multiline example names" do
-          @lexer.scan(%{Examples: I'm a multiline name
+          scan(%{Examples: I'm a multiline name
                           and I'm ok
                           f'real
                           |x|
                           |5|
                           })
           @listener.to_sexp.should == [
-            [:examples, "Examples", "I'm a multiline name\nand I'm ok\nf'real", 1],
+            [:location, 'test.feature', 0],
+            [:examples, "Examples", "I'm a multiline name", "and I'm ok\nf'real", 1],
             [:row, ["x"], 4],
             [:row, ["5"], 5],
             [:eof]
@@ -245,10 +280,11 @@ Given I am a step
       
       describe "Steps" do
         it "should parse steps with inline table" do
-          @lexer.scan(%{Given I have a table 
+          scan(%{Given I have a table 
                           |a|b|
                           })
           @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
             [:step, "Given ", "I have a table", 1],
             [:row, ['a','b'], 2],
             [:eof]
@@ -256,8 +292,9 @@ Given I am a step
         end
         
         it "should parse steps with inline py_string" do
-          @lexer.scan("Given I have a string\n\"\"\"\nhello\nworld\n\"\"\"")
+          scan("Given I have a string\n\"\"\"\nhello\nworld\n\"\"\"")
           @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
             [:step, "Given ", "I have a string", 1],
             [:py_string, "hello\nworld", 2],
             [:eof]
@@ -267,10 +304,11 @@ Given I am a step
             
       describe "A single feature, single scenario, single step" do
         it "should find the feature, scenario, and step" do
-          @lexer.scan("Feature: Feature Text\n  Scenario: Reading a Scenario\n    Given there is a step\n")
+          scan("Feature: Feature Text\n  Scenario: Reading a Scenario\n    Given there is a step\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "there is a step", 3],
             [:eof]
           ]
@@ -279,10 +317,11 @@ Given I am a step
 
       describe "A feature ending in whitespace" do
         it "should not raise an error when whitespace follows the Feature, Scenario, and Steps" do
-          @lexer.scan("Feature: Feature Text\n Scenario: Reading a Scenario\n    Given there is a step\n    ")
+          scan("Feature: Feature Text\n Scenario: Reading a Scenario\n    Given there is a step\n    ")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "there is a step", 3],
             [:eof]
           ]
@@ -292,10 +331,11 @@ Given I am a step
       describe "A single feature, single scenario, three steps" do
         
         it "should find the feature, scenario, and three steps" do
-          @lexer.scan("Feature: Feature Text\n  Scenario: Reading a Scenario\n    Given there is a step\n    And another step\n   And a third step\n")
+          scan("Feature: Feature Text\n  Scenario: Reading a Scenario\n    Given there is a step\n    And another step\n   And a third step\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "there is a step", 3],
             [:step, "And ", "another step", 4],
             [:step, "And ", "a third step", 5],
@@ -306,17 +346,19 @@ Given I am a step
 
       describe "A single feature with no scenario" do
         it "should find the feature" do
-          @lexer.scan("Feature: Feature Text\n")
+          scan("Feature: Feature Text\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
             [:eof]
           ]
         end
 
         it "should parse a one line feature with no newline" do
-          @lexer.scan("Feature: hi")
+          scan("Feature: hi")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "hi", 1],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "hi", "", 1],
             [:eof]
           ]
         end
@@ -324,9 +366,10 @@ Given I am a step
       
       describe "A multi-line feature with no scenario" do
         it "should find the feature" do
-          @lexer.scan("Feature: Feature Text\n  And some more text")
+          scan("Feature: Feature Text\n  And some more text")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text\nAnd some more text", 1],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "And some more text", 1],
             [:eof]
           ]
         end
@@ -334,10 +377,11 @@ Given I am a step
 
       describe "A feature with a scenario but no steps" do
         it "should find the feature and scenario" do
-          @lexer.scan("Feature: Feature Text\nScenario: Reading a Scenario\n")
+          scan("Feature: Feature Text\nScenario: Reading a Scenario\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:eof]
           ]
         end
@@ -345,24 +389,26 @@ Given I am a step
 
       describe "A feature with two scenarios" do
         it "should find the feature and two scenarios" do
-          @lexer.scan("Feature: Feature Text\nScenario: Reading a Scenario\n  Given a step\n\nScenario: A second scenario\n Given another step\n")
+          scan("Feature: Feature Text\nScenario: Reading a Scenario\n  Given a step\n\nScenario: A second scenario\n Given another step\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "a step", 3],
-            [:scenario, "Scenario", "A second scenario", 5],
+            [:scenario, "Scenario", "A second scenario", "", 5],
             [:step, "Given ", "another step", 6],
             [:eof]
           ]
         end
         
         it "should find the feature and two scenarios without indentation" do
-          @lexer.scan("Feature: Feature Text\nScenario: Reading a Scenario\nGiven a step\nScenario: A second scenario\nGiven another step\n")
+          scan("Feature: Feature Text\nScenario: Reading a Scenario\nGiven a step\nScenario: A second scenario\nGiven another step\n")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "a step", 3],
-            [:scenario, "Scenario", "A second scenario", 4],
+            [:scenario, "Scenario", "A second scenario", "", 4],
             [:step, "Given ", "another step", 5],
             [:eof]
           ]
@@ -373,10 +419,11 @@ Given I am a step
         it "should find the feature, scenarios, steps, and comments in the proper order" do
           scan_file("simple_with_comments.feature")
           @listener.to_sexp.should == [
+            [:location, 'simple_with_comments.feature', 0],
             [:comment, "# Here is a comment", 1],
-            [:feature, "Feature", "Feature Text", 2],
+            [:feature, "Feature", "Feature Text", "", 2],
             [:comment, "# Here is another # comment", 3],
-            [:scenario, "Scenario", "Reading a Scenario", 4],
+            [:scenario, "Scenario", "Reading a Scenario", "", 4],
             [:comment, "# Here is a third comment", 5],
             [:step, "Given ", "there is a step", 6],
             [:comment, "# Here is a fourth comment", 7],
@@ -387,10 +434,11 @@ Given I am a step
         it "should support comments in tables" do
           scan_file("comments_in_table.feature")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "x", 1], 
-            [:scenario_outline, "Scenario Outline", "x", 3], 
+            [:location, 'comments_in_table.feature', 0],
+            [:feature, "Feature", "x", "", 1], 
+            [:scenario_outline, "Scenario Outline", "x", "", 3], 
             [:step, "Then ", "x is <state>", 4], 
-            [:examples, "Examples", "", 6], 
+            [:examples, "Examples", "", "", 6], 
             [:row, ["state"], 7], 
             [:comment, "# comment", 8], 
             [:row, ["1"], 9],
@@ -403,18 +451,19 @@ Given I am a step
         it "should find the feature, scenario, step, and tags in the proper order" do
           scan_file("simple_with_tags.feature")
           @listener.to_sexp.should == [
+            [:location, 'simple_with_tags.feature', 0],
             [:comment, "# FC", 1],
             [:tag, "@ft",2],
-            [:feature, "Feature", "hi", 3],
+            [:feature, "Feature", "hi", "", 3],
             [:tag, "@st1", 5],
             [:tag, "@st2", 5],
-            [:scenario, "Scenario", "First", 6],
+            [:scenario, "Scenario", "First", "", 6],
             [:step, "Given ", "Pepper", 7],
             [:tag, "@st3", 9],
             [:tag, "@st4", 10],
             [:tag, "@ST5", 10],
             [:tag, "@#^%&ST6**!", 10],
-            [:scenario, "Scenario", "Second", 11],
+            [:scenario, "Scenario", "Second", "", 11],
             [:eof]
           ]
         end        
@@ -424,11 +473,12 @@ Given I am a step
         it "should lex this feature properly" do
           scan_file("1.feature")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Logging in\nSo that I can be myself", 1],
+            [:location, '1.feature', 0],
+            [:feature, "Feature", "Logging in", "So that I can be myself", 1],
             [:comment, "# Comment", 3],
-            [:scenario, "Scenario", "Anonymous user can get a login form.\nScenery here", 4],
+            [:scenario, "Scenario", "Anonymous user can get a login form.", "Scenery here", 4],
             [:tag, "@tag", 7],
-            [:scenario, "Scenario", "Another one", 8],
+            [:scenario, "Scenario", "Another one", "", 8],
             [:eof]
           ]
         end
@@ -438,23 +488,24 @@ Given I am a step
         it "should find things in the right order" do
           scan_file("complex.feature")
           @listener.to_sexp.should == [
+            [:location, 'complex.feature', 0],
             [:comment, "#Comment on line 1", 1],
             [:comment, "#Comment on line 2", 2],
             [:tag, "@tag1", 3],
             [:tag, "@tag2", 3],
-            [:feature, "Feature", "Feature Text\nIn order to test multiline forms\nAs a ragel writer\nI need to check for complex combinations", 4],
+            [:feature, "Feature", "Feature Text", "In order to test multiline forms\nAs a ragel writer\nI need to check for complex combinations", 4],
             [:comment, "#Comment on line 9", 9],
             [:comment, "#Comment on line 11", 11],
-            [:background, "Background", "", 13],
+            [:background, "Background", "", "", 13],
             [:step, "Given ", "this is a background step", 14],
             [:step, "And ", "this is another one", 15],
             [:tag, "@tag3", 17],
             [:tag, "@tag4", 17],
-            [:scenario, "Scenario", "Reading a Scenario", 18],
+            [:scenario, "Scenario", "Reading a Scenario", "", 18],
             [:step, "Given ", "there is a step", 19],
             [:step, "But ", "not another step", 20],
             [:tag, "@tag3", 22],
-            [:scenario, "Scenario", "Reading a second scenario\nWith two lines of text", 23],
+            [:scenario, "Scenario", "Reading a second scenario", "With two lines of text", 23],
             [:comment, "#Comment on line 24", 25],
             [:step, "Given ", "a third step with a table", 26],
             [:row, %w{a b}, 27],
@@ -468,7 +519,7 @@ Given I am a step
             [:step, "And ", "I am done testing these tables", 35],
             [:comment, "#Comment on line 29", 36],
             [:step, "Then ", "I am happy", 37],
-            [:scenario, "Scenario", "Hammerzeit", 39],
+            [:scenario, "Scenario", "Hammerzeit", "", 39],
             [:step, "Given ", "All work and no play", 40],
             [:py_string, "Makes Homer something something\nAnd something else", 41 ],
             [:step, "Then ", "crazy", 45],
@@ -481,23 +532,24 @@ Given I am a step
         it "should find things in the right order for CRLF features" do
           scan_file("dos_line_endings.feature")
           @listener.to_sexp.should == [
+            [:location, 'dos_line_endings.feature', 0],
             [:comment, "#Comment on line 1", 1],
             [:comment, "#Comment on line 2", 2],
             [:tag, "@tag1", 3],
             [:tag, "@tag2", 3],
-            [:feature, "Feature", "Feature Text\r\nIn order to test multiline forms\r\nAs a ragel writer\r\nI need to check for complex combinations", 4],
+            [:feature, "Feature", "Feature Text", "In order to test multiline forms\r\nAs a ragel writer\r\nI need to check for complex combinations", 4],
             [:comment, "#Comment on line 9", 9],
             [:comment, "#Comment on line 11", 11],
-            [:background, "Background", "", 13],
+            [:background, "Background", "", "", 13],
             [:step, "Given ", "this is a background step", 14],
             [:step, "And ", "this is another one", 15],
             [:tag, "@tag3", 17],
             [:tag, "@tag4", 17],
-            [:scenario, "Scenario", "Reading a Scenario", 18],
+            [:scenario, "Scenario", "Reading a Scenario", "", 18],
             [:step, "Given ", "there is a step", 19],
             [:step, "But ", "not another step", 20],
             [:tag, "@tag3", 22],
-            [:scenario, "Scenario", "Reading a second scenario\r\nWith two lines of text", 23],
+            [:scenario, "Scenario", "Reading a second scenario", "With two lines of text", 23],
             [:comment, "#Comment on line 24", 25],
             [:step, "Given ", "a third step with a table", 26],
             [:row, %w{a b}, 27],
@@ -511,7 +563,7 @@ Given I am a step
             [:step, "And ", "I am done testing these tables", 35],
             [:comment, "#Comment on line 29", 36],
             [:step, "Then ", "I am happy", 37],
-            [:scenario, "Scenario", "Hammerzeit", 39],
+            [:scenario, "Scenario", "Hammerzeit", "", 39],
             [:step, "Given ", "All work and no play", 40],
             [:py_string, "Makes Homer something something\r\nAnd something else", 41],
             [:step, "Then ", "crazy", 45],
@@ -522,8 +574,9 @@ Given I am a step
         it "should cope with the retarded BOM that many Windows editors insert at the beginning of a file" do
           scan_file("with_bom.feature")
           @listener.to_sexp.should == [
-            [:feature, "Feature", "Feature Text", 1],
-            [:scenario, "Scenario", "Reading a Scenario", 2],
+            [:location, 'with_bom.feature', 0],
+            [:feature, "Feature", "Feature Text", "", 1],
+            [:scenario, "Scenario", "Reading a Scenario", "", 2],
             [:step, "Given ", "there is a step", 3],
             [:eof]
           ]
@@ -535,15 +588,32 @@ Given I am a step
           ["Some text\nFeature: Hi", 
             "Feature: Hi\nBackground:\nGiven something\nScenario A scenario",
             "Scenario: My scenario\nGiven foo\nAand bar\nScenario: another one\nGiven blah"].each do |text|
-              lambda { @lexer.scan(text) }.should raise_error(/Lexing error on line/)
+              lambda { scan(text) }.should raise_error(/Lexing error on line/)
           end
         end
         
         it "should include the line number and context of the error" do
           lambda {
-            @lexer.scan("Feature: hello\nScenario: My scenario\nGiven foo\nAand blah\nHmmm wrong\nThen something something")
+            scan("Feature: hello\nScenario: My scenario\nGiven foo\nAand blah\nHmmm wrong\nThen something something")
           }.should raise_error(/Lexing error on line 4/)
         end
+
+        it "Feature keyword should terminate narratives for multiline capable tokens" do
+          scan("Feature:\nBackground:\nFeature:\nScenario Outline:\nFeature:\nScenario:\nFeature:\nExamples:\nFeature:\n")
+          @listener.to_sexp.should == [
+            [:location, 'test.feature', 0],
+            [:feature, "Feature", "", "", 1],
+            [:background, "Background", "", "", 2],
+            [:feature, "Feature", "", "", 3],
+            [:scenario_outline, "Scenario Outline", "", "", 4],
+            [:feature, "Feature", "", "", 5],
+            [:scenario, "Scenario", "", "", 6],
+            [:feature, "Feature", "", "", 7],
+            [:examples, "Examples", "","",  8],
+            [:feature, "Feature", "", "", 9],
+            [:eof]
+          ]
+        end
       end
     end
   end