better_html 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 120de12348f0595aba1d9966e7643111a25e8ef8
-  data.tar.gz: 1d1dd0f4e7e572392e51aa38186bf9bc5973d96c
+  metadata.gz: fbf85413b0af66cc73a928d1b37c0ba104c848ae
+  data.tar.gz: e934a9efa924f0647128517985f66d4dbd97f0de
 SHA512:
-  metadata.gz: 3dcc692abe303950c191c74e3ed9b03481d13721416012ee7760e3b7b03830ef2199fc5d685200272d11fff02384f6394c926419b27243588d9b128981fb4ae4
-  data.tar.gz: 83e8cfdbce133bad899a7d4b079e25424f0673c36f6bff6d78577a4822473e0da10027bd9023277775babeee1f50dc2dcd84a4edffd115820a064f44aa4f2429
+  metadata.gz: eb4ecf3184328d0388000db860f9126edad2e4a438c6c97623dbd29776d8f6a623fcbc5a24989c5fe47f0a92617eb0222738e28e5cae732d403a3bb805d3eb99
+  data.tar.gz: 5843b4928f2983d53e36e44053423b21286daf6b20cc1d7d5ddab3e3ceffe5f3c16f71a6849205732cd4c9e8776e4d86c299ca1603751da5a4056044eb653369

data/lib/better_html/parser.rb

@@ -4,6 +4,7 @@ require_relative 'tokenizer/html_lodash'
 require_relative 'tokenizer/location'
 require_relative 'tokenizer/token_array'
 require_relative 'ast/node'
+require 'parser/source/buffer'
 
 module BetterHtml
   class Parser
@@ -19,16 +20,17 @@ module BetterHtml
       end
     end
 
-    def initialize(document, template_language: :html)
-      @document = document
+    def initialize(buffer, template_language: :html)
+      raise ArgumentError, 'first argument must be Parser::Source::Buffer' unless buffer.is_a?(::Parser::Source::Buffer)
+      @buffer = buffer
       @template_language = template_language
       @erb = case template_language
       when :html
-        Tokenizer::HtmlErb.new(@document)
+        Tokenizer::HtmlErb.new(@buffer)
       when :lodash
-        Tokenizer::HtmlLodash.new(@document)
+        Tokenizer::HtmlLodash.new(@buffer)
       when :javascript
-        Tokenizer::JavascriptErb.new(@document)
+        Tokenizer::JavascriptErb.new(@buffer)
       else
         raise ArgumentError, "template_language can be :html or :javascript"
       end
@@ -47,7 +49,7 @@ module BetterHtml
       @erb.parser.errors.map do |error|
         Error.new(
           error.message,
-          location: Tokenizer::Location.new(@document, error.position, error.position + 1)
+          location: Tokenizer::Location.new(@buffer, error.position, error.position + 1)
         )
       end
     end
@@ -199,11 +201,11 @@ module BetterHtml
     def build_location(enumerable)
       enumerable = enumerable.compact
       raise ArgumentError, "cannot build location for #{enumerable.inspect}" unless enumerable.first && enumerable.last
-      Tokenizer::Location.new(@document, enumerable.first.loc.start, enumerable.last.loc.stop)
+      Tokenizer::Location.new(@buffer, enumerable.first.loc.begin_pos, enumerable.last.loc.end_pos)
     end
 
     def empty_location
-      Tokenizer::Location.new(@document, 0, 0)
+      Tokenizer::Location.new(@buffer, 0, 0)
     end
 
     def shift_all(tokens, *types)

data/lib/better_html/test_helper/safe_erb/tag_interpolation.rb

@@ -158,9 +158,9 @@ module BetterHtml
 
         def nested_location(parent_node, ruby_node)
           Tokenizer::Location.new(
-            parent_node.loc.document,
-            parent_node.loc.start + ruby_node.loc.expression.begin_pos,
-            parent_node.loc.start + ruby_node.loc.expression.end_pos - 1
+            parent_node.loc.source_buffer,
+            parent_node.loc.begin_pos + ruby_node.loc.expression.begin_pos,
+            parent_node.loc.begin_pos + ruby_node.loc.expression.end_pos
           )
         end
       end

data/lib/better_html/test_helper/safe_erb_tester.rb

@@ -37,7 +37,9 @@ EOF
       def assert_erb_safety(data, **options)
         options = options.present? ? options.dup : {}
         options[:template_language] ||= :html
-        parser = BetterHtml::Parser.new(data, options)
+        buffer = ::Parser::Source::Buffer.new(options[:filename] || '(buffer)')
+        buffer.source = data
+        parser = BetterHtml::Parser.new(buffer, options)
 
         tester_classes = [
           SafeErb::NoStatements,
@@ -57,7 +59,7 @@ EOF
 
         messages = errors.map do |error|
           <<~EOL
-          On line #{error.location.line}
+          In #{buffer.name}:#{error.location.line}
           #{error.message}
           #{error.location.line_source_with_underline}\n
           EOL

data/lib/better_html/test_helper/safe_lodash_tester.rb

@@ -28,7 +28,9 @@ Never use <script> tags inside lodash template.
 EOF
 
       def assert_lodash_safety(data, **options)
-        tester = Tester.new(data, **options)
+        buffer = ::Parser::Source::Buffer.new(options[:filename] || '(buffer)')
+        buffer.source = data
+        tester = Tester.new(buffer, **options)
 
         message = ""
         tester.errors.each do |error|
@@ -49,11 +51,11 @@ EOF
       class Tester
         attr_reader :errors
 
-        def initialize(data, config: BetterHtml.config)
-          @data = data
+        def initialize(buffer, config: BetterHtml.config)
+          @buffer = buffer
           @config = config
           @errors = Errors.new
-          @parser = BetterHtml::Parser.new(data, template_language: :lodash)
+          @parser = BetterHtml::Parser.new(buffer, template_language: :lodash)
           validate!
         end
 

data/lib/better_html/tokenizer/base_erb.rb

@@ -1,6 +1,7 @@
 require 'erubi'
 require_relative 'token'
 require_relative 'location'
+require 'parser/source/buffer'
 
 module BetterHtml
   module Tokenizer
@@ -12,11 +13,12 @@ module BetterHtml
       attr_reader :tokens
       attr_reader :current_position
 
-      def initialize(document)
-        @document = document
+      def initialize(buffer)
+        raise ArgumentError, 'first argument must be Parser::Source::Buffer' unless buffer.is_a?(::Parser::Source::Buffer)
+        @buffer = buffer
         @tokens = []
         @current_position = 0
-        super(document, regexp: REGEXP_WITHOUT_TRIM, trim: false)
+        super(buffer.source, regexp: REGEXP_WITHOUT_TRIM, trim: false)
       end
 
       private
@@ -66,10 +68,10 @@ module BetterHtml
         token = add_token(:erb_end, pos, pos + 2)
       end
 
-      def add_token(type, start, stop)
+      def add_token(type, begin_pos, end_pos)
         token = Token.new(
           type: type,
-          loc: Location.new(@document, start, stop - 1)
+          loc: Location.new(@buffer, begin_pos, end_pos)
         )
         @tokens << token
         token

data/lib/better_html/tokenizer/html_erb.rb

@@ -6,9 +6,9 @@ module BetterHtml
     class HtmlErb < BaseErb
       attr_reader :parser
 
-      def initialize(document)
+      def initialize(buffer)
         @parser = HtmlTokenizer::Parser.new
-        super(document)
+        super(buffer)
       end
 
       def current_position
@@ -22,8 +22,8 @@ module BetterHtml
       end
 
       def add_text(text)
-        @parser.parse(text) do |type, start, stop, _line, _column|
-          add_token(type, start, stop)
+        @parser.parse(text) do |type, begin_pos, end_pos, _line, _column|
+          add_token(type, begin_pos, end_pos)
         end
       end
     end

data/lib/better_html/tokenizer/html_lodash.rb

@@ -13,9 +13,9 @@ module BetterHtml
       self.lodash_evaluate = %r{(?:\[\%)(.+?)(?:\%\])}m
       self.lodash_interpolate = %r{(?:\[\%)!(.+?)(?:\%\])}m
 
-      def initialize(document)
-        @document = document
-        @scanner = StringScanner.new(document)
+      def initialize(buffer)
+        @buffer = buffer
+        @scanner = StringScanner.new(buffer.source)
         @parser = HtmlTokenizer::Parser.new
         @tokens = []
         scan!
@@ -43,7 +43,7 @@ module BetterHtml
             end
             @parser.append_placeholder(match)
           else
-            text = @document[(@scanner.pos)..(@document.size)]
+            text = @buffer.source[(@scanner.pos)..(@buffer.source.size)]
             add_text(text) unless text.blank?
             break
           end
@@ -62,32 +62,32 @@ module BetterHtml
       end
 
       def add_text(text)
-        @parser.parse(text) do |type, start, stop, _line, _column|
-          add_token(type, start: start, stop: stop)
+        @parser.parse(text) do |type, begin_pos, end_pos, _line, _column|
+          add_token(type, begin_pos: begin_pos, end_pos: end_pos)
         end
       end
 
       def add_lodash_tokens(indicator, code)
         pos = @parser.document_length
 
-        add_token(:lodash_begin, start: pos, stop: pos + 2)
+        add_token(:lodash_begin, begin_pos: pos, end_pos: pos + 2)
         pos += 2
 
         if indicator
-          add_token(:indicator, start: pos, stop: pos + indicator.length)
+          add_token(:indicator, begin_pos: pos, end_pos: pos + indicator.length)
           pos += indicator.length
         end
 
-        add_token(:code, start: pos, stop: pos + code.length)
+        add_token(:code, begin_pos: pos, end_pos: pos + code.length)
         pos += code.length
 
-        add_token(:lodash_end, start: pos, stop: pos + 2)
+        add_token(:lodash_end, begin_pos: pos, end_pos: pos + 2)
       end
 
-      def add_token(type, start: nil, stop: nil)
+      def add_token(type, begin_pos: nil, end_pos: nil)
         token = Token.new(
           type: type,
-          loc: Location.new(@document, start, stop-1)
+          loc: Location.new(@buffer, begin_pos, end_pos)
         )
         @tokens << token
         token

data/lib/better_html/tokenizer/location.rb

@@ -1,76 +1,59 @@
+require 'parser/source/buffer'
+require 'parser/source/range'
+
 module BetterHtml
   module Tokenizer
-    class Location
-      attr_accessor :document, :start, :stop
-
-      def initialize(document, start, stop)
-        raise ArgumentError, "start location #{start} is out of range for document of size #{document.size}" if start > document.size
-        raise ArgumentError, "stop location #{stop} is out of range for document of size #{document.size}" if stop > document.size
-        raise ArgumentError, "end of range must be greater than start of range (#{stop} < #{start})" if stop < start
+    class Location < ::Parser::Source::Range
+      def initialize(buffer, begin_pos, end_pos)
+        raise ArgumentError, 'first argument must be Parser::Source::Buffer' unless buffer.is_a?(::Parser::Source::Buffer)
+        raise ArgumentError, "begin_pos location #{begin_pos} is out of range for document of size #{buffer.source.size}" if begin_pos > buffer.source.size
+        raise ArgumentError, "end_pos location #{end_pos} is out of range for document of size #{buffer.source.size}" if (end_pos - 1) > buffer.source.size
 
-        @document = document
-        @start = start
-        @stop = stop
+        super(buffer, begin_pos, end_pos)
       end
 
       def range
-        Range.new(start, stop)
+        Range.new(begin_pos, end_pos, true)
       end
 
       def line_range
         Range.new(start_line, stop_line)
       end
 
-      def source
-        @document[range]
-      end
-
-      def start_line
-        @start_line ||= calculate_line(start)
-      end
-
-      def line
-        start_line
-      end
-
-      def stop_line
-        @stop_line ||= calculate_line(stop)
-      end
+      alias_method :start_line, :line
+      alias_method :stop_line, :last_line
+      alias_method :start_column, :column
+      alias_method :stop_column, :last_column
 
-      def start_column
-        @start_column ||= calculate_column(start)
+      def line_source_with_underline
+        spaces = source_line.scan(/\A\s*/).first
+        column_without_spaces = [column - spaces.length, 0].max
+        underscore_length = [[end_pos - begin_pos, source_line.length - column_without_spaces].min, 1].max
+        "#{source_line.gsub(/\A\s*/, '')}\n#{' ' * column_without_spaces}#{'^' * underscore_length}"
       end
 
-      def column
-        start_column
+      def with(begin_pos: @begin_pos, end_pos: @end_pos)
+        self.class.new(@source_buffer, begin_pos, end_pos)
       end
 
-      def stop_column
-        @stop_column ||= calculate_column(stop)
+      def adjust(begin_pos: 0, end_pos: 0)
+        self.class.new(@source_buffer, @begin_pos + begin_pos, @end_pos + end_pos)
       end
 
-      def line_source_with_underline
-        line_content = extract_line(line: start_line)
-        spaces = line_content.scan(/\A\s*/).first
-        column_without_spaces = [column - spaces.length, 0].max
-        underscore_length = [[stop - start + 1, line_content.length - column_without_spaces].min, 1].max
-        "#{line_content.gsub(/\A\s*/, '')}\n#{' ' * column_without_spaces}#{'^' * underscore_length}"
+      def resize(new_size)
+        with(end_pos: @begin_pos + new_size)
       end
 
-      private
-
-      def calculate_line(pos)
-        return 1 if pos == 0
-        @document[0..pos-1].scan("\n").count + 1
+      def offset(offset)
+        with(begin_pos: offset + @begin_pos, end_pos: offset + @end_pos)
       end
 
-      def calculate_column(pos)
-        return 0 if pos == 0
-        @document[0..pos-1]&.split("\n", -1)&.last&.length || 0
+      def begin
+        with(end_pos: @begin_pos)
       end
 
-      def extract_line(line:)
-        @document.split("\n", -1)[line - 1]&.gsub(/\n$/, '') || ""
+      def end
+        with(begin_pos: @end_pos)
       end
     end
   end

data/lib/better_html/version.rb

@@ -1,3 +1,3 @@
 module BetterHtml
-  VERSION = "1.0.5"
+  VERSION = "1.0.6"
 end

data/test/better_html/parser_test.rb

@@ -7,17 +7,17 @@ module BetterHtml
     include ::AST::Sexp
 
     test "parse empty document" do
-      tree = Parser.new('')
+      tree = Parser.new(buffer(''))
 
       assert_equal s(:document), tree.ast
     end
 
     test "parser errors" do
-      tree = Parser.new('<>')
+      tree = Parser.new(buffer('<>'))
 
       assert_equal 1, tree.parser_errors.size
       assert_equal "expected '/' or tag name", tree.parser_errors[0].message
-      assert_equal 1..2, tree.parser_errors[0].location.range
+      assert_equal 1...2, tree.parser_errors[0].location.range
       assert_equal <<~EOF.strip, tree.parser_errors[0].location.line_source_with_underline
         <>
          ^
@@ -26,7 +26,7 @@ module BetterHtml
 
     test "consume cdata nodes" do
       code = "<![CDATA[ foo ]]>"
-      tree = Parser.new(code)
+      tree = Parser.new(buffer(code))
 
       assert_equal s(:document, s(:cdata, ' foo ')), tree.ast
       assert_equal code, tree.ast.loc.source
@@ -34,7 +34,7 @@ module BetterHtml
 
     test "unterminated cdata nodes are consumed until end" do
       code = "<![CDATA[ foo"
-      tree = Parser.new(code)
+      tree = Parser.new(buffer(code))
 
       assert_equal s(:document, s(:cdata, ' foo')), tree.ast
       assert_equal code, tree.ast.loc.source
@@ -42,7 +42,7 @@ module BetterHtml
 
     test "consume cdata with interpolation" do
       code = "<![CDATA[ foo <%= bar %> baz ]]>"
-      tree = Parser.new(code)
+      tree = Parser.new(buffer(code))
 
       assert_equal s(:document,
         s(:cdata,
@@ -55,19 +55,19 @@ module BetterHtml
     end
 
     test "consume comment nodes" do
-      tree = Parser.new("<!-- foo -->")
+      tree = Parser.new(buffer("<!-- foo -->"))
 
       assert_equal s(:document, s(:comment, ' foo ')), tree.ast
     end
 
     test "unterminated comment nodes are consumed until end" do
-      tree = Parser.new("<!-- foo")
+      tree = Parser.new(buffer("<!-- foo"))
 
       assert_equal s(:document, s(:comment, ' foo')), tree.ast
     end
 
     test "consume comment with interpolation" do
-      tree = Parser.new("<!-- foo <%= bar %> baz -->")
+      tree = Parser.new(buffer("<!-- foo <%= bar %> baz -->"))
 
       assert_equal s(:document,
         s(:comment,
@@ -79,12 +79,12 @@ module BetterHtml
     end
 
     test "consume tag nodes" do
-      tree = Parser.new("<div>")
+      tree = Parser.new(buffer("<div>"))
       assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, nil)), tree.ast
     end
 
     test "tag without name" do
-      tree = Parser.new("foo < bar")
+      tree = Parser.new(buffer("foo < bar"))
       assert_equal s(:document,
         s(:text, "foo "),
         s(:tag, nil, nil,
@@ -97,25 +97,25 @@ module BetterHtml
     end
 
     test "consume tag nodes with solidus" do
-      tree = Parser.new("</div>")
+      tree = Parser.new(buffer("</div>"))
       assert_equal s(:document, s(:tag, s(:solidus), s(:tag_name, "div"), nil, nil)), tree.ast
     end
 
     test "sets self_closing when appropriate" do
-      tree = Parser.new("<div/>")
+      tree = Parser.new(buffer("<div/>"))
       assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, s(:solidus))), tree.ast
     end
 
     test "consume tag nodes until name ends" do
-      tree = Parser.new("<div/>")
+      tree = Parser.new(buffer("<div/>"))
       assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, s(:solidus))), tree.ast
 
-      tree = Parser.new("<div ")
+      tree = Parser.new(buffer("<div "))
       assert_equal s(:document, s(:tag, nil, s(:tag_name, "div"), nil, nil)), tree.ast
     end
 
     test "consume tag nodes with interpolation" do
-      tree = Parser.new("<ns:<%= name %>-thing>")
+      tree = Parser.new(buffer("<ns:<%= name %>-thing>"))
       assert_equal s(:document,
         s(:tag,
           nil,
@@ -126,7 +126,7 @@ module BetterHtml
     end
 
     test "consume tag attributes with erb" do
-      tree = Parser.new("<div class=foo <%= erb %> name=bar>")
+      tree = Parser.new(buffer("<div class=foo <%= erb %> name=bar>"))
       assert_equal s(:document,
         s(:tag, nil,
           s(:tag_name, "div"),
@@ -149,7 +149,7 @@ module BetterHtml
     end
 
     test "consume tag attributes nodes unquoted value" do
-      tree = Parser.new("<div foo=bar>")
+      tree = Parser.new(buffer("<div foo=bar>"))
       assert_equal s(:document,
         s(:tag, nil,
           s(:tag_name, "div"),
@@ -165,7 +165,7 @@ module BetterHtml
     end
 
     test "consume attributes without name" do
-      tree = Parser.new("<div 'thing'>")
+      tree = Parser.new(buffer("<div 'thing'>"))
       assert_equal s(:document,
         s(:tag, nil,
           s(:tag_name, "div"),
@@ -181,7 +181,7 @@ module BetterHtml
     end
 
     test "consume tag attributes nodes quoted value" do
-      tree = Parser.new("<div foo=\"bar\">")
+      tree = Parser.new(buffer("<div foo=\"bar\">"))
       assert_equal s(:document,
         s(:tag, nil,
           s(:tag_name, "div"),
@@ -197,7 +197,7 @@ module BetterHtml
     end
 
     test "consume tag attributes nodes interpolation in name and value" do
-      tree = Parser.new("<div data-<%= foo %>=\"some <%= value %> foo\">")
+      tree = Parser.new(buffer("<div data-<%= foo %>=\"some <%= value %> foo\">"))
       assert_equal s(:document,
         s(:tag, nil,
           s(:tag_name, "div"),
@@ -219,7 +219,7 @@ module BetterHtml
     end
 
     test "consume text nodes" do
-      tree = Parser.new("here is <%= some %> text")
+      tree = Parser.new(buffer("here is <%= some %> text"))
 
       assert_equal s(:document,
         s(:text,
@@ -230,7 +230,7 @@ module BetterHtml
     end
 
     test "javascript template parsing works" do
-      tree = Parser.new("here is <%= some %> text", template_language: :javascript)
+      tree = Parser.new(buffer("here is <%= some %> text"), template_language: :javascript)
 
       assert_equal s(:document,
         s(:text,
@@ -241,7 +241,7 @@ module BetterHtml
     end
 
     test "javascript template does not consume html tags" do
-      tree = Parser.new("<div <%= some %> />", template_language: :javascript)
+      tree = Parser.new(buffer("<div <%= some %> />"), template_language: :javascript)
 
       assert_equal s(:document,
         s(:text,
@@ -252,7 +252,7 @@ module BetterHtml
     end
 
     test "lodash template parsing works" do
-      tree = Parser.new('<div class="[%= foo %]">', template_language: :lodash)
+      tree = Parser.new(buffer('<div class="[%= foo %]">'), template_language: :lodash)
 
       assert_equal s(:document,
         s(:tag,
@@ -275,7 +275,7 @@ module BetterHtml
     end
 
     test "nodes are all nested under document" do
-      tree = Parser.new(<<~HTML)
+      tree = Parser.new(buffer(<<~HTML))
         some text
         <!-- a comment -->
         some more text

data/test/better_html/test_helper/safe_erb/allowed_script_type_test.rb

@@ -34,7 +34,7 @@ module BetterHtml
 
         private
         def validate(data, template_language: :html)
-          parser = BetterHtml::Parser.new(data, template_language: template_language)
+          parser = BetterHtml::Parser.new(buffer(data), template_language: template_language)
           tester = BetterHtml::TestHelper::SafeErb::AllowedScriptType.new(parser, config: @config)
           tester.validate
           tester

data/test/better_html/test_helper/safe_erb/no_javascript_tag_helper_test.rb

@@ -26,7 +26,7 @@ module BetterHtml
 
         private
         def validate(data, template_language: :html)
-          parser = BetterHtml::Parser.new(data, template_language: template_language)
+          parser = BetterHtml::Parser.new(buffer(data), template_language: template_language)
           tester = BetterHtml::TestHelper::SafeErb::NoJavascriptTagHelper.new(parser, config: @config)
           tester.validate
           tester

data/test/better_html/test_helper/safe_erb/no_statements_test.rb

@@ -117,7 +117,7 @@ module BetterHtml
 
         private
         def validate(data, template_language: :html)
-          parser = BetterHtml::Parser.new(data, template_language: template_language)
+          parser = BetterHtml::Parser.new(buffer(data), template_language: template_language)
           tester = BetterHtml::TestHelper::SafeErb::NoStatements.new(parser, config: @config)
           tester.validate
           tester

data/test/better_html/test_helper/safe_erb/script_interpolation_test.rb

@@ -138,7 +138,7 @@ module BetterHtml
 
         private
         def validate(data, template_language: :html)
-          parser = BetterHtml::Parser.new(data, template_language: template_language)
+          parser = BetterHtml::Parser.new(buffer(data), template_language: template_language)
           tester = BetterHtml::TestHelper::SafeErb::ScriptInterpolation.new(parser, config: @config)
           tester.validate
           tester

data/test/better_html/test_helper/safe_erb/tag_interpolation_test.rb

@@ -292,7 +292,7 @@ module BetterHtml
 
         private
         def validate(data, template_language: :html)
-          parser = BetterHtml::Parser.new(data, template_language: template_language)
+          parser = BetterHtml::Parser.new(buffer(data), template_language: template_language)
           tester = BetterHtml::TestHelper::SafeErb::TagInterpolation.new(parser, config: @config)
           tester.validate
           tester

data/test/better_html/test_helper/safe_lodash_tester_test.rb

@@ -83,7 +83,7 @@ module BetterHtml
       private
 
       def parse(data)
-        SafeLodashTester::Tester.new(data)
+        SafeLodashTester::Tester.new(buffer(data))
       end
     end
   end

data/test/better_html/tokenizer/html_erb_test.rb

@@ -5,92 +5,92 @@ module BetterHtml
   module Tokenizer
     class HtmlErbTest < ActiveSupport::TestCase
       test "text" do
-        scanner = HtmlErb.new("just some text")
+        scanner = HtmlErb.new(buffer("just some text"))
         assert_equal 1, scanner.tokens.size
 
         assert_attributes ({
           type: :text,
-          loc: { start: 0, stop: 13, source: 'just some text' }
+          loc: { begin_pos: 0, end_pos: 14, source: 'just some text' }
         }), scanner.tokens[0]
       end
 
       test "statement" do
-        scanner = HtmlErb.new("<% statement %>")
+        scanner = HtmlErb.new(buffer("<% statement %>"))
         assert_equal 3, scanner.tokens.size
 
-        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
-        assert_attributes ({ type: :code, loc: { start: 2, stop: 12, source: ' statement ' } }), scanner.tokens[1]
-        assert_attributes ({ type: :erb_end, loc: { start: 13, stop: 14, source: '%>' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_begin, loc: { begin_pos: 0, end_pos: 2, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :code, loc: { begin_pos: 2, end_pos: 13, source: ' statement ' } }), scanner.tokens[1]
+        assert_attributes ({ type: :erb_end, loc: { begin_pos: 13, end_pos: 15, source: '%>' } }), scanner.tokens[2]
       end
 
       test "debug statement" do
-        scanner = HtmlErb.new("<%# statement %>")
+        scanner = HtmlErb.new(buffer("<%# statement %>"))
         assert_equal 4, scanner.tokens.size
 
-        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
-        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: '#' } }), scanner.tokens[1]
-        assert_attributes ({ type: :code, loc: { start: 3, stop: 13, source: ' statement ' } }), scanner.tokens[2]
-        assert_attributes ({ type: :erb_end, loc: { start: 14, stop: 15, source: '%>' } }), scanner.tokens[3]
+        assert_attributes ({ type: :erb_begin, loc: { begin_pos: 0, end_pos: 2, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 2, end_pos: 3, source: '#' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { begin_pos: 3, end_pos: 14, source: ' statement ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { begin_pos: 14, end_pos: 16, source: '%>' } }), scanner.tokens[3]
       end
 
       test "when multi byte characters are present in erb" do
         code = "<% ui_helper 'your store’s' %>"
-        scanner = HtmlErb.new(code)
+        scanner = HtmlErb.new(buffer(code))
         assert_equal 3, scanner.tokens.size
 
-        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
-        assert_attributes ({ type: :code, loc: { start: 2, stop: 27, source: " ui_helper 'your store’s' " } }), scanner.tokens[1]
-        assert_attributes ({ type: :erb_end, loc: { start: 28, stop: 29, source: '%>' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_begin, loc: { begin_pos: 0, end_pos: 2, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :code, loc: { begin_pos: 2, end_pos: 28, source: " ui_helper 'your store’s' " } }), scanner.tokens[1]
+        assert_attributes ({ type: :erb_end, loc: { begin_pos: 28, end_pos: 30, source: '%>' } }), scanner.tokens[2]
         assert_equal code.length, scanner.current_position
       end
 
       test "when multi byte characters are present in text" do
         code = "your store’s"
-        scanner = HtmlErb.new(code)
+        scanner = HtmlErb.new(buffer(code))
         assert_equal 1, scanner.tokens.size
 
-        assert_attributes ({ type: :text, loc: { start: 0, stop: 11, source: 'your store’s' } }), scanner.tokens[0]
+        assert_attributes ({ type: :text, loc: { begin_pos: 0, end_pos: 12, source: 'your store’s' } }), scanner.tokens[0]
         assert_equal code.length, scanner.current_position
       end
 
       test "when multi byte characters are present in html" do
         code = "<div title='your store’s'>foo</div>"
-        scanner = HtmlErb.new(code)
+        scanner = HtmlErb.new(buffer(code))
         assert_equal 14, scanner.tokens.size
 
-        assert_attributes ({ type: :tag_start, loc: { start: 0, stop: 0, source: '<' } }), scanner.tokens[0]
-        assert_attributes ({ type: :tag_name, loc: { start: 1, stop: 3, source: "div" } }), scanner.tokens[1]
-        assert_attributes ({ type: :whitespace, loc: { start: 4, stop: 4, source: " " } }), scanner.tokens[2]
-        assert_attributes ({ type: :attribute_name, loc: { start: 5, stop: 9, source: "title" } }), scanner.tokens[3]
-        assert_attributes ({ type: :equal, loc: { start: 10, stop: 10, source: "=" } }), scanner.tokens[4]
-        assert_attributes ({ type: :attribute_quoted_value_start, loc: { start: 11, stop: 11, source: "'" } }), scanner.tokens[5]
-        assert_attributes ({ type: :attribute_quoted_value, loc: { start: 12, stop: 23, source: "your store’s" } }), scanner.tokens[6]
-        assert_attributes ({ type: :attribute_quoted_value_end, loc: { start: 24, stop: 24, source: "'" } }), scanner.tokens[7]
+        assert_attributes ({ type: :tag_start, loc: { begin_pos: 0, end_pos: 1, source: '<' } }), scanner.tokens[0]
+        assert_attributes ({ type: :tag_name, loc: { begin_pos: 1, end_pos: 4, source: "div" } }), scanner.tokens[1]
+        assert_attributes ({ type: :whitespace, loc: { begin_pos: 4, end_pos: 5, source: " " } }), scanner.tokens[2]
+        assert_attributes ({ type: :attribute_name, loc: { begin_pos: 5, end_pos: 10, source: "title" } }), scanner.tokens[3]
+        assert_attributes ({ type: :equal, loc: { begin_pos: 10, end_pos: 11, source: "=" } }), scanner.tokens[4]
+        assert_attributes ({ type: :attribute_quoted_value_start, loc: { begin_pos: 11, end_pos: 12, source: "'" } }), scanner.tokens[5]
+        assert_attributes ({ type: :attribute_quoted_value, loc: { begin_pos: 12, end_pos: 24, source: "your store’s" } }), scanner.tokens[6]
+        assert_attributes ({ type: :attribute_quoted_value_end, loc: { begin_pos: 24, end_pos: 25, source: "'" } }), scanner.tokens[7]
         assert_equal code.length, scanner.current_position
       end
 
       test "expression literal" do
-        scanner = HtmlErb.new("<%= literal %>")
+        scanner = HtmlErb.new(buffer("<%= literal %>"))
         assert_equal 4, scanner.tokens.size
 
-        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
-        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: '=' } }), scanner.tokens[1]
-        assert_attributes ({ type: :code, loc: { start: 3, stop: 11, source: ' literal ' } }), scanner.tokens[2]
-        assert_attributes ({ type: :erb_end, loc: { start: 12, stop: 13, source: '%>' } }), scanner.tokens[3]
+        assert_attributes ({ type: :erb_begin, loc: { begin_pos: 0, end_pos: 2, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 2, end_pos: 3, source: '=' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { begin_pos: 3, end_pos: 12, source: ' literal ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { begin_pos: 12, end_pos: 14, source: '%>' } }), scanner.tokens[3]
       end
 
       test "expression escaped" do
-        scanner = HtmlErb.new("<%== escaped %>")
+        scanner = HtmlErb.new(buffer("<%== escaped %>"))
         assert_equal 4, scanner.tokens.size
 
-        assert_attributes ({ type: :erb_begin, loc: { start: 0, stop: 1, source: '<%' } }), scanner.tokens[0]
-        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 3, source: '==' } }), scanner.tokens[1]
-        assert_attributes ({ type: :code, loc: { start: 4, stop: 12, source: ' escaped ' } }), scanner.tokens[2]
-        assert_attributes ({ type: :erb_end, loc: { start: 13, stop: 14, source: '%>' } }), scanner.tokens[3]
+        assert_attributes ({ type: :erb_begin, loc: { begin_pos: 0, end_pos: 2, source: '<%' } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 2, end_pos: 4, source: '==' } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { begin_pos: 4, end_pos: 13, source: ' escaped ' } }), scanner.tokens[2]
+        assert_attributes ({ type: :erb_end, loc: { begin_pos: 13, end_pos: 15, source: '%>' } }), scanner.tokens[3]
       end
 
       test "line number for multi-line statements" do
-        scanner = HtmlErb.new("before <% multi\nline %> after")
+        scanner = HtmlErb.new(buffer("before <% multi\nline %> after"))
         assert_equal 5, scanner.tokens.size
 
         assert_attributes ({ type: :text, loc: { line: 1, source: 'before ' } }), scanner.tokens[0]
@@ -101,7 +101,7 @@ module BetterHtml
       end
 
       test "multi-line statements with trim" do
-        scanner = HtmlErb.new("before\n<% multi\nline -%>\nafter")
+        scanner = HtmlErb.new(buffer("before\n<% multi\nline -%>\nafter"))
         assert_equal 7, scanner.tokens.size
 
         assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]
@@ -114,7 +114,7 @@ module BetterHtml
       end
 
       test "right trim with =%>" do
-        scanner = HtmlErb.new("<% trim =%>")
+        scanner = HtmlErb.new(buffer("<% trim =%>"))
         assert_equal 4, scanner.tokens.size
 
         assert_attributes ({ type: :erb_begin, loc: { line: 1, source: '<%' } }), scanner.tokens[0]
@@ -124,7 +124,7 @@ module BetterHtml
       end
 
       test "multi-line expression with trim" do
-        scanner = HtmlErb.new("before\n<%= multi\nline -%>\nafter")
+        scanner = HtmlErb.new(buffer("before\n<%= multi\nline -%>\nafter"))
         assert_equal 8, scanner.tokens.size
 
         assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]
@@ -138,7 +138,7 @@ module BetterHtml
       end
 
       test "line counts with comments" do
-        scanner = HtmlErb.new("before\n<%# BO$$ Mode %>\nafter")
+        scanner = HtmlErb.new(buffer("before\n<%# BO$$ Mode %>\nafter"))
         assert_equal 7, scanner.tokens.size
 
         assert_attributes ({ type: :text, loc: { line: 1, source: "before\n" } }), scanner.tokens[0]

data/test/better_html/tokenizer/html_lodash_test.rb

@@ -5,55 +5,55 @@ module BetterHtml
   module Tokenizer
     class HtmlLodashTest < ActiveSupport::TestCase
       test "matches text" do
-        scanner = HtmlLodash.new("just some text")
+        scanner = HtmlLodash.new(buffer("just some text"))
         assert_equal 1, scanner.tokens.size
 
-        assert_attributes ({ type: :text, loc: { start: 0, stop: 13, source: "just some text" } }), scanner.tokens[0]
+        assert_attributes ({ type: :text, loc: { begin_pos: 0, end_pos: 14, source: "just some text" } }), scanner.tokens[0]
       end
 
       test "matches strings to be escaped" do
-        scanner = HtmlLodash.new("[%= foo %]")
+        scanner = HtmlLodash.new(buffer("[%= foo %]"))
         assert_equal 4, scanner.tokens.size
 
-        assert_attributes ({ type: :lodash_begin, loc: { start: 0, stop: 1, source: "[%" } }), scanner.tokens[0]
-        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: "=" } }), scanner.tokens[1]
-        assert_attributes ({ type: :code, loc: { start: 3, stop: 7, source: " foo " } }), scanner.tokens[2]
-        assert_attributes ({ type: :lodash_end, loc: { start: 8, stop: 9, source: "%]" } }), scanner.tokens[3]
+        assert_attributes ({ type: :lodash_begin, loc: { begin_pos: 0, end_pos: 2, source: "[%" } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 2, end_pos: 3, source: "=" } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { begin_pos: 3, end_pos: 8, source: " foo " } }), scanner.tokens[2]
+        assert_attributes ({ type: :lodash_end, loc: { begin_pos: 8, end_pos: 10, source: "%]" } }), scanner.tokens[3]
       end
 
       test "matches interpolate" do
-        scanner = HtmlLodash.new("[%! foo %]")
+        scanner = HtmlLodash.new(buffer("[%! foo %]"))
         assert_equal 4, scanner.tokens.size
 
-        assert_attributes ({ type: :lodash_begin, loc: { start: 0, stop: 1, source: "[%" } }), scanner.tokens[0]
-        assert_attributes ({ type: :indicator, loc: { start: 2, stop: 2, source: "!" } }), scanner.tokens[1]
-        assert_attributes ({ type: :code, loc: { start: 3, stop: 7, source: " foo " } }), scanner.tokens[2]
-        assert_attributes ({ type: :lodash_end, loc: { start: 8, stop: 9, source: "%]" } }), scanner.tokens[3]
+        assert_attributes ({ type: :lodash_begin, loc: { begin_pos: 0, end_pos: 2, source: "[%" } }), scanner.tokens[0]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 2, end_pos: 3, source: "!" } }), scanner.tokens[1]
+        assert_attributes ({ type: :code, loc: { begin_pos: 3, end_pos: 8, source: " foo " } }), scanner.tokens[2]
+        assert_attributes ({ type: :lodash_end, loc: { begin_pos: 8, end_pos: 10, source: "%]" } }), scanner.tokens[3]
       end
 
       test "matches statement" do
-        scanner = HtmlLodash.new("[% foo %]")
+        scanner = HtmlLodash.new(buffer("[% foo %]"))
         assert_equal 3, scanner.tokens.size
 
-        assert_attributes ({ type: :lodash_begin, loc: { start: 0, stop: 1, source: "[%" } }), scanner.tokens[0]
-        assert_attributes ({ type: :code, loc: { start: 2, stop: 6, source: " foo " } }), scanner.tokens[1]
-        assert_attributes ({ type: :lodash_end, loc: { start: 7, stop: 8, source: "%]" } }), scanner.tokens[2]
+        assert_attributes ({ type: :lodash_begin, loc: { begin_pos: 0, end_pos: 2, source: "[%" } }), scanner.tokens[0]
+        assert_attributes ({ type: :code, loc: { begin_pos: 2, end_pos: 7, source: " foo " } }), scanner.tokens[1]
+        assert_attributes ({ type: :lodash_end, loc: { begin_pos: 7, end_pos: 9, source: "%]" } }), scanner.tokens[2]
       end
 
       test "matches text before and after" do
-        scanner = HtmlLodash.new("before\n[%= foo %]\nafter")
+        scanner = HtmlLodash.new(buffer("before\n[%= foo %]\nafter"))
         assert_equal 6, scanner.tokens.size
 
-        assert_attributes ({ type: :text, loc: { start: 0, stop: 6, source: "before\n" } }), scanner.tokens[0]
-        assert_attributes ({ type: :lodash_begin, loc: { start: 7, stop: 8, source: "[%" } }), scanner.tokens[1]
-        assert_attributes ({ type: :indicator, loc: { start: 9, stop: 9, source: "=" } }), scanner.tokens[2]
-        assert_attributes ({ type: :code, loc: { start: 10, stop: 14, source: " foo " } }), scanner.tokens[3]
-        assert_attributes ({ type: :lodash_end, loc: { start: 15, stop: 16, source: "%]" } }), scanner.tokens[4]
-        assert_attributes ({ type: :text, loc: { start: 17, stop: 22, source: "\nafter" } }), scanner.tokens[5]
+        assert_attributes ({ type: :text, loc: { begin_pos: 0, end_pos: 7, source: "before\n" } }), scanner.tokens[0]
+        assert_attributes ({ type: :lodash_begin, loc: { begin_pos: 7, end_pos: 9, source: "[%" } }), scanner.tokens[1]
+        assert_attributes ({ type: :indicator, loc: { begin_pos: 9, end_pos: 10, source: "=" } }), scanner.tokens[2]
+        assert_attributes ({ type: :code, loc: { begin_pos: 10, end_pos: 15, source: " foo " } }), scanner.tokens[3]
+        assert_attributes ({ type: :lodash_end, loc: { begin_pos: 15, end_pos: 17, source: "%]" } }), scanner.tokens[4]
+        assert_attributes ({ type: :text, loc: { begin_pos: 17, end_pos: 23, source: "\nafter" } }), scanner.tokens[5]
       end
 
       test "matches multiple" do
-        scanner = HtmlLodash.new("[% if() { %][%= foo %][% } %]")
+        scanner = HtmlLodash.new(buffer("[% if() { %][%= foo %][% } %]"))
         assert_equal 10, scanner.tokens.size
 
         assert_attributes ({ type: :lodash_begin, loc: { source: "[%" } }), scanner.tokens[0]
@@ -71,7 +71,7 @@ module BetterHtml
       end
 
       test "parses out html correctly" do
-        scanner = HtmlLodash.new('<div class="[%= foo %]">')
+        scanner = HtmlLodash.new(buffer('<div class="[%= foo %]">'))
         assert_equal 12, scanner.tokens.size
         assert_equal [:tag_start, :tag_name, :whitespace, :attribute_name,
           :equal, :attribute_quoted_value_start,

data/test/better_html/tokenizer/location_test.rb

@@ -6,37 +6,53 @@ module BetterHtml
     class LocationTest < ActiveSupport::TestCase
       test "location start out of bounds" do
         e = assert_raises(ArgumentError) do
-          Location.new("foo", 5, 6)
+          Location.new(buffer("foo"), 5, 6)
         end
-        assert_equal "start location 5 is out of range for document of size 3", e.message
+        assert_equal "begin_pos location 5 is out of range for document of size 3", e.message
       end
 
       test "location stop out of bounds" do
         e = assert_raises(ArgumentError) do
-          Location.new("foo", 2, 6)
+          Location.new(buffer("foo"), 2, 6)
         end
-        assert_equal "stop location 6 is out of range for document of size 3", e.message
+        assert_equal "end_pos location 6 is out of range for document of size 3", e.message
       end
 
       test "location stop < start" do
         e = assert_raises(ArgumentError) do
-          Location.new("aaaaaa", 5, 2)
+          Location.new(buffer("aaaaaa"), 5, 2)
         end
-        assert_equal "end of range must be greater than start of range (2 < 5)", e.message
+        assert_equal "Parser::Source::Range: end_pos must not be less than begin_pos", e.message
+      end
+
+      test "location stop == start" do
+        loc = Location.new(buffer("aaaaaa"), 5, 5)
+        assert_equal "", loc.source
+        assert_equal 0, loc.size
+      end
+
+      test "end_pos is stop+1" do
+        loc = Location.new(buffer("aaaaaa"), 5, 5)
+        assert_equal 5, loc.end_pos
+      end
+
+      test "range is exclusive of last char" do
+        loc = Location.new(buffer("aaaaaa"), 5, 5)
+        assert_equal 5...5, loc.range
       end
 
       test "location calulates start and stop line and column" do
-        loc = Location.new("foo\nbar\nbaz", 5, 9)
+        loc = Location.new(buffer("foo\nbar\nbaz"), 5, 10)
 
         assert_equal "ar\nba", loc.source
         assert_equal 2, loc.start_line
         assert_equal 1, loc.start_column
         assert_equal 3, loc.stop_line
-        assert_equal 1, loc.stop_column
+        assert_equal 2, loc.stop_column
       end
 
       test "line_source_with_underline" do
-        loc = Location.new("ui_helper(foo)", 10, 12)
+        loc = Location.new(buffer("ui_helper(foo)"), 10, 13)
 
         assert_equal "foo", loc.source
         assert_equal <<~EOL.strip, loc.line_source_with_underline
@@ -46,7 +62,7 @@ module BetterHtml
       end
 
       test "line_source_with_underline removes empty spaces" do
-        loc = Location.new("   \t   ui_helper(foo)", 17, 19)
+        loc = Location.new(buffer("   \t   ui_helper(foo)"), 17, 20)
 
         assert_equal "foo", loc.source
         assert_equal <<~EOL.strip, loc.line_source_with_underline

data/test/better_html/tokenizer/token_array_test.rb

@@ -8,10 +8,11 @@ module BetterHtml
     class TokenArrayTest < ActiveSupport::TestCase
       setup do
         @document = "<x>"
+        @buffer = buffer(@document)
         @tokens = [
-          Token.new(type: :lquote, loc: Location.new(@document, 0, 0)),
-          Token.new(type: :name, loc: Location.new(@document, 1, 1)),
-          Token.new(type: :rquote, loc: Location.new(@document, 2, 2)),
+          Token.new(type: :lquote, loc: Location.new(@buffer, 0, 0)),
+          Token.new(type: :name, loc: Location.new(@buffer, 1, 1)),
+          Token.new(type: :rquote, loc: Location.new(@buffer, 2, 2)),
         ]
         @array = TokenArray.new(@tokens)
       end

data/test/better_html/tokenizer/token_test.rb

@@ -6,7 +6,7 @@ module BetterHtml
   module Tokenizer
     class TokenTest < ActiveSupport::TestCase
       test "token inspect" do
-        loc = Location.new("foo", 0, 2)
+        loc = Location.new(buffer("foo"), 0, 3)
         token = Token.new(type: :foo, loc: loc)
         assert_equal "t(:foo, \"foo\")", token.inspect
       end

data/test/test_helper.rb

@@ -18,3 +18,13 @@ if ActiveSupport::TestCase.respond_to?(:fixture_path=)
 end
 
 require 'mocha/mini_test'
+
+class ActiveSupport::TestCase
+  private
+
+  def buffer(string)
+    buffer = ::Parser::Source::Buffer.new('(test)')
+    buffer.source = string
+    buffer
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: better_html
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Francois Chagnon
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-29 00:00:00.000000000 Z
+date: 2018-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast