sass 3.2.0.alpha.53 → 3.2.0.alpha.54

data/REVISION

@@ -1 +1 @@
-d73070e1f6966118f181117faaf9c4cdd6a7d2a0
+594a6d23b2e7b60d1b522c1c2575c9db8681e9f7

data/VERSION

@@ -1 +1 @@
-3.2.0.alpha.53
+3.2.0.alpha.54

data/lib/sass/css.rb

@@ -1,7 +1,6 @@
 require File.dirname(__FILE__) + '/../sass'
 require 'sass/tree/node'
 require 'sass/scss/css_parser'
-require 'strscan'
 
 module Sass
   # This class converts CSS documents into Sass or SCSS templates.

data/lib/sass/engine.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'set'
 require 'digest/sha1'
 require 'sass/cache_stores'
@@ -570,7 +569,7 @@ WARNING
     end
 
     def parse_property_or_rule(line)
-      scanner = StringScanner.new(line.text)
+      scanner = Sass::Util::MultibyteStringScanner.new(line.text)
       hack_char = scanner.scan(/[:\*\.]|\#(?!\{)/)
       parser = Sass::SCSS::SassParser.new(scanner, @options[:filename], @line)
 
@@ -758,7 +757,7 @@ WARNING
       raise SyntaxError.new("Illegal nesting: Nothing may be nested beneath import directives.",
         :line => @line + 1) unless line.children.empty?
 
-      scanner = StringScanner.new(value)
+      scanner = Sass::Util::MultibyteStringScanner.new(value)
       values = []
 
       loop do

data/lib/sass/script.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'sass/script/node'
 require 'sass/script/variable'
 require 'sass/script/funcall'

data/lib/sass/script/lexer.rb

@@ -1,7 +1,5 @@
 require 'sass/scss/rx'
 
-require 'strscan'
-
 module Sass
   module Script
     # The lexical analyzer for SassScript.
@@ -126,7 +124,7 @@ module Sass
       # @param options [{Symbol => Object}] An options hash;
       #   see {file:SASS_REFERENCE.md#sass_options the Sass options documentation}
       def initialize(str, line, offset, options)
-        @scanner = str.is_a?(StringScanner) ? str : StringScanner.new(str)
+        @scanner = str.is_a?(StringScanner) ? str : Sass::Util::MultibyteStringScanner.new(str)
         @line = line
         @offset = offset
         @options = options

data/lib/sass/scss/parser.rb

@@ -1,4 +1,3 @@
-require 'strscan'
 require 'set'
 
 module Sass
@@ -50,7 +49,7 @@ module Sass
           if @template.is_a?(StringScanner)
             @template
           else
-            StringScanner.new(@template.gsub("\r", ""))
+            Sass::Util::MultibyteStringScanner.new(@template.gsub("\r", ""))
           end
       end
 
@@ -915,7 +914,7 @@ MESSAGE
         if @throw_err
           throw :_sass_parser_error, err
         else
-          @scanner = StringScanner.new(@scanner.string)
+          @scanner = Sass::Util::MultibyteStringScanner.new(@scanner.string)
           @scanner.pos = err[:pos]
           @line = err[:line]
           @expected = err[:expected]

data/lib/sass/shared.rb

@@ -1,5 +1,3 @@
-require 'strscan'
-
 module Sass
   # This module contains functionality that's shared between Haml and Sass.
   module Shared
@@ -16,7 +14,7 @@ module Sass
     # @yieldparam scan [StringScanner] The scanner scanning through the string
     # @return [String] The text remaining in the scanner after all `#{`s have been processed
     def handle_interpolation(str)
-      scan = StringScanner.new(str)
+      scan = Sass::Util::MultibyteStringScanner.new(str)
       yield scan while scan.scan(/(.*?)(\\*)\#\{/m)
       scan.rest
     end
@@ -40,7 +38,7 @@ module Sass
     #   `["Foo (Bar (Baz bang) bop)", " (Bang (bop bip))"]` in the example above.
     def balance(scanner, start, finish, count = 0)
       str = ''
-      scanner = StringScanner.new(scanner) unless scanner.is_a? StringScanner
+      scanner = Sass::Util::MultibyteStringScanner.new(scanner) unless scanner.is_a? StringScanner
       regexp = Regexp.new("(.*?)[\\#{start.chr}\\#{finish.chr}]", Regexp::MULTILINE)
       while scanner.scan(regexp)
         str << scanner.matched

data/lib/sass/util.rb

@@ -2,7 +2,6 @@ require 'erb'
 require 'set'
 require 'enumerator'
 require 'stringio'
-require 'strscan'
 require 'rbconfig'
 
 require 'sass/root'
@@ -719,3 +718,5 @@ MSG
     end
   end
 end
+
+require 'sass/util/multibyte_string_scanner'

data/lib/sass/util/multibyte_string_scanner.rb

@@ -0,0 +1,134 @@
+require 'strscan'
+
+if Sass::Util.ruby1_8?
+  Sass::Util::MultibyteStringScanner = StringScanner
+else
+  # A wrapper of the native StringScanner class that works correctly with
+  # multibyte character encodings. The native class deals only in bytes, not
+  # characters, for methods like [#pos] and [#matched_size]. This class deals
+  # only in characters, instead.
+  class Sass::Util::MultibyteStringScanner < StringScanner
+    def self.new(str)
+      return StringScanner.new(str) if str.ascii_only?
+      super
+    end
+
+    def initialize(str)
+      super
+      @mb_pos = 0
+      @mb_matched_size = nil
+      @mb_last_pos = nil
+    end
+
+    alias_method :byte_pos, :pos
+    alias_method :byte_matched_size, :matched_size
+
+    def check(pattern); _match super; end
+    def check_until(pattern); _matched super; end
+    def getch; _forward _match super; end
+    def match?(pattern); _size check(pattern); end
+    def matched_size; @mb_matched_size; end
+    def peek(len); string[@mb_pos, len]; end
+    alias_method :peep, :peek
+    def pos; @mb_pos; end
+    alias_method :pointer, :pos
+    def rest_size; rest.size; end
+    def scan(pattern); _forward _match super; end
+    def scan_until(pattern); _forward _matched super; end
+    def skip(pattern); _size scan(pattern); end
+    def skip_until(pattern); _matched _size scan_until(pattern); end
+
+    def get_byte
+      raise "MultibyteStringScanner doesn't support #get_byte."
+    end
+
+    def getbyte
+      raise "MultibyteStringScanner doesn't support #getbyte."
+    end
+
+    def pos=(n)
+      @mb_last_pos = nil
+
+      # We set position kind of a lot during parsing, so we want it to be as
+      # efficient as possible. This is complicated by the fact that UTF-8 is a
+      # variable-length encoding, so it's difficult to find the byte length that
+      # corresponds to a given character length.
+      #
+      # Our heuristic here is to try to count the fewest possible characters. So
+      # if the new position is close to the current one, just count the
+      # characters between the two; if the new position is closer to the
+      # beginning of the string, just count the characters from there.
+      if @mb_pos - n < @mb_pos / 2
+        # New position is close to old position
+        byte_delta = @mb_pos > n ? -string[n...@mb_pos].bytesize : string[@mb_pos...n].bytesize
+        super(byte_pos + byte_delta)
+      else
+        # New position is close to BOS
+        super(string[0...n].bytesize)
+      end
+      @mb_pos = n
+    end
+
+    def reset
+      @mb_pos = 0
+      @mb_matched_size = nil
+      @mb_last_pos = nil
+      super
+    end
+
+    def scan_full(pattern, advance_pointer_p, return_string_p)
+      res = _match super(pattern, advance_pointer_p, true)
+      _forward res if advance_pointer_p
+      return res if return_string_p
+    end
+
+    def search_full(pattern, advance_pointer_p, return_string_p)
+      res = super(pattern, advance_pointer_p, true)
+      _forward res if advance_pointer_p
+      _matched((res if return_string_p))
+    end
+
+    def string=(str)
+      @mb_pos = 0
+      @mb_matched_size = nil
+      @mb_last_pos = nil
+      super
+    end
+
+    def terminate
+      @mb_pos = string.size
+      @mb_matched_size = nil
+      @mb_last_pos = nil
+      super
+    end
+    alias_method :clear, :terminate
+
+    def unscan
+      super
+      @mb_pos = @mb_last_pos
+      @mb_last_pos = @mb_matched_size = nil
+    end
+
+    private
+
+    def _size(str)
+      str && str.size
+    end
+
+    def _match(str)
+      @mb_matched_size = str && str.size
+      str
+    end
+
+    def _matched(res)
+      _match matched
+      res
+    end
+
+    def _forward(str)
+      @mb_last_pos = @mb_pos
+      @mb_pos += str.size if str
+      str
+    end
+  end
+end

data/test/sass/engine_test.rb

@@ -2545,6 +2545,31 @@ CSS
   a: b
 SASS
     end
+
+    # Encoding Regression Test
+
+    def test_multibyte_prop_name
+      assert_equal(<<CSS, render(<<SASS))
+@charset "UTF-8";
+#bar {
+  cölor: blue; }
+CSS
+#bar
+  cölor: blue
+SASS
+    end
+
+    def test_multibyte_and_interpolation
+      assert_equal(<<CSS, render(<<SCSS, :syntax => :scss))
+#bar {
+  background: a 0%; }
+CSS
+#bar {
+  // 
+  background: \#{a} 0%;
+}
+SCSS
+    end
   end
 
   def test_original_filename_set

data/test/sass/util/multibyte_string_scanner_test.rb

@@ -0,0 +1,145 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+require File.dirname(__FILE__) + '/../../test_helper'
+
+class MultibyteStringScannerTest < Test::Unit::TestCase
+  def setup
+    @scanner = Sass::Util::MultibyteStringScanner.new("cölorfül")
+  end
+
+  def test_initial
+    assert_scanner_state 0, 0, nil, nil
+  end
+
+  def test_check
+    assert_equal 'cö', @scanner.check(/../)
+    assert_scanner_state 0, 0, 2, 3
+    assert_equal 0, @scanner.pos
+    assert_equal 0, @scanner.pos
+    assert_equal 2, @scanner.matched_size
+    assert_equal 3, @scanner.byte_matched_size
+  end
+
+  def test_check_until
+    assert_equal 'cölorfü', @scanner.check_until(/f./)
+    assert_scanner_state 0, 0, 2, 3
+  end
+
+  def test_getch
+    assert_equal 'c', @scanner.getch
+    assert_equal 'ö', @scanner.getch
+    assert_scanner_state 2, 3, 1, 2
+  end
+
+  def test_match?
+    assert_equal 2, @scanner.match?(/../)
+    assert_scanner_state 0, 0, 2, 3
+  end
+
+  def test_peek
+    assert_equal 'cö', @scanner.peek(2)
+    assert_scanner_state 0, 0, nil, nil
+  end
+
+  def test_rest_size
+    assert_equal 'cö', @scanner.scan(/../)
+    assert_equal 6, @scanner.rest_size
+  end
+
+  def test_scan
+    assert_equal 'cö', @scanner.scan(/../)
+    assert_scanner_state 2, 3, 2, 3
+  end
+
+  def test_scan_until
+    assert_equal 'cölorfü', @scanner.scan_until(/f./)
+    assert_scanner_state 7, 9, 2, 3
+  end
+
+  def test_skip
+    assert_equal 2, @scanner.skip(/../)
+    assert_scanner_state 2, 3, 2, 3
+  end
+
+  def test_skip_until
+    assert_equal 7, @scanner.skip_until(/f./)
+    assert_scanner_state 7, 9, 2, 3
+  end
+
+  def test_set_pos
+    @scanner.pos = 7
+    assert_scanner_state 7, 9, nil, nil
+    @scanner.pos = 6
+    assert_scanner_state 6, 7, nil, nil
+    @scanner.pos = 1
+    assert_scanner_state 1, 1, nil, nil
+  end
+
+  def test_reset
+    @scanner.scan(/../)
+    @scanner.reset
+    assert_scanner_state 0, 0, nil, nil
+  end
+
+  def test_scan_full
+    assert_equal 'cö', @scanner.scan_full(/../, true, true)
+    assert_scanner_state 2, 3, 2, 3
+
+    @scanner.reset
+    assert_equal 'cö', @scanner.scan_full(/../, false, true)
+    assert_scanner_state 0, 0, 2, 3
+
+    @scanner.reset
+    assert_nil @scanner.scan_full(/../, true, false)
+    assert_scanner_state 2, 3, 2, 3
+
+    @scanner.reset
+    assert_nil @scanner.scan_full(/../, false, false)
+    assert_scanner_state 0, 0, 2, 3
+  end
+
+  def test_search_full
+    assert_equal 'cölorfü', @scanner.search_full(/f./, true, true)
+    assert_scanner_state 7, 9, 2, 3
+
+    @scanner.reset
+    assert_equal 'cölorfü', @scanner.search_full(/f./, false, true)
+    assert_scanner_state 0, 0, 2, 3
+
+    @scanner.reset
+    assert_nil @scanner.search_full(/f./, true, false)
+    assert_scanner_state 7, 9, 2, 3
+
+    @scanner.reset
+    assert_nil @scanner.search_full(/f./, false, false)
+    assert_scanner_state 0, 0, 2, 3
+  end
+
+  def test_set_string
+    @scanner.scan(/../)
+    @scanner.string = 'föóbâr'
+    assert_scanner_state 0, 0, nil, nil
+  end
+
+  def test_terminate
+    @scanner.scan(/../)
+    @scanner.terminate
+    assert_scanner_state 8, 10, nil, nil
+  end
+
+  def test_unscan
+    @scanner.scan(/../)
+    @scanner.scan_until(/f./)
+    @scanner.unscan
+    assert_scanner_state 2, 3, nil, nil
+  end
+
+  private
+
+  def assert_scanner_state(pos, byte_pos, matched_size, byte_matched_size)
+    assert_equal pos, @scanner.pos, 'pos'
+    assert_equal byte_pos, @scanner.byte_pos, 'byte_pos'
+    assert_equal matched_size, @scanner.matched_size, 'matched_size'
+    assert_equal byte_matched_size, @scanner.byte_matched_size, 'byte_matched_size'
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: sass
 version: !ruby/object:Gem::Version 
-  hash: 592302967
+  hash: 592302961
   prerelease: 6
   segments: 
   - 3
   - 2
   - 0
   - alpha
-  - 53
-  version: 3.2.0.alpha.53
+  - 54
+  version: 3.2.0.alpha.54
 platform: ruby
 authors: 
 - Nathan Weizenbaum
@@ -19,7 +19,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-01-04 00:00:00 -05:00
+date: 2012-01-05 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -164,6 +164,7 @@ files:
 - lib/sass/tree/while_node.rb
 - lib/sass/util.rb
 - lib/sass/util/subset_map.rb
+- lib/sass/util/multibyte_string_scanner.rb
 - lib/sass/version.rb
 - vendor/fssm/Gemfile
 - vendor/fssm/LICENSE
@@ -306,6 +307,7 @@ files:
 - test/sass/templates/warn_imported.sass
 - test/sass/test_helper.rb
 - test/sass/util/subset_map_test.rb
+- test/sass/util/multibyte_string_scanner_test.rb
 - test/sass/util_test.rb
 - test/test_helper.rb
 - extra/update_watch.rb
@@ -374,4 +376,5 @@ test_files:
 - test/sass/scss/rx_test.rb
 - test/sass/scss/scss_test.rb
 - test/sass/util/subset_map_test.rb
+- test/sass/util/multibyte_string_scanner_test.rb
 - test/sass/util_test.rb