tilt 1.3.7 → 1.4.0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## master
 
+## 1.4.0 (2013-05-01)
+
+* Better encoding support
+
 ## 1.3.7 (2013-04-09)
 
 * Erubis: Check for the correct constant (#183, mattwildig)

data/README.md

@@ -194,6 +194,27 @@ template, but if you depend on a specific implementation, you should use #prefer
 When a file extension has a preferred template class, Tilt will *always* use
 that class, even if it raises an exception.
 
+Encodings
+---------
+
+Tilt needs to know the encoding of the template in order to work properly:
+
+Tilt will use `Encoding.default_external` as the encoding when reading external
+files. If you're mostly working with one encoding (e.g. UTF-8) we *highly*
+recommend setting this option. When providing a custom reader block (`Tilt.new
+{ custom_string }`) you'll have ensure the string is properly encoded yourself.
+
+Most of the template engines in Tilt also allows you to override the encoding
+using the `:default_encoding`-option:
+
+```ruby
+tmpl = Tilt.new('hello.erb', :default_encoding => 'Big5')
+```
+
+Ultimately it's up to the template engine how to handle the encoding: It might
+respect `:default_encoding`, it might always assume it's UTF-8 (like
+CoffeScript), or it can do its own encoding detection.
+
 Template Compilation
 --------------------
 

data/lib/tilt.rb

@@ -1,5 +1,5 @@
 module Tilt
-  VERSION = '1.3.7'
+  VERSION = '1.4.0'
 
   @preferred_mappings = Hash.new
   @template_mappings = Hash.new { |h, k| h[k] = [] }

data/lib/tilt/template.rb

@@ -65,11 +65,37 @@ module Tilt
       @default_encoding = @options.delete :default_encoding
 
       # load template data and prepare (uses binread to avoid encoding issues)
-      @reader = block || lambda { |t| File.respond_to?(:binread) ? File.binread(@file) : File.read(@file) }
+      @reader = block || lambda { |t| read_template_file }
       @data = @reader.call(self)
+
+      if @data.respond_to?(:force_encoding)
+        @data.force_encoding(default_encoding) if default_encoding
+
+        if !@data.valid_encoding?
+          raise Encoding::InvalidByteSequenceError, "#{eval_file} is not valid #{@data.encoding}"
+        end
+      end
+
       prepare
     end
 
+    # The encoding of the source data. Defaults to the
+    # default_encoding-option if present. You may override this method
+    # in your template class if you have a better hint of the data's
+    # encoding.
+    def default_encoding
+      @default_encoding
+    end
+
+    def read_template_file
+      data = File.open(file, 'rb') { |io| io.read }
+      if data.respond_to?(:force_encoding)
+        # Set it to the default external (without verifying)
+        data.force_encoding(Encoding.default_external) if Encoding.default_external
+      end
+      data
+    end
+
     # Render the template in the given scope with the locals specified. If a
     # block is given, it is typically available within the template via
     # +yield+.
@@ -156,26 +182,29 @@ module Tilt
     def precompiled(locals)
       preamble = precompiled_preamble(locals)
       template = precompiled_template(locals)
-      magic_comment = extract_magic_comment(template)
-      if magic_comment
-        # Magic comment e.g. "# coding: utf-8" has to be in the first line.
-        # So we copy the magic comment to the first line.
-        preamble = magic_comment + "\n" + preamble
+      postamble = precompiled_postamble(locals)
+      source = ''
+
+      # Ensure that our generated source code has the same encoding as the
+      # the source code generated by the template engine.
+      if source.respond_to?(:force_encoding)
+        template_encoding = extract_encoding(template)
+
+        source.force_encoding(template_encoding)
+        template.force_encoding(template_encoding)
       end
-      parts = [
-        preamble,
-        template,
-        precompiled_postamble(locals)
-      ]
-      [parts.join("\n"), preamble.count("\n") + 1]
+
+      source << preamble << "\n" << template << "\n" << postamble
+
+      [source, preamble.count("\n")+1]
     end
 
     # A string containing the (Ruby) source code for the template. The
-    # default Template#evaluate implementation requires either this method
-    # or the #precompiled method be overridden. When defined, the base
-    # Template guarantees correct file/line handling, locals support, custom
-    # scopes, and support for template compilation when the scope object
-    # allows it.
+    # default Template#evaluate implementation requires either this
+    # method or the #precompiled method be overridden. When defined,
+    # the base Template guarantees correct file/line handling, locals
+    # support, custom scopes, proper encoding, and support for template
+    # compilation.
     def precompiled_template(locals)
       raise NotImplementedError
     end
@@ -212,8 +241,13 @@ module Tilt
     def compile_template_method(locals)
       source, offset = precompiled(locals)
       method_name = "__tilt_#{Thread.current.object_id.abs}"
-      method_source = <<-RUBY
-        #{extract_magic_comment source}
+      method_source = ""
+
+      if method_source.respond_to?(:force_encoding)
+        method_source.force_encoding(source.encoding) 
+      end
+
+      method_source << <<-RUBY
         TOPOBJECT.class_eval do
           def #{method_name}(locals)
             Thread.current[:tilt_vars] = [self, locals]
@@ -234,13 +268,22 @@ module Tilt
       method
     end
 
+    def extract_encoding(script)
+      extract_magic_comment(script) || script.encoding
+    end
+
     def extract_magic_comment(script)
-      comment = script.slice(/\A[ \t]*\#.*coding\s*[=:]\s*([[:alnum:]\-_]+).*$/)
-      if comment && !%w[ascii-8bit binary].include?($1.downcase)
-        comment
-      elsif @default_encoding
-        "# coding: #{@default_encoding}"
+      binary script do
+        script[/\A[ \t]*\#.*coding\s*[=:]\s*([[:alnum:]\-_]+).*$/n, 1]
       end
     end
+
+    def binary(string)
+      original_encoding = string.encoding
+      string.force_encoding(Encoding::BINARY)
+      yield
+    ensure
+      string.force_encoding(original_encoding)
+    end
   end
 end

data/test/tilt_markdown_test.rb

@@ -18,7 +18,7 @@ module MarkdownTests
   end
 
   def normalize(html)
-    Nokogiri::HTML.fragment(html).to_s
+    Nokogiri::HTML.fragment(html).to_s.strip
   end
 
   def nrender(text, options = {})

data/test/tilt_template_test.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'contest'
 require 'tilt'
 require 'tempfile'
@@ -176,4 +177,100 @@ class TiltTemplateTest < Test::Unit::TestCase
     inst = SourceGeneratingMockTemplate.new { |t| 'Hey #{CONSTANT}!' }
     assert_equal "Hey Bob!", inst.render(Person.new("Joe"))
   end
+
+  ##
+  # Encodings
+
+  class DynamicMockTemplate < MockTemplate
+    def precompiled_template(locals)
+      options[:code]
+    end
+  end
+
+  class UTF8Template < MockTemplate
+    def default_encoding
+      Encoding::UTF_8
+    end
+  end
+
+  if ''.respond_to?(:encoding)
+    original_encoding = Encoding.default_external
+
+    setup do
+      @file = Tempfile.open('template')
+      @file.puts "stuff"
+      @file.close
+      @template = @file.path
+    end
+
+    teardown do
+      Encoding.default_external = original_encoding
+      Encoding.default_internal = nil
+      @file.delete
+    end
+
+    test "reading from file assumes default external encoding" do
+      Encoding.default_external = 'Big5'
+      inst = MockTemplate.new(@template)
+      assert_equal 'Big5', inst.data.encoding.to_s
+    end
+
+    test "reading from file with a :default_encoding overrides default external" do
+      Encoding.default_external = 'Big5'
+      inst = MockTemplate.new(@template, :default_encoding => 'GBK')
+      assert_equal 'GBK', inst.data.encoding.to_s
+    end
+
+    test "reading from file with default_internal set does no transcoding" do
+      Encoding.default_internal = 'utf-8'
+      Encoding.default_external = 'Big5'
+      inst = MockTemplate.new(@template)
+      assert_equal 'Big5', inst.data.encoding.to_s
+    end
+
+    test "using provided template data verbatim when given as string" do
+      Encoding.default_internal = 'Big5'
+      inst = MockTemplate.new(@template) { "blah".force_encoding('GBK') }
+      assert_equal 'GBK', inst.data.encoding.to_s
+    end
+
+    test "uses the template from the generated source code" do
+      tmpl = "ふが"
+      code = tmpl.inspect.encode('Shift_JIS')
+      inst = DynamicMockTemplate.new(:code => code) { '' }
+      res = inst.render
+      assert_equal 'Shift_JIS', res.encoding.to_s
+      assert_equal tmpl, res.encode(tmpl.encoding)
+    end
+
+    test "uses the magic comment from the generated source code" do
+      tmpl = "ふが"
+      code = ("# coding: Shift_JIS\n" + tmpl.inspect).encode('Shift_JIS')
+      # Set it to an incorrect encoding
+      code.force_encoding('UTF-8')
+
+      inst = DynamicMockTemplate.new(:code => code) { '' }
+      res = inst.render
+      assert_equal 'Shift_JIS', res.encoding.to_s
+      assert_equal tmpl, res.encode(tmpl.encoding)
+    end
+
+    test "uses #default_encoding instead of default_external" do
+      Encoding.default_external = 'Big5'
+      inst = UTF8Template.new(@template)
+      assert_equal 'UTF-8', inst.data.encoding.to_s
+    end
+
+    test "uses #default_encoding instead of current encoding" do
+      tmpl = "".force_encoding('Big5')
+      inst = UTF8Template.new(@template) { tmpl }
+      assert_equal 'UTF-8', inst.data.encoding.to_s
+    end
+
+    test "raises error if the encoding is not valid" do
+      assert_raises(Encoding::InvalidByteSequenceError) do
+        UTF8Template.new(@template) { "\xe4" }
+      end
+    end
+  end
 end

data/tilt.gemspec

@@ -3,8 +3,8 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
 
   s.name = 'tilt'
-  s.version = '1.3.7'
-  s.date = '2013-04-09'
+  s.version = '1.4.0'
+  s.date = '2013-05-01'
 
   s.description = "Generic interface to multiple Ruby template engines"
   s.summary     = s.description

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tilt
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-09 00:00:00.000000000 Z
+date: 2013-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asciidoctor