live_ast 0.2.1 → 0.2.2

data/CHANGES.rdoc

@@ -1,6 +1,10 @@
 
 = LiveAST ChangeLog
 
+== Version 0.2.2
+
+* handle utf-8 BOMs in loading
+
 == Version 0.2.1
 
 * fixed some rdoc issues

data/MANIFEST

@@ -12,6 +12,7 @@ lib/live_ast/evaler.rb
 lib/live_ast/linker.rb
 lib/live_ast/loader.rb
 lib/live_ast/parser.rb
+lib/live_ast/reader.rb
 lib/live_ast/replace_load.rb
 lib/live_ast/replace_raise.rb
 lib/live_ast/to_ast.rb
@@ -31,8 +32,11 @@ test/encoding_test/default.rb
 test/encoding_test/eucjp.rb
 test/encoding_test/koi8.rb
 test/encoding_test/koi8_shebang.rb
+test/encoding_test/koi8_with_utf8bom.rb
 test/encoding_test/usascii.rb
+test/encoding_test/usascii_with_utf8bom.rb
 test/encoding_test/utf8.rb
+test/encoding_test/utf8bom.rb
 test/error_test.rb
 test/eval_test.rb
 test/flush_cache_test.rb

data/lib/live_ast/base.rb

@@ -1,9 +1,10 @@
 require 'thread'
 
+require 'live_ast/reader'
 require 'live_ast/parser'
-require 'live_ast/loader'
 require 'live_ast/evaler'
 require 'live_ast/linker'
+require 'live_ast/loader'
 require 'live_ast/error'
 
 module LiveAST

data/lib/live_ast/linker.rb

@@ -85,7 +85,7 @@ module LiveAST
           # File was loaded by 'require'.
           # Play catch-up: assume it has not changed in the meantime.
           #
-          _, cache = new_cache(Loader.read(file), file, 1, true)
+          _, cache = new_cache(Reader.read(file), file, 1, true)
         end
         cache.fetch_ast(line) if cache
       end

data/lib/live_ast/loader.rb

@@ -1,9 +1,5 @@
-# encoding: us-ascii
-
 module LiveAST
   module Loader
-    MAGIC_COMMENT = /\A(?:#!.*?\n)?\s*\#.*(?:en)?coding\s*[:=]\s*([^\s;]+)/
-
     class << self
       def load(file, wrap)
         file = find_file(file)
@@ -11,7 +7,7 @@ module LiveAST
         # guards to protect toplevel locals
         header, footer, warnings_ok = header_footer(wrap)
   
-        parser_src = read(file)
+        parser_src = Reader.read(file)
         evaler_src = header << parser_src << footer
         
         run = lambda do
@@ -21,12 +17,6 @@ module LiveAST
         true
       end
   
-      def read(file)
-        contents = File.read(file, :encoding => "BINARY")
-        encoding = contents[MAGIC_COMMENT, 1] || "US-ASCII"
-        contents.force_encoding(encoding)
-      end
-
       def header_footer(wrap)
         if wrap
           return "class << Object.new;", ";end", true

data/lib/live_ast/reader.rb

@@ -0,0 +1,18 @@
+# encoding: us-ascii
+module LiveAST
+  module Reader
+    UTF8_BOM = /\A\xef\xbb\xbf/
+    MAGIC_COMMENT = /\A(?:#!.*?\n)?\s*\#.*(?:en)?coding\s*[:=]\s*([^\s;]+)/
+
+    def self.read(file)
+      contents = File.read(file, :encoding => "BINARY")
+      
+      utf8 = contents.sub!(UTF8_BOM, "")
+
+      # magic comment overrides BOM
+      encoding = contents[MAGIC_COMMENT, 1] || utf8 || "US-ASCII"
+
+      contents.force_encoding(encoding)
+    end
+  end
+end

data/lib/live_ast/version.rb

@@ -1,3 +1,3 @@
 module LiveAST
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/test/encoding_test/koi8_with_utf8bom.rb

@@ -0,0 +1,6 @@
+# -*- coding: koi8-r -*-
+module EncodingTest
+  def koi8_with_utf8bom_string
+    "� ���������� ������� ���������� �� 20 ��������"
+  end
+end

data/test/encoding_test/usascii_with_utf8bom.rb

@@ -0,0 +1,6 @@
+# encoding: us-ascii
+module EncodingTest
+  def usascii_with_utf8bom_string
+    "Cats and large boxes."
+  end
+end

data/test/encoding_test/utf8bom.rb

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+module EncodingTest
+  def utf8bom_string
+    "大きな箱とねこ。"
+  end
+end

data/test/encoding_test.rb

@@ -3,6 +3,9 @@ require_relative 'shared/main'
 require_relative 'encoding_test/default.rb'
 require_relative 'encoding_test/usascii.rb'
 require_relative 'encoding_test/utf8.rb'
+require_relative 'encoding_test/utf8bom.rb'
+require_relative 'encoding_test/usascii_with_utf8bom.rb'
+require_relative 'encoding_test/koi8_with_utf8bom.rb'
 require_relative 'encoding_test/cp932.rb'
 require_relative 'encoding_test/eucjp.rb'
 require_relative 'encoding_test/koi8.rb'
@@ -16,6 +19,9 @@ class AllEncodingTest < RegularTest
     default US-ASCII
     usascii US-ASCII
     utf8 UTF-8
+    utf8bom UTF-8
+    usascii_with_utf8bom US-ASCII
+    koi8_with_utf8bom KOI8-R
     cp932 Windows-31J
     eucjp EUC-JP
     koi8 KOI8-R

metadata

@@ -2,7 +2,7 @@
 name: live_ast
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors: 
 - James M. Lawrence
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-02-20 00:00:00 -05:00
+date: 2011-02-21 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,6 +47,7 @@ files:
 - lib/live_ast/linker.rb
 - lib/live_ast/loader.rb
 - lib/live_ast/parser.rb
+- lib/live_ast/reader.rb
 - lib/live_ast/replace_load.rb
 - lib/live_ast/replace_raise.rb
 - lib/live_ast/to_ast.rb
@@ -66,8 +67,11 @@ files:
 - test/encoding_test/eucjp.rb
 - test/encoding_test/koi8.rb
 - test/encoding_test/koi8_shebang.rb
+- test/encoding_test/koi8_with_utf8bom.rb
 - test/encoding_test/usascii.rb
+- test/encoding_test/usascii_with_utf8bom.rb
 - test/encoding_test/utf8.rb
+- test/encoding_test/utf8bom.rb
 - test/error_test.rb
 - test/eval_test.rb
 - test/flush_cache_test.rb
@@ -118,6 +122,8 @@ rdoc_options:
 - --exclude
 - lib/live_ast/parser.rb
 - --exclude
+- lib/live_ast/reader.rb
+- --exclude
 - lib/live_ast/replace_load.rb
 - --exclude
 - lib/live_ast/replace_raise.rb
@@ -150,10 +156,16 @@ rdoc_options:
 - --exclude
 - test/encoding_test/koi8_shebang.rb
 - --exclude
+- test/encoding_test/koi8_with_utf8bom.rb
+- --exclude
 - test/encoding_test/usascii.rb
 - --exclude
+- test/encoding_test/usascii_with_utf8bom.rb
+- --exclude
 - test/encoding_test/utf8.rb
 - --exclude
+- test/encoding_test/utf8bom.rb
+- --exclude
 - test/error_test.rb
 - --exclude
 - test/eval_test.rb