file_with_bom 0.1.0

data/lib/file_with_bom.rb

@@ -0,0 +1,133 @@
+=begin rdoc
+==Write and read Unicode-Files with Bom
+
+Unicode may include a
+(BOM)[http://en.wikipedia.org/wiki/Byte_order_mark].
+When reading such a file, you must remove the BOM.
+When saving such a file, you must add the BOM at the begin of the file.
+ 
+This gem extends the File class to provide some additional values.
+
+==Ruby >= 1.9.2
+
+Ruby has a BOM-supports since Ruby 1.9.2 Revision 28199
+
+You may use:
+
+    File.open('file.txt', "r:bom|utf-8"){|f|
+      content = f.read
+    }
+
+or
+
+    File.open('file.txt', "r", :encoding => "BOM|UTF-8"){|f|
+      content = f.read
+    }
+
+Details see http://redmine.ruby-lang.org/issues/show/3407 and http://bugs.ruby-lang.org/projects/ruby-trunk/repository/revisions/28199
+
+There is no support for w:bom|...
+
+==History
+This gem is based on
+http://stackoverflow.com/questions/9886705/how-to-write-bom-marker-to-a-file-in-ruby/9887927#9887927
+
+=end
+
+
+=begin rdoc
+Extend File with some BOM-Handling.
+=end
+class File
+  #BOMs for different encodings.
+  BOM_LIST_hex = {
+      'UTF_8'      => "\xEF\xBB\xBF", #"\uEFBBBF"
+      'UTF_16BE' => "\xFE\xFF", #"\uFEFF",
+      'UTF_16LE' => "\xFF\xFE",
+      'UTF_32BE' => "\x00\x00\xFE\xFF",
+      'UTF_32LE' => "\xFE\xFF\x00\x00",
+    }
+    #Define alias names for ruby 1.9
+    if defined? Encoding
+      BOM_LIST_hex[Encoding::UTF_8]     = BOM_LIST_hex['UTF_8']
+      BOM_LIST_hex[Encoding::UTF_16BE] = BOM_LIST_hex['UTF_16BE']
+      BOM_LIST_hex[Encoding::UTF_16LE] = BOM_LIST_hex['UTF_16LE']
+      BOM_LIST_hex[Encoding::UTF_32BE] = BOM_LIST_hex['UTF_32BE']
+      BOM_LIST_hex[Encoding::UTF_32LE] = BOM_LIST_hex['UTF_32LE']
+    end
+
+  #~ BOM_LIST_hex.freeze
+
+=begin rdoc
+Get BOM for the 'external_encoding'.
+
+You may use it like this:
+
+    File.open(filename, "w:utf-16le"){|f|
+      f << f.utf_bom  #add the BOM manual
+      f << 'some content'
+    }
+=end
+  def utf_bom_hex(encoding = external_encoding )
+    BOM_LIST_hex[encoding].force_encoding(encoding) #ruby 1.9
+  end
+  #Class options
+  class << File
+    #Store the old File.open
+    alias :open_old :open
+=begin rdoc
+Redefine open to support BOM.
+
+This modification allow the usage of encodings like "utf-8-bom".
+This encodings can be used in read- and write-mode. 
+
+Examples:
+
+  File.open("file.txt", "w:utf-16le-bom"){|f|
+    f << 'some content'
+  }
+  File.open("file.txt", "w:utf-16le", :bom => true ){|f|
+    f << 'some content'
+  }
+
+==Remark
+Ruby 1.9.2 supports already BOMs in read mode (e.g. "r:bom|utf-8").
+
+The syntactical difference (uft-8-bom instead bom|utf-8) is wanted to separate 
+the two logics.
+
+* This gem does not support ruby 1.8 (makes no sense, you may store the BOM, but the conntent will not ne unicode).
+* This gem supports also BOMs in write mode.
+
+=end
+    def open(filename, mode_string = 'r', options = {}, &block)
+      #~ puts "! %-10s %-20s %s" % [mode_string, filename, options.inspect] #only for tests
+      
+      #check for bom-flag in mode_string
+      options[:bom] = true if mode_string.sub!('-bom','')
+      
+      f = open_old(filename, mode_string, options)
+        
+      if options[:bom]
+        case mode_string
+          when /\Ar/   #read mode -> remove BOM
+            #remove BOM
+            bom = f.read(f.utf_bom_hex.bytesize) 
+            #check, if it was really a bom
+            if bom != f.utf_bom_hex
+              f.rewind  #return to position 0 if BOM was no BOM
+            end
+          when /\Aw/  #write mode -> attach BOM
+            f << f.utf_bom_hex
+        end #mode_string
+      end
+      
+      if block_given?
+        yield f 
+        f.close
+      end
+    end
+  end #File - class options
+ 
+end #File
+

data/unittest/test_file_with_bom.rb

@@ -0,0 +1,81 @@
+#encoding: utf-8
+$:.unshift('../lib')
+require 'file_with_bom'
+
+#~ gem 'test-unit'
+require 'test/unit'
+
+EXAMPLE_TEXT = 'some content öäü'
+TESTDIR = File.join(File.dirname(File.expand_path(__FILE__)), 'testfiles')
+
+#Hash with testfiles
+TESTFILE = {}
+def make_testdata
+    Dir.mkdir(TESTDIR) unless File.exist?(TESTDIR)
+    %w{utf-8 utf-16le utf-16be utf-32le utf-32be}.each{|encoding|
+    
+      filename = TESTFILE[encoding] = File.join(TESTDIR, "#{encoding}.txt")
+      File.open(filename, "w", :encoding => encoding){|f|
+        f << EXAMPLE_TEXT 
+      } unless File.exist?(filename)
+      
+      filename = TESTFILE[encoding+'-bom'] = File.join(TESTDIR, "#{encoding}_bom.txt")
+      File.open(filename, "w", :encoding => encoding){|f|
+        f << f.utf_bom_hex #add the BOM
+        f << EXAMPLE_TEXT 
+      } unless File.exist?(filename)
+    } #encodings
+end #make_testdata
+make_testdata #testdata should be part of the distribution. Call is needed to fill TESTFILE
+
+
+class BOM_Test < Test::Unit::TestCase
+  def setup
+    @files_to_delete = []
+  end
+  def teardown
+    @files_to_delete.each{|filename|
+      File.delete(filename) if File.exist?(filename)
+    }
+  end
+  def test_utf_bom_hex
+
+    assert_equal( File::BOM_LIST_hex['UTF_8'], File.new(TESTFILE['utf-8'], 'r:utf-8').utf_bom_hex )
+    assert_equal( File::BOM_LIST_hex['UTF_16LE'], File.new(TESTFILE['utf-16le'], 'rb:utf-16le').utf_bom_hex )
+    assert_equal( File::BOM_LIST_hex['UTF_16BE'], File.new(TESTFILE['utf-16be'], 'rb:utf-16be').utf_bom_hex )
+    assert_equal( File::BOM_LIST_hex['UTF_32LE'], File.new(TESTFILE['utf-32le'], 'rb:utf-32le').utf_bom_hex )
+    assert_equal( File::BOM_LIST_hex['UTF_32BE'], File.new(TESTFILE['utf-32be'], 'rb:utf-32be').utf_bom_hex )
+
+    if defined? Encoding  #ruby 1.9
+      assert_equal( File::BOM_LIST_hex[Encoding::UTF_8], File.new(TESTFILE['utf-8'], 'r:utf-8').utf_bom_hex )
+      assert_equal( File::BOM_LIST_hex[Encoding::UTF_16LE], File.new(TESTFILE['utf-16le'], 'rb:utf-16le').utf_bom_hex )
+      assert_equal( File::BOM_LIST_hex[Encoding::UTF_16BE], File.new(TESTFILE['utf-16be'], 'rb:utf-16be').utf_bom_hex )
+      assert_equal( File::BOM_LIST_hex[Encoding::UTF_32LE], File.new(TESTFILE['utf-32le'], 'rb:utf-32le').utf_bom_hex )
+      assert_equal( File::BOM_LIST_hex[Encoding::UTF_32BE], File.new(TESTFILE['utf-32be'], 'rb:utf-32be').utf_bom_hex )
+    end
+
+  end
+  def mk_file_test(encoding)
+    @files_to_delete << filename = "testfile1_#{encoding}.txt"
+    File.open( filename, "w:#{encoding}"){|f| f << EXAMPLE_TEXT }
+    assert_equal( File.read(TESTFILE[encoding]), File.read(filename))
+    
+    #Same test with other interface
+    encoding2 = encoding.gsub('-bom','')
+    @files_to_delete << filename = "testfile2_#{encoding2}.txt"
+    File.open( filename, "w", :encoding => encoding2, :bom => encoding.include?('-bom')){|f| f << EXAMPLE_TEXT }
+    assert_equal( File.read(TESTFILE[encoding]), File.read(filename))
+  end
+
+  def test_utf_8;        mk_file_test('utf-8'); end
+  def test_utf_8_bom;    mk_file_test('utf-8-bom'); end
+  def test_utf_16le;     mk_file_test('utf-16le'); end
+  def test_utf_16le_bom; mk_file_test('utf-16le-bom'); end
+  def test_utf_16be;     mk_file_test('utf-16be'); end
+  def test_utf_16be_bom; mk_file_test('utf-16be-bom'); end
+  def test_utf_32le;     mk_file_test('utf-32le'); end
+  def test_utf_32le_bom; mk_file_test('utf-32le-bom'); end
+  def test_utf_32be;     mk_file_test('utf-32be'); end
+  def test_utf_32be_bom; mk_file_test('utf-32be-bom'); end
+  
+end #class BOM_Test

data/unittest/testfiles/utf-8.txt

@@ -0,0 +1 @@
+some content öäü

data/unittest/testfiles/utf-8_bom.txt

@@ -0,0 +1 @@
+some content öäü

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: file_with_bom
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Knut Lickert
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-28 00:00:00.000000000Z
+dependencies: []
+description: ! 'Handle BOM for unicode files
+
+'
+email: knut@lickert.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/file_with_bom.rb
+- unittest/test_file_with_bom.rb
+- unittest/testfiles/utf-8.txt
+- unittest/testfiles/utf-8_bom.txt
+- unittest/testfiles/utf-16be.txt
+- unittest/testfiles/utf-16be_bom.txt
+- unittest/testfiles/utf-16le.txt
+- unittest/testfiles/utf-16le_bom.txt
+- unittest/testfiles/utf-32be.txt
+- unittest/testfiles/utf-32be_bom.txt
+- unittest/testfiles/utf-32le.txt
+- unittest/testfiles/utf-32le_bom.txt
+homepage: http://stackoverflow.com/questions/9886705/how-to-write-bom-marker-to-a-file-in-ruby/9887927#9887927
+licenses: []
+post_install_message: 
+rdoc_options:
+- --main
+- lib/file_with_bom.rb
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: Handle BOM for unicode files
+test_files:
+- unittest/test_file_with_bom.rb
+- unittest/testfiles/utf-8.txt
+- unittest/testfiles/utf-8_bom.txt
+- unittest/testfiles/utf-16be.txt
+- unittest/testfiles/utf-16be_bom.txt
+- unittest/testfiles/utf-16le.txt
+- unittest/testfiles/utf-16le_bom.txt
+- unittest/testfiles/utf-32be.txt
+- unittest/testfiles/utf-32be_bom.txt
+- unittest/testfiles/utf-32le.txt
+- unittest/testfiles/utf-32le_bom.txt