onix 0.7.3 → 0.7.4

data/CHANGELOG

@@ -1,3 +1,9 @@
+v0.7.4 (2nd September 2009)
+- Expand ONIX::Normaliser
+  - strip control chars
+  - add encoding declaration to valid utf-8 files that aren't declared
+    as such
+
 v0.7.3 (19th August 2009)
 - Switch from java to xsltproc to convert short tag ONIX files
   to reference tags

data/lib/onix.rb

@@ -16,7 +16,7 @@ module ONIX
   module Version #:nodoc:
     Major = 0
     Minor = 7
-    Tiny  = 3
+    Tiny  = 4
 
     String = [Major, Minor, Tiny].join('.')
   end

data/lib/onix/normaliser.rb

@@ -43,6 +43,7 @@ module ONIX
       raise "isutf8 app not found"   unless app_available?("isutf8")
       raise "iconv app not found"    unless app_available?("iconv")
       raise "sed app not found"      unless app_available?("sed")
+      raise "tr app not found"       unless app_available?("tr")
 
       @oldfile = oldfile
       @newfile = newfile
@@ -64,6 +65,11 @@ module ONIX
       to_utf8(@curfile, dest)
       @curfile = dest
 
+      # remove control chars
+      dest = next_tempfile
+      remove_control_chars(@curfile, dest)
+      @curfile = dest
+
       # remove entities
       replace_named_entities(@curfile)
 
@@ -102,7 +108,18 @@ module ONIX
       `xsltproc -o #{outpath} #{xsltpath} #{inpath}`
     end
 
-    # ensure the file is valid utf8, then make sure it's declared as such
+    # ensure the file is valid utf8, then make sure it's declared as such.
+    #
+    # The following behaviour is  expected:
+    #
+    #   file is valid utf8, is marked correctly
+    #     - copied untouched
+    #   file is valid utf8, is marked incorrectly or has no marked encoding
+    #     - copied and encoding mark fixed or added
+    #   file is no utf8, encoding is marked
+    #     - file is converted to utf8 and enecoding mark is updated
+    #   file is not utf8, encoding is not marked
+    #     - file is copied untouched
     #
     def to_utf8(src, dest)
       inpath = File.expand_path(src)
@@ -112,17 +129,28 @@ module ONIX
 
       # ensure the file is actually utf8
       if `isutf8 #{inpath}`.strip == ""
-        FileUtils.cp(inpath, outpath)
-      else
+        if src_enc.to_s.downcase == "utf-8"
+          FileUtils.cp(inpath, outpath)
+        else
+          FileUtils.cp(inpath, outpath)
+          `sed -i 's/<?xml.*?>/<?xml version=\"1.0\" encoding=\"UTF-8\"?>/' #{outpath}`
+        end
+      elsif src_enc
         `iconv --from-code=#{src_enc} --to-code=UTF-8 #{inpath} > #{outpath}`
-      end
-
-      # ensure the encoding delcaration is correct
-      if src_enc.downcase != "utf-8"
         `sed -i 's/#{src_enc}/UTF-8/' #{outpath}`
+      else
+          FileUtils.cp(inpath, outpath)
       end
     end
 
+    # XML files shouldn't contain low ASCII control chars. Strip them.
+    #
+    def remove_control_chars(src, dest)
+      inpath = File.expand_path(src)
+      outpath = File.expand_path(dest)
+      `cat #{inpath} | tr -d "\\000-\\010\\013\\014\\016-\\037" > #{outpath}`
+    end
+
     # replace all named entities in the specified file with
     # numeric entities.
     #

data/spec/normaliser_spec.rb

@@ -21,7 +21,6 @@ context "ONIX::Normaliser", "with a simple short tag file" do
 
     File.file?(@outfile).should be_true
     content = File.read(@outfile)
-    puts content
     content.include?("<m174>").should be_false
     content.include?("<FromCompany>").should be_true
   end
@@ -76,3 +75,51 @@ context "ONIX::Normaliser", "with an file using entities" do
     content.include?("&#x02013;").should be_true
   end
 end
+
+context "ONIX::Normaliser", "with a utf8 file that has no declared encoding" do
+
+  before(:each) do
+    @data_path = File.join(File.dirname(__FILE__),"..","data")
+    @filename  = File.join(@data_path, "no_encoding.xml")
+    @outfile   = @filename + ".new"
+  end
+
+  after(:each) do
+    File.unlink(@outfile) if File.file?(@outfile)
+  end
+
+  # this is to test for a bug where an exception was raised on files that
+  # had no declared encoding
+  specify "should add a utf-8 marker to the file" do
+    ONIX::Normaliser.process(@filename, @outfile)
+
+    File.file?(@outfile).should be_true
+    content = File.read(@outfile)
+
+    content.include?("encoding=\"UTF-8\"").should be_true
+  end
+end
+
+context "ONIX::Normaliser", "with a utf8 file that has illegal control chars" do
+
+  before(:each) do
+    @data_path = File.join(File.dirname(__FILE__),"..","data")
+    @filename  = File.join(@data_path, "control_chars.xml")
+    @outfile   = @filename + ".new"
+  end
+
+  after(:each) do
+    File.unlink(@outfile) if File.file?(@outfile)
+  end
+
+  # this is to test for a bug where an exception was raised on files that
+  # had no declared encoding
+  specify "should remove all control chars except LF, CR and TAB" do
+    ONIX::Normaliser.process(@filename, @outfile)
+
+    File.file?(@outfile).should be_true
+    content = File.read(@outfile)
+
+    content.include?("<TitleText>OXFORDPICTURE DICTIONARY CHINESE</TitleText>").should be_true
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: onix
 version: !ruby/object:Gem::Version 
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors: 
 - James Healy
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-19 00:00:00 +10:00
+date: 2009-09-02 00:00:00 +10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency