onix 0.8.0 → 0.8.1

data/CHANGELOG

@@ -1,3 +1,16 @@
+v0.8.1 (5th January 2010)
+- Use libxml2's support for transparent entity conversion when reading an ONIX file
+- Removed entity replacement from ONIX::Normaliser
+  - the external dependency on sed made me uncomfortable, and it wasn't really
+    necessary now that nokogiri can do it for us
+- Removed utf-8 normalisation from ONIX::Normaliser
+  - nokogiri also handles this really cleanly and transparently. Regardless of
+    the source file encoding, Nokogiri::Reader returns utf-8 encoded data
+- Add the release attribute to files we generate
+  - it's optional in 2.1, but mandatory in 3.0. As we start to see 3.0 files in the
+    wild it will help to have a rapid way to distinguish between them
+- Add ONIX::Reader#release - to detect the release version of files we read in
+
 v0.8.0 (31st October 2009)
 - Replace LibXML dependency with Nokogiri. Nokogiri is under active development, has
   a responsive maintainer and is significantly more stable

data/README.markdown

@@ -10,6 +10,29 @@ and writing ONIX files in your ruby applications.
 This replaces the obsolete rbook-onix gem that was spectacular in its crapness.
 Let us never speak of it again.
 
+## Feature Support
+
+This library currently only handles ONIX 2.1 files (all revisions). At some
+point I'll need to work out what to do about supporting ONIX 3.0 files. I
+suspect a separate library will be the simplest solution.
+
+ONIX::Reader only handles the reference tag versions of ONIX 2.1. Use
+ONIX::Normaliser to convert any short tag files to reference tags.
+
+ONIX::Writer only generates reference tag ONIX files.
+
+## DTD Loading
+
+To correctly handle named entities when reading an ONIX file, this gem attempts
+to load the DTD describing the ONIX format into memory. By default, this means
+each file you read will require several hundred Kb of data to be downloaded
+over the net.
+
+This is obviously not desirable in most cases. To avoid it, you need to add copies
+of the ONIX DTDs into your system XML catalog. On Debian and Ubuntu systems,
+the quickest way to do that is to build and install the package available @
+http://github.com/yob/onix-dtd
+
 ## Installation
 
     gem install onix
@@ -36,5 +59,4 @@ To be honest, I'm not really expecting any, this is a niche library.
 ## Further Reading
 
 - The source: [http://github.com/yob/onix/tree/master](http://github.com/yob/onix/tree/master)
-- Rubyforge project: [http://rubyforge.org/projects/rbook/](http://rubyforge.org/projects/rbook/)
-- The official specs [http://www.editeur.org/onix.html](http://www.editeur.org/onix.html)
+- The official specs [http://www.editeur.org/8/ONIX/](http://www.editeur.org/8/ONIX/)

data/lib/onix.rb

@@ -9,7 +9,7 @@ module ONIX
   module Version #:nodoc:
     Major = 0
     Minor = 8
-    Tiny  = 0
+    Tiny  = 1
 
     String = [Major, Minor, Tiny].join('.')
   end

data/lib/onix/lists/contributor_role.rb

@@ -0,0 +1,99 @@
+# coding: utf-8
+
+module ONIX
+  module Lists
+    # Code list 17
+    CONTRIBUTOR_ROLE = {
+      "A01" => "By (author)",
+      "A02" => "With",
+      "A03" => "Screenplay by",
+      "A04" => "Libretto by",
+      "A05" => "Lyrics by",
+      "A06" => "By (composer)",
+      "A07" => "By (artist)",
+      "A08" => "By (photographer)",
+      "A09" => "Created by",
+      "A10" => "From an idea by",
+      "A11" => "Designed by",
+      "A12" => "Illustrated by",
+      "A13" => "Photographs by",
+      "A14" => "Text by",
+      "A15" => "Preface by",
+      "A16" => "Prologue by",
+      "A17" => "Summary by",
+      "A18" => "Supplement by",
+      "A19" => "Afterword by",
+      "A20" => "Notes by",
+      "A21" => "Commentaries by",
+      "A22" => "Epilogue by",
+      "A23" => "Foreword by",
+      "A24" => "Introduction by",
+      "A25" => "Footnotes by",
+      "A26" => "Memoir by",
+      "A27" => "Experiments by",
+      "A29" => "Introduction and notes by",
+      "A30" => "Software written by",
+      "A31" => "Book and lyrics by",
+      "A32" => "Contributions by",
+      "A33" => "Appendix by",
+      "A34" => "Index by",
+      "A35" => "Drawings by",
+      "A36" => "Cover design or artwork by",
+      "A37" => "Preliminary work by",
+      "A38" => "Original author",
+      "A39" => "Maps by",
+      "A40" => "Inked or colored by",
+      "A41" => "Pop-ups by",
+      "A42" => "Continued by",
+      "A43" => "Interviewer",
+      "A44" => "Interviewee",
+      "A99" => "Other primary creator",
+      "B01" => "Edited by",
+      "B02" => "Revised by",
+      "B03" => "Retold by",
+      "B04" => "Abridged by",
+      "B05" => "Adapted by",
+      "B06" => "Translated by",
+      "B07" => "As told by",
+      "B08" => "Translated with commentary by",
+      "B09" => "Series edited by",
+      "B10" => "Edited and translated by",
+      "B11" => "Editor-in-chief",
+      "B12" => "Guest editor",
+      "B13" => "Volume editor",
+      "B14" => "Editorial board member",
+      "B15" => "Editorial coordination by",
+      "B16" => "Managing editor",
+      "B17" => "Founded by",
+      "B18" => "Prepared for publication by",
+      "B19" => "Associate editor",
+      "B20" => "Consultant editor",
+      "B21" => "General editor",
+      "B22" => "Dramatized by",
+      "B23" => "General rapporteur",
+      "B24" => "Literary editor",
+      "B25" => "Arranged by (music)",
+      "B99" => "Other adaptation by",
+      "C01" => "Compiled by",
+      "C02" => "Selected by",
+      "C99" => "Other compilation by",
+      "D01" => "Producer",
+      "D02" => "Director",
+      "D03" => "Conductor",
+      "D99" => "Other direction by",
+      "E01" => "Actor",
+      "E02" => "Dancer",
+      "E03" => "Narrator",
+      "E04" => "Commentator",
+      "E05" => "Vocal soloist",
+      "E06" => "Instrumental soloist",
+      "E07" => 'Read by',
+      "E08" => "Performed by (orchestra, band, ensemble)",
+      "E99" => "Performed by",
+      "F01" => "Filmed/photographed by",
+      "F99" => "Other recording by",
+      "Z01" => "Assisted by",
+      "Z99" => "Other",
+    }
+  end
+end

data/lib/onix/normaliser.rb

@@ -40,9 +40,6 @@ module ONIX
       raise ArgumentError, "#{oldfile} does not exist" unless File.file?(oldfile)
       raise ArgumentError, "#{newfile} already exists" if File.file?(newfile)
       raise "xsltproc app not found" unless app_available?("xsltproc")
-      raise "isutf8 app not found"   unless app_available?("isutf8")
-      raise "iconv app not found"    unless app_available?("iconv")
-      raise "sed app not found"      unless app_available?("sed")
       raise "tr app not found"       unless app_available?("tr")
 
       @oldfile = oldfile
@@ -60,21 +57,11 @@ module ONIX
         @curfile = dest
       end
 
-      # convert to utf8
-      dest = next_tempfile
-      to_utf8(@curfile, dest)
-      @curfile = dest
-
       # remove control chars
       dest = next_tempfile
       remove_control_chars(@curfile, dest)
       @curfile = dest
 
-      # remove entities
-      dest = next_tempfile
-      replace_named_entities(@curfile, dest)
-      @curfile = dest
-
       FileUtils.cp(@curfile, @newfile)
     end
 
@@ -110,41 +97,6 @@ module ONIX
       `xsltproc -o #{outpath} #{xsltpath} #{inpath}`
     end
 
-    # ensure the file is valid utf8, then make sure it's declared as such.
-    #
-    # The following behaviour is  expected:
-    #
-    #   file is valid utf8, is marked correctly
-    #     - copied untouched
-    #   file is valid utf8, is marked incorrectly or has no marked encoding
-    #     - copied and encoding mark fixed or added
-    #   file is no utf8, encoding is marked
-    #     - file is converted to utf8 and enecoding mark is updated
-    #   file is not utf8, encoding is not marked
-    #     - file is copied untouched
-    #
-    def to_utf8(src, dest)
-      inpath = File.expand_path(src)
-      outpath = File.expand_path(dest)
-
-      m, src_enc = *@head.match(/encoding=.([a-zA-Z0-9\-]+)./i)
-
-      # ensure the file is actually utf8
-      if `isutf8 #{inpath}`.strip == ""
-        if src_enc.to_s.downcase == "utf-8"
-          FileUtils.cp(inpath, outpath)
-        else
-          FileUtils.cp(inpath, outpath)
-          `sed -i 's/<?xml.*?>/<?xml version=\"1.0\" encoding=\"UTF-8\"?>/g' #{outpath}`
-        end
-      elsif src_enc
-        `iconv --from-code=#{src_enc} --to-code=UTF-8 #{inpath} > #{outpath}`
-        `sed -i 's/#{src_enc}/UTF-8/' #{outpath}`
-      else
-          FileUtils.cp(inpath, outpath)
-      end
-    end
-
     # XML files shouldn't contain low ASCII control chars. Strip them.
     #
     def remove_control_chars(src, dest)
@@ -153,35 +105,6 @@ module ONIX
       `cat #{inpath} | tr -d "\\000-\\010\\013\\014\\016-\\037" > #{outpath}`
     end
 
-    # replace all named entities in the specified file with
-    # numeric entities.
-    #
-    def replace_named_entities(src, dest)
-      inpath = File.expand_path(src)
-      outpath = File.expand_path(dest)
-
-      cmd = "sed " + entity_map.map do |named, numeric|
-        "-e 's/\\&#{named};/\\&#{numeric};/g'"
-      end.join(" ") + " #{inpath} > #{outpath}"
-      #raise cmd
-      `#{cmd}`
-    end
-
-    # return a named entity to numeric entity mapping, build by extracting
-    # data from the ONIX DTD
-    #
-    def entity_map
-      return @map if @map
-
-      path = File.dirname(__FILE__) + "/../../support/entities.txt"
-      @map = {}
-      File.read(path).split.each do |line|
-        elements = line.split(":")
-        @map[elements.first] = elements.last
-      end
-      @map
-    end
-
   end
 
 end

data/lib/onix/reader.rb

@@ -53,20 +53,21 @@ module ONIX
   class Reader
     include Enumerable
 
-    attr_reader :header , :version, :xml_lang, :xml_version
+    attr_reader :header, :release
 
     def initialize(input, product_klass = ::ONIX::Product)
       if input.kind_of?(String)
         @file   = File.open(input, "r")
-        @reader = Nokogiri::XML::Reader.from_io(@file)
+        @reader = Nokogiri::XML::Reader(@file) { |cfg| cfg.dtdload.noent }
       elsif input.kind_of?(IO)
-        @reader = Nokogiri::XML::Reader.from_io(input)
+        @reader = Nokogiri::XML::Reader(input) { |cfg| cfg.dtdload.noent }
       else
         raise ArgumentError, "Unable to read from file or IO stream"
       end
 
       @product_klass = product_klass
 
+      @release = find_release
       @header = find_header
 
       @xml_lang    ||= @reader.lang
@@ -94,6 +95,23 @@ module ONIX
 
     private
 
+    def find_release
+      2.times do
+        @reader.read
+        if @reader.node_type == 1 && @reader.name == "ONIXMessage"
+          value = @reader.attributes["release"]
+          if value
+            return BigDecimal.new(value)
+          else
+            return nil
+          end
+        elsif @reader.node_type == 14
+          return nil
+        end
+      end
+      return nil
+    end
+
     def find_header
       100.times do
         @reader.read

data/lib/onix/writer.rb

@@ -77,7 +77,7 @@ module ONIX
     def start_document
       @output.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
       @output.write("<!DOCTYPE ONIXMessage SYSTEM \"#{DOCTYPE}\">\n")
-      @output.write("<ONIXMessage>\n")
+      @output.write("<ONIXMessage release=\"2.1\">\n")
       @output.write(@header.to_xml.to_s)
       @output.write("\n")
     end

data/spec/normaliser_spec.rb

@@ -25,79 +25,6 @@ context "ONIX::Normaliser", "with a simple short tag file" do
 
 end
 
-context "ONIX::Normaliser", "with an ISO-8859-1 file" do
-
-  before(:each) do
-    @data_path = File.join(File.dirname(__FILE__),"..","data")
-    @filename  = File.join(@data_path, "iso_8859_1.xml")
-    @outfile   = @filename + ".new"
-  end
-
-  after(:each) do
-    File.unlink(@outfile) if File.file?(@outfile)
-  end
-
-  specify "should correctly convert an iso-8859-1 file to UTF-8" do
-    ONIX::Normaliser.process(@filename, @outfile)
-
-    File.file?(@outfile).should be_true
-    content = File.read(@outfile)
-
-    content.include?("ISO-8859-1").should be_false
-    content.include?("UTF-8").should be_true
-
-    `isutf8 #{File.expand_path(@outfile)}`.strip.should eql("")
-  end
-
-end
-
-context "ONIX::Normaliser", "with an file using entities" do
-
-  before(:each) do
-    @data_path = File.join(File.dirname(__FILE__),"..","data")
-    @filename  = File.join(@data_path, "entities.xml")
-    @outfile   = @filename + ".new"
-  end
-
-  after(:each) do
-    File.unlink(@outfile) if File.file?(@outfile)
-  end
-
-  specify "should correctly convert named entities to numeric entities" do
-    ONIX::Normaliser.process(@filename, @outfile)
-
-    File.file?(@outfile).should be_true
-    content = File.read(@outfile)
-
-    content.include?("–").should be_false
-    content.include?("–").should be_true
-  end
-end
-
-context "ONIX::Normaliser", "with a utf8 file that has no declared encoding" do
-
-  before(:each) do
-    @data_path = File.join(File.dirname(__FILE__),"..","data")
-    @filename  = File.join(@data_path, "no_encoding.xml")
-    @outfile   = @filename + ".new"
-  end
-
-  after(:each) do
-    File.unlink(@outfile) if File.file?(@outfile)
-  end
-
-  # this is to test for a bug where an exception was raised on files that
-  # had no declared encoding
-  specify "should add a utf-8 marker to the file" do
-    ONIX::Normaliser.process(@filename, @outfile)
-
-    File.file?(@outfile).should be_true
-    content = File.read(@outfile)
-
-    content.include?("encoding=\"UTF-8\"").should be_true
-  end
-end
-
 context "ONIX::Normaliser", "with a utf8 file that has illegal control chars" do
 
   before(:each) do
@@ -110,8 +37,6 @@ context "ONIX::Normaliser", "with a utf8 file that has illegal control chars" do
     File.unlink(@outfile) if File.file?(@outfile)
   end
 
-  # this is to test for a bug where an exception was raised on files that
-  # had no declared encoding
   specify "should remove all control chars except LF, CR and TAB" do
     ONIX::Normaliser.process(@filename, @outfile)
 

data/spec/reader_spec.rb

@@ -5,13 +5,13 @@ require File.dirname(__FILE__) + '/spec_helper.rb'
 context "ONIX::Reader" do
 
   before(:each) do
-    data_path = File.join(File.dirname(__FILE__),"..","data")
-    @file1    = File.join(data_path, "9780194351898.xml")
-    @file2    = File.join(data_path, "two_products.xml")
-    @long_file   = File.join(data_path, "Bookwise_July_2008.xml")
-    @entity_file = File.join(data_path, "entities.xml")
-    @utf_16_file = File.join(data_path, "utf_16.xml")
-    @iso_8859_1_file = File.join(data_path, "iso_8859_1.xml")
+    @data_path = File.join(File.dirname(__FILE__),"..","data")
+    @file1    = File.join(@data_path, "9780194351898.xml")
+    @file2    = File.join(@data_path, "two_products.xml")
+    @long_file   = File.join(@data_path, "Bookwise_July_2008.xml")
+    @entity_file = File.join(@data_path, "entities.xml")
+    @utf_16_file = File.join(@data_path, "utf_16.xml")
+    @iso_8859_1_file = File.join(@data_path, "iso_8859_1.xml")
   end
 
   specify "should initialize with a filename" do
@@ -26,16 +26,11 @@ context "ONIX::Reader" do
     end
   end
 
-  # This is commented out as the code that I was using to read the ONIX version from the
-  # input was causing segfaults and other stability issues
-  specify "should provide access to various XML metadata from file"
-  #do
-  #  reader = ONIX::Reader.new(@file1)
-  #  reader.encoding.should eql("utf-8")
-  #  reader.xml_lang.should eql(nil)
-  #  reader.xml_version.should eql(1.0)
-  #  reader.version.should eql([2,1,0])
-  #end
+  specify "should provide access to various XML metadata from file" do
+    filename = File.join(@data_path, "reference_with_release_attrib.xml")
+    reader = ONIX::Reader.new(filename)
+    reader.release.should eql(BigDecimal.new("2.1"))
+  end
 
   specify "should provide access to the header in an ONIX file" do
     reader = ONIX::Reader.new(@file1)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: onix
 version: !ruby/object:Gem::Version 
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors: 
 - James Healy
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-31 00:00:00 +11:00
+date: 2010-01-05 00:00:00 +11:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -73,6 +73,7 @@ files:
 - lib/onix/lists/product_availability.rb
 - lib/onix/lists/audience_code.rb
 - lib/onix/lists/language_code.rb
+- lib/onix/lists/contributor_role.rb
 - lib/onix/market_representation.rb
 - lib/onix/audience_range.rb
 - lib/onix/product_identifier.rb