onix 0.6.0 → 0.7.0

data/CHANGELOG

@@ -1,3 +1,42 @@
+v0.7.0 (17th June 2009)
+- try using LibXML for reader again
+  - retrieving the ONIX version of the input file is currently disabled, as 
+    that seems to be the source of our instability
+- Various Ruby 1.9 compatability tweaks
+  - add source file coding declarations. All source files are UTF-8
+  - ONIX::Reader ensures all input data is converted to UTF-8
+  - the ROXML based objects seem to forget the encoding when they're marshalled,
+    so force string based attributes *back* to UTF-8
+
+v0.6.7 (Unreleased)
+- add some accessors to the Title composite
+
+v0.6.6 (Unreleased)
+- Forget the S on an element name
+
+v0.6.5 (Unreleased)
+- Ruby 1.9 compat
+
+v0.6.4 (Unreleased)
+- Add APAProduct#price
+
+v0.6.3(Unreleased)
+- Bump ROXML dependency to 2.5.3 to get libxml-ruby 1.1.3 compatibility
+
+v0.6.2 (Unreleased)
+- Fix a small typo in APAProduct
+
+v0.6.1 (Unreleased)
+- Stopped using LibXMLs Reader class as the basis for our reader.
+  - We were getting too many segfaults (even 1 is too many!)
+  - until we resolve it, reverted to manual string parsing
+  - This is a fairly major regression of functionality. For 99% of files
+    it won't matter, but for some corner cases it will. ie UTF-16 encoded
+    files
+  - Will also be noticeably slower
+  - Hopefully only a short term fix, until I work out what is going on with
+    libxml
+
 v0.6.0 (18th March 2009)
 - remove use of threads in ONIX::Reader
   - a producer/consumer pattern was useful in the REXML stream parsing days, but

data/lib/onix.rb

@@ -1,9 +1,11 @@
+# coding: utf-8
+
 require 'rubygems'
 require 'bigdecimal'
 require 'cgi'
 
 # ensure we load the correct gem versions
-gem 'roxml', '2.5.2'
+gem 'roxml', '2.5.3'
 gem 'andand'
 
 # and now load the actual gems
@@ -13,7 +15,7 @@ require 'andand'
 module ONIX
   module Version #:nodoc:
     Major = 0
-    Minor = 6
+    Minor = 7
     Tiny  = 0
 
     String = [Major, Minor, Tiny].join('.')
@@ -22,7 +24,9 @@ module ONIX
   class Formatters
     def self.decimal
       lambda do |val|
-        if val.kind_of?(BigDecimal)
+        if val.nil?
+          nil
+        elsif val.kind_of?(BigDecimal)
           val.to_s("F")
         else
           val.to_s
@@ -57,7 +61,11 @@ module ONIX
 end
 
 # silence some warnings from ROXML
-ROXML::SILENCE_XML_NAME_WARNING = true
+unless ROXML.const_defined?("SILENCE_XML_NAME_WARNING")
+  ROXML::SILENCE_XML_NAME_WARNING = true
+end
+
+require File.join(File.dirname(__FILE__), "onix", "common")
 
 # core files
 # - ordering is important, classes need to be defined before any

data/lib/onix/addressee_identifier.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class AddresseeIdentifier
     include ROXML
+    include ONIX::Common
 
     xml_accessor :addressee_id_type, :from => "AddresseeIDType", :as => Fixnum # should be a 2 digit num
     xml_accessor :id_type_name,      :from => "IDTypeName"

data/lib/onix/apa_product.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 module ONIX
   class APAProduct < SimpleProduct
 
@@ -408,6 +410,12 @@ module ONIX
       price_set(2, num)
     end
 
+    # just get the first price we can find, regardless of the type.
+    # useful as a backup for reading files from that don't contain a type
+    def price
+      price_get(nil).andand.price_amount
+    end
+
     # retrieve the height of the product
     #
     # If APAProduct#measurement_system is metric, these will be in mm, otherwise they
@@ -682,7 +690,7 @@ module ONIX
       if text.nil?
         text = ONIX::OtherText.new
         text.text_type_code = type
-        self.text << text
+        product.text << text
       end
 
       # store the new value

data/lib/onix/common.rb

@@ -0,0 +1,26 @@
+module ONIX
+  # methods we want to be available on all our ROXML classes
+  module Common
+
+    # After an input string has been marshalled into an object,
+    # ROXML will automatically call this method.
+    #
+    # LibXML (and therefore ROXML) appears to forget the encoding
+    # of the input data, a damn shame considering the Ruby 1.9 encoding
+    # support can be vey useful.
+    #
+    # Since we only ever pass our ROXML objects strings we know are utf-8
+    # encoded, it should be safe enough to force every string attribute
+    # back once the marshalling is done.
+    #
+    def after_parse
+      self.class.roxml_attrs.each do |attr|
+        var = instance_variable_get("@#{attr.variable_name}")
+        if var && var.respond_to?(:force_encoding)
+          var.force_encoding("utf-8")
+        end
+      end
+    end
+  end
+end
+

data/lib/onix/contributor.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Contributor
     include ROXML
+    include ONIX::Common
 
     xml_name "Contributor"
 
@@ -10,7 +13,7 @@ module ONIX
     xml_accessor :sequence_number_within_role,  :from => "SequenceNumberWithinRole", :as => Fixnum
     xml_accessor :person_name,          :from => "PersonName"
     xml_accessor :person_name_inverted, :from => "PersonNameInverted"
-    xml_accessor :titles_before_name,   :from => "TitlesBeforeName"
+    xml_accessor :titles_before_names,  :from => "TitlesBeforeNames"
     xml_accessor :names_before_key,     :from => "NamesBeforeKey"
     xml_accessor :prefix_to_key,        :from => "PrefixToKey"
     xml_accessor :key_names,            :from => "KeyNames"

data/lib/onix/header.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Header
     include ROXML
+    include ONIX::Common
 
     xml_name "Header"
 

data/lib/onix/imprint.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Imprint
     include ROXML
+    include ONIX::Common
 
     xml_name "Imprint"
 

data/lib/onix/lists/product_availability.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 module ONIX
   module Lists
     # Code list 65

data/lib/onix/lists/product_form.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 module ONIX
   module Lists
     # code list 7
@@ -62,13 +64,13 @@ module ONIX
       "PO" => "Print - Wallchart",
       "PZ" => "Print - Other",
       "VA" => "Video",
-      "VB" => "Video – VHS PAL",
-      "VC" => "Video – VHS NTSC",
-      "VD" => "Video – Betamax PAL",
-      "VE" => "Video – Betamax NTSC",
-      "VF" => "Video – disk",
-      "VG" => "Video – VHS SECAM",
-      "VH" => "Video – Betamax SECAM",
+      "VB" => "Video - VHS PAL",
+      "VC" => "Video - VHS NTSC",
+      "VD" => "Video - Betamax PAL",
+      "VE" => "Video - Betamax NTSC",
+      "VF" => "Video - disk",
+      "VG" => "Video - VHS SECAM",
+      "VH" => "Video - Betamax SECAM",
       "VZ" => "Video -  Other Format",
       "WW" => "Multiple - Mixed Media Product",
       "WX" => "Misc - Quantity Pack",

data/lib/onix/market_representation.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class MarketRepresentation
     include ROXML
+    include ONIX::Common
 
     xml_name "MarketRepresentation"
 

data/lib/onix/measure.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Measure
     include ROXML
+    include ONIX::Common
 
     xml_name "Measure"
 

data/lib/onix/media_file.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class MediaFile
     include ROXML
+    include ONIX::Common
 
     xml_name "MediaFile"
 

data/lib/onix/other_text.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class OtherText
     include ROXML
+    include ONIX::Common
 
     xml_name "OtherText"
 

data/lib/onix/price.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Price
     include ROXML
+    include ONIX::Common
 
     xml_name "Price"
 

data/lib/onix/product.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Product
     include ROXML
+    include ONIX::Common
 
     xml_name "Product"
 

data/lib/onix/product_identifier.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class ProductIdentifier
     include ROXML
+    include ONIX::Common
 
     xml_name "ProductIdentifier"
 

data/lib/onix/publisher.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Publisher
     include ROXML
+    include ONIX::Common
 
     xml_name "Publisher"
 

data/lib/onix/reader.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 require 'stringio'
 
 module ONIX
@@ -49,72 +51,95 @@ module ONIX
   # version it is.
   #
   class Reader
+    include Enumerable
 
-    attr_reader :header, :version, :xml_lang, :xml_version, :encoding, :queue
+    attr_reader :header , :version, :xml_lang, :xml_version, :encoding
 
-    # Create a new ONIX::Reader object
-    #
     def initialize(input, product_klass = ::ONIX::Product)
-      if input.kind_of? String
+      if input.kind_of?(String)
         @reader = LibXML::XML::Reader.file(input)
       elsif input.kind_of?(IO)
         @reader = LibXML::XML::Reader.io(input)
       else
-        throw "Unable to read from path or file"
+        raise ArgumentError, "Unable to read from file or IO stream"
       end
 
       @product_klass = product_klass
-      @header = nil
 
-      while @header.nil? 
-        obj = read_next
-        if obj.kind_of?(ONIX::Header)
-          @header = obj
-        end
-      end
+      @header = read_next
+
+      @xml_lang    ||= @reader.xml_lang
+      @xml_version ||= @reader.xml_version.to_f
+      @encoding    ||= encoding_const_to_name(@reader.encoding)
     end
 
     # Iterate over all the products in an ONIX file
     #
     def each(&block)
-      while !(obj = read_next).nil?
+      while obj = read_next
         yield obj
       end
     end
 
+    def close
+      @reader.close if @reader
+    end
+
     private
 
-    # Walk the ONIX file, and grab the next header or product fragment. If we
-    # encounter other useful bits of info along the way (encoding, etc) then
-    # store them for later.
+    # Walk the ONIX file, and grab the next header or product fragment.
     #
     def read_next
       while @reader.read
 
-        @xml_lang    = @reader.xml_lang         if @xml_lang.nil?
-        @xml_version = @reader.xml_version.to_f if @xml_version.nil?
-        @encoding    = encoding_const_to_name(@reader.encoding) if @encoding.nil?
-
         if @reader.node_type == LibXML::XML::Reader::TYPE_DOCUMENT_TYPE
-          uri = @reader.expand.to_s
-          m, major, minor, rev = *uri.match(/.+(\d)\.(\d)\/(\d*).*/)
-          @version = [major.to_i, minor.to_i, rev.to_i]
-        elsif @reader.name == "Header" && @reader.node_type == LibXML::XML::Reader::TYPE_ELEMENT
-          str = @reader.read_outer_xml
-          @reader.next_sibling
-          return ONIX::Header.from_xml(str)
-        elsif @reader.name == "Product" && @reader.node_type == LibXML::XML::Reader::TYPE_ELEMENT
-          str = @reader.read_outer_xml
-          @reader.next_sibling
-          return @product_klass.from_xml(str)
+          # TODO restore ONIX version extraction. The following expand()
+          #      call is triggering unpredictable behaviour in libxml-ruby
+          #      1.1.3 with libxml2 2.7.3. Sometimes segfaults, othertimes
+          #      cryptic errors about the input file being truncated or
+          #      incomplete
+          #uri = @reader.expand.to_s.dup
+          #m, major, minor, rev = *uri.match(/.+(\d)\.(\d)\/(\d*).*/)
+          #@version = [major.to_i, minor.to_i, rev.to_i]
+        elsif @reader.node_type == LibXML::XML::Reader::TYPE_ELEMENT
+          if @reader.name == "Header"
+            str = normalise_string_encoding(@reader.read_outer_xml.to_s.dup)
+            if str.size == 0
+              return ONIX::Header.new
+            else
+              return ONIX::Header.from_xml(str)
+            end
+          elsif @reader.name == "Product"
+            str = normalise_string_encoding(@reader.read_outer_xml.to_s.dup)
+            if str.size == 0
+              return @product_klass.new
+            else
+              return @product_klass.from_xml(str)
+            end
+          end
         end
       end
+
+      return nil
+    rescue LibXML::XML::Error => e
       return nil
     end
 
+    # XML::Reader seems to transparently convert all input data to utf-8, howver
+    # on Ruby 1.9 it fails to correctly set the encoding on the strings.
+    #
+    def normalise_string_encoding(str)
+      if RUBY_VERSION >= "1.9"
+        return str.dup.force_encoding("utf-8")
+      else
+        str
+      end
+    end
+
     # simple mapping of encoding constants to a string
     #
     def encoding_const_to_name(const)
+      return nil if const.nil?
       case const
       when LibXML::XML::Encoding::UTF_8
         "utf-8"
@@ -154,6 +179,8 @@ module ONIX
         "euc-jp"
       when LibXML::XML::Encoding::ASCII
         "ascii"
+      else
+        nil
       end
     end
   end

data/lib/onix/sales_restriction.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class SalesRestriction
     include ROXML
+    include ONIX::Common
 
     xml_name "SalesRestriction"
 

data/lib/onix/sender_identifier.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class SenderIdentifier
     include ROXML
+    include ONIX::Common
 
     xml_name "SenderIdentifier"
 

data/lib/onix/simple_product.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 require 'forwardable'
 
 module ONIX

data/lib/onix/stock.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Stock
     include ROXML
+    include ONIX::Common
 
     xml_name "Stock"
 

data/lib/onix/subject.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Subject
     include ROXML
+    include ONIX::Common
 
     xml_name "Subject"
 

data/lib/onix/supply_detail.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class SupplyDetail
     include ROXML
+    include ONIX::Common
 
     xml_name "SupplyDetail"
 

data/lib/onix/title.rb

@@ -1,11 +1,16 @@
+# coding: utf-8
+
 module ONIX
   class Title
     include ROXML 
+    include ONIX::Common
 
     xml_name "Title"
 
     xml_accessor :title_type, :from => "TitleType", :as => Fixnum, :to_xml => ONIX::Formatters.two_digit
     xml_accessor :title_text, :from => "TitleText"
+    xml_accessor :title_prefix, :from => "TitlePrefix"
+    xml_accessor :title_without_prefix, :from => "TitleWithoutPrefix"
     xml_accessor :subtitle,   :from => "Subtitle"
 
   end

data/lib/onix/website.rb

@@ -1,6 +1,9 @@
+# coding: utf-8
+
 module ONIX
   class Website
     include ROXML
+    include ONIX::Common
 
     xml_name "Website"
 

data/lib/onix/writer.rb

@@ -1,3 +1,5 @@
+# coding: utf-8
+
 module ONIX
   # The primary way to write a new ONIX file.
   #

data/spec/reader_spec.rb

@@ -10,7 +10,10 @@ context "ONIX::Reader" do
     data_path = File.join(File.dirname(__FILE__),"..","data")
     @file1    = File.join(data_path, "9780194351898.xml")
     @file2    = File.join(data_path, "two_products.xml")
+    @long_file   = File.join(data_path, "Bookwise_July_2008.xml")
     @entity_file = File.join(data_path, "entities.xml")
+    @utf_16_file = File.join(data_path, "utf_16.xml")
+    @iso_8859_1_file = File.join(data_path, "iso_8859_1.xml")
   end
 
   specify "should initialize with a filename" do
@@ -25,13 +28,16 @@ context "ONIX::Reader" do
     end
   end
 
-  specify "should provide access to various XML metadata from file" do
-    reader = ONIX::Reader.new(@file1)
-    reader.encoding.should eql("utf-8")
-    reader.xml_lang.should eql(nil)
-    reader.xml_version.should eql(1.0)
-    reader.version.should eql([2,1,0])
-  end
+  # This is commented out as the code that I was using to read the ONIX version from the
+  # input was causing segfaults and other stability issues
+  specify "should provide access to various XML metadata from file"
+  #do
+  #  reader = ONIX::Reader.new(@file1)
+  #  reader.encoding.should eql("utf-8")
+  #  reader.xml_lang.should eql(nil)
+  #  reader.xml_version.should eql(1.0)
+  #  reader.version.should eql([2,1,0])
+  #end
 
   specify "should provide access to the header in an ONIX file" do
     reader = ONIX::Reader.new(@file1)
@@ -62,18 +68,64 @@ context "ONIX::Reader" do
   end
 
   # libxml can handle the 3 standard entities fine (& < and ^gt;) but
-  # barfs when it encounters others. In theory other entityies are defined in the
+  # barfs when it encounters others. In theory other entities are defined in the
   # ONIX DTD, but I can't work out how to get libxml to recognise them
   specify "should correctly parse a file that has an entity in it" do
     reader = ONIX::Reader.new(@entity_file)
-    
+
     products = []
     reader.each do |product|
       products << product
     end
 
     products.size.should eql(1)
-    products.first.record_reference.should eql("9780732287573")
+    products.first.titles.size.should eql(1)
     products.first.titles.first.title_text.should eql("High Noon\342\200\223in Nimbin")
+    products.first.record_reference.should eql("9780732287573")
+  end
+
+  # for some reason I'm getting segfaults when I read a file with more than 7 records
+  specify "should correctly parse a file with more than 7 records in in" do
+    reader = ONIX::Reader.new(@long_file)
+    counter = 0
+    reader.each do |product|
+      counter += 1
+    end
+
+    counter.should eql(346)
+  end
+
+  specify "should transparently convert a iso-8859-1 file to utf-8" do
+    reader = ONIX::Reader.new(@iso_8859_1_file)
+    product = nil
+    reader.each do |p|
+      product = p
+    end
+
+    # ROXML appears to munge the string encodings
+    if RUBY_VERSION >= "1.9"
+      utf8 = Encoding.find("utf-8")
+      product.contributors[0].person_name_inverted.encoding.should eql(utf8)
+    end
+
+    product.contributors[0].person_name_inverted.should eql("Küng, Hans")
+
+  end
+
+  specify "should transparently convert a utf-16 file to utf-8" do
+    reader = ONIX::Reader.new(@utf_16_file)
+    product = nil
+    reader.each do |p|
+      product = p
+    end
+
+    # ROXML appears to munge the string encodings
+    if RUBY_VERSION >= "1.9"
+      utf8 = Encoding.find("utf-8")
+      product.contributors[0].person_name_inverted.encoding.should eql(utf8)
+    end
+
+    product.contributors[0].person_name_inverted.should eql("Küng, Hans")
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: onix
 version: !ruby/object:Gem::Version 
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors: 
 - James Healy
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-18 00:00:00 +11:00
+date: 2009-06-17 00:00:00 +10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirements: 
     - - "="
       - !ruby/object:Gem::Version 
-        version: 2.5.2
+        version: 2.5.3
     version: 
 - !ruby/object:Gem::Dependency 
   name: libxml-ruby
@@ -30,7 +30,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.9.8
+        version: 1.1.3
     version: 
 - !ruby/object:Gem::Dependency 
   name: andand
@@ -76,12 +76,15 @@ files:
 - lib/onix/market_representation.rb
 - lib/onix/measure.rb
 - lib/onix/writer.rb
+- lib/onix/common.rb
 - lib/onix.rb
 - README.markdown
 - TODO
 - CHANGELOG
 has_rdoc: true
 homepage: http://github.com/yob/onix/tree/master
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --title
@@ -104,9 +107,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: rbook
-rubygems_version: 1.3.1
+rubygems_version: 1.3.4
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: A convient mapping between ruby objects and the ONIX XML specification
 test_files: 
 - spec/header_spec.rb
@@ -116,6 +119,7 @@ test_files:
 - spec/contributor_spec.rb
 - spec/imprint_spec.rb
 - spec/market_representation_spec.rb
+- spec/website_spec.rb
 - spec/measure_spec.rb
 - spec/media_file_spec.rb
 - spec/other_text_spec.rb
@@ -128,4 +132,3 @@ test_files:
 - spec/subject_spec.rb
 - spec/supply_detail_spec.rb
 - spec/title_spec.rb
-- spec/website_spec.rb