amee 2.5.1 → 2.6.0

data/CHANGELOG

@@ -1,5 +1,14 @@
 = Changelog
 
+== 2.6.0
+ * Add auto-retry for certain HTTP failures.
+ * Add options to Connection.new for timeout and retry.
+ * Add options to amee.yml for timeout and retry.
+ * More useful exception reporting in event of failures.
+ 
+== 2.5.1
+ * Fix for bug in empty tag parsing
+
 == 2.5.0
  * Use ActiveSupport::Cache for caching
  * Add ability to specify cache and debug options in amee.yml for Rails projects.
@@ -17,10 +26,10 @@
 
 == 2.2.0
 
-* Add SSL support. SSL connections are now used by default.
-* Log4r support
-* Support for reading ItemValueDefinitions
+ * Add SSL support. SSL connections are now used by default.
+ * Log4r support
+ * Support for reading ItemValueDefinitions
     * Include accessors from other objects
-* Internal improvements including
+ * Internal improvements including
     * Improved paging support
     * Tidier code for collections

data/README

@@ -107,6 +107,17 @@ To enable caching, pass ':cache => :memory_store' to AMEE::Connection.new, or ad
 same caching configuration as your Rails app, you can add 'cache: rails' to 
 amee.yml instead. Caching is disabled by default.
 
+== RETRY / TIMEOUT SUPPORT
+
+The AMEE::Connection object can now automatically retry if a request fails due to 
+network problems or connection failures. To enable this feature, pass ':retries => 3'
+to AMEE::Connection.new, or add 'retries: 3' to your amee.yml if using Rails. You can
+change the number of retry attempts, 3 is just used as an example above.
+
+The Connection object also allows a timeout to be set for requests. By default this is
+set to 60 seconds, but if you want to provide a different value (30 seconds for 
+instance), pass ':timeout => 30' to AMEE::Connection.new, or 'timeout: 30' in amee.yml.
+
 == UPGRADING TO VERSION > 2
 
 There are a few changes to the API exposed by this gem for version 2. The main

data/lib/amee/collection.rb

@@ -12,10 +12,14 @@ module AMEE
       each_page do
         parse_page
       end
-    rescue => err
-      #raise AMEE::BadData.new("Couldn't load #{self.class.name}.\n#{response} due to #{err}")
-      raise AMEE::BadData.new("Couldn't load #{self.class.name}.\n#{response}")
+    rescue JSONParseError, XMLParseError
+      @connection.expire(collectionpath)
+      raise AMEE::BadData.new("Couldn't load #{self.class.name}.\n#{@response}")
+    rescue AMEE::BadData
+      @connection.expire(collectionpath)
+      raise
     end
+
     def parse_page
       if json
         jsoncollector.each do |p|
@@ -26,7 +30,7 @@ module AMEE
         end
       else
         doc.xpath(xmlcollectorpath.split('/')[1...-1].join('/')).first or
-          raise AMEE::BadData.new("Couldn't load #{self.class.name}. parp\n#{response}")
+          raise AMEE::BadData.new("Couldn't load #{self.class.name}. \n#{@response}")
         doc.xpath(xmlcollectorpath).each do |p|
           obj=klass.new(parse_xml(p))
           obj.connection = connection
@@ -39,12 +43,23 @@ module AMEE
 
     def fetch
       @options.merge! @pager.options if @pager
-      @response= @connection.get(collectionpath, @options).body
-      if @response.is_json?
-        @json = true
-        @doc = JSON.parse(@response)
-      else
-        @doc = load_xml_doc(@response)
+      retries = [1] * connection.retries
+      begin
+        @response= @connection.get(collectionpath, @options).body
+        if @response.is_json?
+          @json = true
+          @doc = JSON.parse(@response)
+        else
+          @doc = load_xml_doc(@response)
+        end
+      rescue JSON::ParserError, Nokogiri::XML::SyntaxError
+        @connection.expire(collectionpath)
+        if delay = retries.shift
+          sleep delay
+          retry
+        else
+          raise
+        end
       end
     end
 

data/lib/amee/connection.rb

@@ -18,6 +18,7 @@ module AMEE
       @auth_token = nil
       @format = options[:format] || (defined?(JSON) ? :json : :xml)
       @amee_source = options[:amee_source]
+      @retries = options[:retries] || 0
       if !valid?
        raise "You must supply connection details - server, username and password are all required!"
       end
@@ -52,7 +53,7 @@ module AMEE
           @http.verify_depth = 5
         end
       end
-      @http.read_timeout = 60
+      self.timeout = options[:timeout] || 60
       @http.set_debug_output($stdout) if options[:enable_debug]
       @debug = options[:enable_debug]
     end
@@ -61,13 +62,14 @@ module AMEE
     attr_reader :server
     attr_reader :username
     attr_reader :password
+    attr_reader :retries
 
     def timeout
       @http.read_timeout
     end
 
     def timeout=(t)
-      @http.read_timeout = t
+      @http.open_timeout = @http.read_timeout = t
     end
 
     def version
@@ -217,20 +219,22 @@ module AMEE
           if response.body.include? "would have resulted in a duplicate resource being created"
             raise AMEE::DuplicateResource.new("The specified resource already exists. This is most often caused by creating an item that overlaps another in time.\nRequest: #{request.method} #{request.path}\n#{request.body}\Response: #{response.body}")
           else
-            raise AMEE::UnknownError.new("An error occurred while talking to AMEE: HTTP response code #{response.code}.\nRequest: #{request.method} #{request.path}\n#{request.body}\Response: #{response.body}")
+            raise AMEE::BadRequest.new("Bad request. This is probably due to malformed input data.\nRequest: #{request.method} #{request.path}\n#{request.body}\Response: #{response.body}")
           end
-        else
-          raise AMEE::UnknownError.new("An error occurred while talking to AMEE: HTTP response code #{response.code}.\nRequest: #{request.method} #{request.path}\n#{request.body}\Response: #{response.body}")
       end
+      raise AMEE::UnknownError.new("An error occurred while talking to AMEE: HTTP response code #{response.code}.\nRequest: #{request.method} #{request.path}\n#{request.body}\Response: #{response.body}")
     end
 
     def do_request(request, format = @format)
       # Open HTTP connection
       @http.start
-      # Do request
       begin
+        # Set auth token in cookie (and header just in case someone's stripping cookies)
+        request['Cookie'] = "authToken=#{@auth_token}"
+        request['authToken'] = @auth_token
+        # Do request
         timethen=Time.now
-        response = send_request(request, format)
+        response = send_request(@http, request, format)
         Logger.log.debug("Requesting #{request.class} at #{request.path} with #{request.body} in format #{format}, taking #{(Time.now-timethen)*1000} miliseconds")
       end while !response_ok?(response, request)
       # Return response
@@ -242,16 +246,28 @@ module AMEE
       @http.finish if @http.started?
     end
 
-    def send_request(request, format = @format)
-      # Set auth token in cookie (and header just in case someone's stripping cookies)
-      request['Cookie'] = "authToken=#{@auth_token}"
-      request['authToken'] = @auth_token
+    def send_request(connection, request, format = @format)
       # Set accept header
       request['Accept'] = content_type(format)
       # Set AMEE source header if set
       request['X-AMEE-Source'] = @amee_source if @amee_source
       # Do the business
-      response = @http.request(request)
+      retries = [1] * @retries
+      begin
+        response = connection.request(request)
+        # 500-series errors fail early
+        if ['502', '503', '504'].include? response.code
+          raise AMEE::ConnectionFailed.new("A connection error occurred while talking to AMEE: HTTP response code #{response.code}.\nRequest: #{request.method} #{request.path}")
+        end
+      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, AMEE::ConnectionFailed => e
+        if delay = retries.shift
+          sleep delay
+          retry
+        else
+          raise
+        end
+      end
       # Done
       response
     end

data/lib/amee/data_category.rb

@@ -25,37 +25,39 @@ module AMEE
       def self.from_json(json)
         # Parse json
         doc = JSON.parse(json)
-        data = {}
-        data[:uid] = doc['dataCategory']['uid']
-        data[:created] = DateTime.parse(doc['dataCategory']['created'])
-        data[:modified] = DateTime.parse(doc['dataCategory']['modified'])
-        data[:name] = doc['dataCategory']['name']
-        data[:path] = doc['path']
-        data[:children] = []
-        data[:pager] = AMEE::Pager.from_json(doc['children']['pager'])
-        itemdef=doc['dataCategory']['itemDefinition']
-        data[:itemdef] = itemdef ? itemdef['uid'] : nil
-        doc['children']['dataCategories'].each do |child|
-          category_data = {}
-          category_data[:name] = child['name']
-          category_data[:path] = child['path']
-          category_data[:uid] = child['uid']
-          data[:children] << category_data
-        end
-        data[:items] = []
-        if doc['children']['dataItems']['rows']
-          doc['children']['dataItems']['rows'].each do |item|
-            item_data = {}
-            item.each_pair do |key, value|
-              item_data[key.to_sym] = value
+        begin
+          data = {}
+          data[:uid] = doc['dataCategory']['uid']
+          data[:created] = DateTime.parse(doc['dataCategory']['created'])
+          data[:modified] = DateTime.parse(doc['dataCategory']['modified'])
+          data[:name] = doc['dataCategory']['name']
+          data[:path] = doc['path']
+          data[:children] = []
+          data[:pager] = AMEE::Pager.from_json(doc['children']['pager'])
+          itemdef=doc['dataCategory']['itemDefinition']
+          data[:itemdef] = itemdef ? itemdef['uid'] : nil
+          doc['children']['dataCategories'].each do |child|
+            category_data = {}
+            category_data[:name] = child['name']
+            category_data[:path] = child['path']
+            category_data[:uid] = child['uid']
+            data[:children] << category_data
+          end
+          data[:items] = []
+          if doc['children']['dataItems']['rows']
+            doc['children']['dataItems']['rows'].each do |item|
+              item_data = {}
+              item.each_pair do |key, value|
+                item_data[key.to_sym] = value
+              end
+              data[:items] << item_data
             end
-            data[:items] << item_data
           end
+          # Create object
+          Category.new(data)
+        rescue AMEE::JSONParseError
+          raise AMEE::BadData.new("Couldn't load DataCategory from JSON data. Check that your URL is correct.\n#{json}")
         end
-        # Create object
-        Category.new(data)
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataCategory from JSON data. Check that your URL is correct.\n#{json}")
       end
 
       def self.xmlpathpreamble
@@ -64,39 +66,41 @@ module AMEE
       def self.from_xml(xml)
         # Parse XML
         @doc = doc= REXML::Document.new(xml)
-        data = {}
-        data[:uid] = x '@uid'
-        data[:created] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataCategoryResource/DataCategory/@created").to_s)
-        data[:modified] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataCategoryResource/DataCategory/@modified").to_s)
-        data[:name] = (REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/Name') || REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/name')).text
-        data[:path] = (REXML::XPath.first(doc, '/Resources/DataCategoryResource/Path') || REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/path')).text || ""
-        data[:pager] = AMEE::Pager.from_xml(REXML::XPath.first(doc, '//Pager'))
-        itemdefattrib=REXML::XPath.first(doc, '/Resources/DataCategoryResource//ItemDefinition/@uid')
-        data[:itemdef] = itemdefattrib ? itemdefattrib.to_s : nil
-        data[:children] = []
-        REXML::XPath.each(doc, '/Resources/DataCategoryResource//Children/DataCategories/DataCategory') do |child|
-          category_data = {}
-          category_data[:name] = (child.elements['Name'] || child.elements['name']).text
-          category_data[:path] = (child.elements['Path'] || child.elements['path']).text
-          category_data[:uid] = child.attributes['uid'].to_s
-          data[:children] << category_data
-        end
-        data[:items] = []
-        REXML::XPath.each(doc, '/Resources/DataCategoryResource//Children/DataItems/DataItem') do |item|
-          item_data = {}
-          item_data[:uid] = item.attributes['uid'].to_s
-          item.elements.each do |element|
-            item_data[element.name.to_sym] = element.text
+        begin
+          data = {}
+          data[:uid] = x '@uid'
+          data[:created] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataCategoryResource/DataCategory/@created").to_s)
+          data[:modified] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataCategoryResource/DataCategory/@modified").to_s)
+          data[:name] = (REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/Name') || REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/name')).text
+          data[:path] = (REXML::XPath.first(doc, '/Resources/DataCategoryResource/Path') || REXML::XPath.first(doc, '/Resources/DataCategoryResource/DataCategory/path')).text || ""
+          data[:pager] = AMEE::Pager.from_xml(REXML::XPath.first(doc, '//Pager'))
+          itemdefattrib=REXML::XPath.first(doc, '/Resources/DataCategoryResource//ItemDefinition/@uid')
+          data[:itemdef] = itemdefattrib ? itemdefattrib.to_s : nil
+          data[:children] = []
+          REXML::XPath.each(doc, '/Resources/DataCategoryResource//Children/DataCategories/DataCategory') do |child|
+            category_data = {}
+            category_data[:name] = (child.elements['Name'] || child.elements['name']).text
+            category_data[:path] = (child.elements['Path'] || child.elements['path']).text
+            category_data[:uid] = child.attributes['uid'].to_s
+            data[:children] << category_data
           end
-          if item_data[:path].nil?
-            item_data[:path] = item_data[:uid]
+          data[:items] = []
+          REXML::XPath.each(doc, '/Resources/DataCategoryResource//Children/DataItems/DataItem') do |item|
+            item_data = {}
+            item_data[:uid] = item.attributes['uid'].to_s
+            item.elements.each do |element|
+              item_data[element.name.to_sym] = element.text
+            end
+            if item_data[:path].nil?
+              item_data[:path] = item_data[:uid]
+            end
+            data[:items] << item_data
           end
-          data[:items] << item_data
+          # Create object
+          Category.new(data)
+        rescue AMEE::XMLParseError
+          raise AMEE::BadData.new("Couldn't load DataCategory from XML data. Check that your URL is correct.\n#{xml}")
         end
-        # Create object
-        Category.new(data)
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataCategory from XML data. Check that your URL is correct.\n#{xml}")
       end
 
       def self.get(connection, path, orig_options = {})
@@ -112,20 +116,12 @@ module AMEE
         end
 
         # Load data from path
-        response = connection.get(path, options).body
+        cat = get_and_parse(connection, path, options)
 
-        # Parse data from response
-        if response.is_json?
-          cat = Category.from_json(response)
-        else
-          cat = Category.from_xml(response)
-        end
         # Store connection in object for future use
         cat.connection = connection
         # Done
         return cat
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataCategory. Check that your URL is correct.\n#{response}")
       end
 
       def self.root(connection)

data/lib/amee/data_item.rb

@@ -34,102 +34,107 @@ module AMEE
       def self.from_json(json)
         # Read JSON
         doc = JSON.parse(json)
-        data = {}
-        data[:uid] = doc['dataItem']['uid']
-        data[:created] = DateTime.parse(doc['dataItem']['created'])
-        data[:modified] = DateTime.parse(doc['dataItem']['modified'])
-        data[:name] = doc['dataItem']['name']
-        data[:path] = doc['path']
-        data[:label] = doc['dataItem']['label']
-        data[:item_definition] = doc['dataItem']['itemDefinition']['uid']
-        data[:category_uid] = doc['dataItem']['dataCategory']['uid']
-        # Read v2 total
-        data[:total_amount] = doc['amount']['value'] rescue nil
-        data[:total_amount_unit] = doc['amount']['unit'] rescue nil
-        # Read v1 total
-        if data[:total_amount].nil?
-          data[:total_amount] = doc['amountPerMonth'] rescue nil
-          data[:total_amount_unit] = "kg/month"
-        end
-        # Get values
-        data[:values] = []
-        doc['dataItem']['itemValues'].each do |value|
-          value_data = {}
-          value_data[:name] = value['name']
-          value_data[:path] = value['path']
-          value_data[:value] = value['value']
-          value_data[:drill] = value['itemValueDefinition']['drillDown'] rescue nil
-          value_data[:uid] = value['uid']
-          data[:values] << value_data
-        end
-        # Get choices
-        data[:choices] = []
-        doc['userValueChoices']['choices'].each do |choice|
-          choice_data = {}
-          choice_data[:name] = choice['name']
-          choice_data[:value] = choice['value']
-          data[:choices] << choice_data
+        begin
+          data = {}
+          data[:uid] = doc['dataItem']['uid']
+          data[:created] = DateTime.parse(doc['dataItem']['created'])
+          data[:modified] = DateTime.parse(doc['dataItem']['modified'])
+          data[:name] = doc['dataItem']['name']
+          data[:path] = doc['path']
+          data[:label] = doc['dataItem']['label']
+          data[:item_definition] = doc['dataItem']['itemDefinition']['uid']
+          data[:category_uid] = doc['dataItem']['dataCategory']['uid']
+          # Read v2 total
+          data[:total_amount] = doc['amount']['value'] rescue nil
+          data[:total_amount_unit] = doc['amount']['unit'] rescue nil
+          # Read v1 total
+          if data[:total_amount].nil?
+            data[:total_amount] = doc['amountPerMonth'] rescue nil
+            data[:total_amount_unit] = "kg/month"
+          end
+          # Get values
+          data[:values] = []
+          doc['dataItem']['itemValues'].each do |value|
+            value_data = {}
+            value_data[:name] = value['name']
+            value_data[:path] = value['path']
+            value_data[:value] = value['value']
+            value_data[:drill] = value['itemValueDefinition']['drillDown'] rescue nil
+            value_data[:uid] = value['uid']
+            data[:values] << value_data
+          end
+          # Get choices
+          data[:choices] = []
+          doc['userValueChoices']['choices'].each do |choice|
+            choice_data = {}
+            choice_data[:name] = choice['name']
+            choice_data[:value] = choice['value']
+            data[:choices] << choice_data
+          end
+          data[:start_date] = DateTime.parse(doc['dataItem']['startDate']) rescue nil
+          # Create object
+          Item.new(data)
+        rescue
+          raise AMEE::BadData.new("Couldn't load DataItem from JSON. Check that your URL is correct.\n#{json}")
         end
-        data[:start_date] = DateTime.parse(doc['dataItem']['startDate']) rescue nil
-        # Create object
-        Item.new(data)
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataItem from JSON. Check that your URL is correct.\n#{json}")
       end
 
       def self.from_xml(xml)
         # Parse data from response into hash
         doc = REXML::Document.new(xml)
-        data = {}
-        data[:uid] = REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@uid").to_s
-        data[:created] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@created").to_s)
-        data[:modified] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@modified").to_s)
-        data[:name] = (REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/Name') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/name')).text
-        data[:path] = (REXML::XPath.first(doc, '/Resources/DataItemResource/Path') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/path')).text
-        data[:label] = (REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/Label') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/label')).text
-        data[:item_definition] = REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/ItemDefinition/@uid').to_s
-        data[:category_uid] = REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/DataCategory/@uid').to_s
-        # Read v2 total
-        data[:total_amount] = REXML::XPath.first(doc, '/Resources/DataItemResource/Amount').text.to_f rescue nil
-        data[:total_amount_unit] = REXML::XPath.first(doc, '/Resources/DataItemResource/Amount/@unit').to_s rescue nil
-        # Read v1 total
-        if data[:total_amount].nil?
-          data[:total_amount] = REXML::XPath.first(doc, '/Resources/DataItemResource/AmountPerMonth').text.to_f rescue nil
-          data[:total_amount_unit] = "kg/month"
-        end
-        # Get values
-        data[:values] = []
-        REXML::XPath.each(doc, '/Resources/DataItemResource/DataItem/ItemValues/ItemValue') do |value|
-          value_data = {}
-          value_data[:name] = (value.elements['Name'] || value.elements['name']).text
-          value_data[:path] = (value.elements['Path'] || value.elements['path']).text
-          value_data[:value] = (value.elements['Value'] || value.elements['value']).text
-          value_data[:drill] = value.elements['ItemValueDefinition'].elements['DrillDown'].text == "false" ? false : true rescue nil
-          value_data[:uid] = value.attributes['uid'].to_s
-          data[:values] << value_data
-        end
-        # Get choices
-        data[:choices] = []
-        REXML::XPath.each(doc, '/Resources/DataItemResource/Choices/Choices/Choice') do |choice|
-          choice_data = {}
-          choice_data[:name] = (choice.elements['Name']).text
-          choice_data[:value] = (choice.elements['Value']).text || ""
-          data[:choices] << choice_data
+        begin
+          data = {}
+          data[:uid] = REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@uid").to_s
+          data[:created] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@created").to_s)
+          data[:modified] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/@modified").to_s)
+          data[:name] = (REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/Name') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/name')).text
+          data[:path] = (REXML::XPath.first(doc, '/Resources/DataItemResource/Path') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/path')).text
+          data[:label] = (REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/Label') || REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/label')).text
+          data[:item_definition] = REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/ItemDefinition/@uid').to_s
+          data[:category_uid] = REXML::XPath.first(doc, '/Resources/DataItemResource/DataItem/DataCategory/@uid').to_s
+          # Read v2 total
+          data[:total_amount] = REXML::XPath.first(doc, '/Resources/DataItemResource/Amount').text.to_f rescue nil
+          data[:total_amount_unit] = REXML::XPath.first(doc, '/Resources/DataItemResource/Amount/@unit').to_s rescue nil
+          # Read v1 total
+          if data[:total_amount].nil?
+            data[:total_amount] = REXML::XPath.first(doc, '/Resources/DataItemResource/AmountPerMonth').text.to_f rescue nil
+            data[:total_amount_unit] = "kg/month"
+          end
+          # Get values
+          data[:values] = []
+          REXML::XPath.each(doc, '/Resources/DataItemResource/DataItem/ItemValues/ItemValue') do |value|
+            value_data = {}
+            value_data[:name] = (value.elements['Name'] || value.elements['name']).text
+            value_data[:path] = (value.elements['Path'] || value.elements['path']).text
+            value_data[:value] = (value.elements['Value'] || value.elements['value']).text
+            value_data[:drill] = value.elements['ItemValueDefinition'].elements['DrillDown'].text == "false" ? false : true rescue nil
+            value_data[:uid] = value.attributes['uid'].to_s
+            data[:values] << value_data
+          end
+          # Get choices
+          data[:choices] = []
+          REXML::XPath.each(doc, '/Resources/DataItemResource/Choices/Choices/Choice') do |choice|
+            choice_data = {}
+            choice_data[:name] = (choice.elements['Name']).text
+            choice_data[:value] = (choice.elements['Value']).text || ""
+            data[:choices] << choice_data
+          end
+          data[:start_date] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/StartDate").text) rescue nil
+          # Create object
+          Item.new(data)
+        rescue
+          raise AMEE::BadData.new("Couldn't load DataItem from XML. Check that your URL is correct.\n#{xml}")
         end
-        data[:start_date] = DateTime.parse(REXML::XPath.first(doc, "/Resources/DataItemResource/DataItem/StartDate").text) rescue nil
-        # Create object
-        Item.new(data)
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataItem from XML. Check that your URL is correct.\n#{xml}")
       end
 
 
       def self.get(connection, path, options = {})
         # Load data from path
-        response = connection.get(path, options).body
-        AMEE::Data::Item.parse(connection, response)
-      rescue
-        raise AMEE::BadData.new("Couldn't load DataItem. Check that your URL is correct.\n#{response}")
+        item= get_and_parse(connection, path, options)
+        # Store connection in object for future use
+        item.connection = connection
+        # Done
+        return item
       end
 
       def self.parse(connection, response)

data/lib/amee/exceptions.rb

@@ -18,6 +18,9 @@ module AMEE
     end
   end
   
+  class BadRequest < Exception
+  end
+
   class AuthFailed < Exception  
   end
 
@@ -42,4 +45,28 @@ module AMEE
   class Deprecated < Exception
   end
   
-end
+  # These are used to classify exceptions that can occur during parsing
+  
+  module XMLParseError
+  end
+  
+  module JSONParseError
+  end
+  
+end
+
+# Set up possible XML parse errors
+[
+  ArgumentError, # thrown by Date.parse
+].each do |m| 
+  m.send(:include, AMEE::XMLParseError)
+end
+
+# Set up possible JSON parse errors
+[
+  ArgumentError, # thrown by Date.parse,
+  NoMethodError, # missing elements in JSON, thrown by nil[]
+].each do |m| 
+  m.send(:include, AMEE::JSONParseError)
+end
+

data/lib/amee/object.rb

@@ -21,6 +21,37 @@ module AMEE
     def expire_cache
       @connection.expire_matching(full_path+'.*')
     end
-
+    
+    # A nice shared get/parse handler that handles retry on parse errors
+    def self.get_and_parse(connection, path, options)
+      # Note that we don't check the number of times retry has been done lower down
+      # and count separately instead.
+      # Worst case could be retries squared given the right pattern of failure, but
+      # that seems unlikely. Would need, for instance, repeated 503 503 200->parse_fail
+      retries = [1] * connection.retries
+      begin
+        # Load data from path
+        response = connection.get(path, options).body
+        # Parse data from response
+        if response.is_json?
+          from_json(response)
+        else
+          from_xml(response)
+        end
+      rescue JSON::ParserError, Nokogiri::XML::SyntaxError, REXML::ParseException => e
+        # Invalid JSON or XML received, try the GET again in case it got cut off mid-stream
+        connection.expire(path)
+        if delay = retries.shift
+          sleep delay
+          retry
+        else
+          raise
+        end
+      rescue AMEE::BadData
+        connection.expire(path)
+        raise
+      end
+    end
+    
   end
 end

data/lib/amee/parse_helper.rb

@@ -34,10 +34,10 @@ module ParseHelper
     ''
   end
   def load_xml_doc(xml)
-    doc = Nokogiri.XML(xml)
+    doc = Nokogiri.XML(xml) { |config| config.strict }
     doc.remove_namespaces!
     doc
   end
   private :x
-
+  
 end

data/lib/amee/rails.rb

@@ -12,6 +12,12 @@ module AMEE
           if $AMEE_CONFIG['ssl'] == false
             options.merge! :ssl => false
           end
+          if $AMEE_CONFIG['retries']
+            options.merge! :retries => $AMEE_CONFIG['retries'].to_i
+          end
+          if $AMEE_CONFIG['timeout']
+            options.merge! :timeout => $AMEE_CONFIG['timeout'].to_i
+          end
           if $AMEE_CONFIG['cache'] == 'rails'
             # Pass in the rails cache store
             options[:cache_store] = ActionController::Base.cache_store

data/lib/amee/version.rb

@@ -2,8 +2,8 @@ module AMEE
   
   module VERSION #:nodoc:
     MAJOR = 2
-    MINOR = 5
-    TINY  = 1
+    MINOR = 6
+    TINY  = 0
     STRING = [MAJOR, MINOR, TINY].join('.')
   end
   

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: amee
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 23
   prerelease: false
   segments: 
   - 2
-  - 5
-  - 1
-  version: 2.5.1
+  - 6
+  - 0
+  version: 2.6.0
 platform: ruby
 authors: 
 - James Smith
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-08 00:00:00 +00:00
+date: 2011-02-16 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency