atdis 0.1 → 0.2

data/README.md

@@ -1,10 +1,10 @@
 # Atdis
 
-[![Build Status](https://travis-ci.org/openaustralia/atdis.png?branch=master)](https://travis-ci.org/openaustralia/atdis) [![Coverage Status](https://coveralls.io/repos/openaustralia/atdis/badge.png?branch=master)](https://coveralls.io/r/openaustralia/atdis?branch=master) [![Code Climate](https://codeclimate.com/github/openaustralia/atdis.png)](https://codeclimate.com/github/openaustralia/atdis)
+[![Build Status](https://travis-ci.org/openaustralia/atdis.png?branch=master)](https://travis-ci.org/openaustralia/atdis) [![Coverage Status](https://coveralls.io/repos/openaustralia/atdis/badge.png?branch=master)](https://coveralls.io/r/openaustralia/atdis?branch=master) [![Code Climate](https://codeclimate.com/github/openaustralia/atdis.png)](https://codeclimate.com/github/openaustralia/atdis) [![Gem Version](https://badge.fury.io/rb/atdis.png)](http://badge.fury.io/rb/atdis)
 
 A ruby interface to the application tracking data interchange specification (ATDIS) API
 
-We're developing this against version ATDIS 1.0.4.
+We're developing this against version ATDIS 1.0.7.
 
 This is **highly alpha** software that probably doesn't yet do what it says on the tin. It is very much a work in progress.
 

data/lib/atdis/application.rb

@@ -2,41 +2,72 @@ require 'multi_json'
 
 module ATDIS
   class Application < Model
-    field_mappings :application => {
-      :info => {
-        :dat_id => [:dat_id, String],
-        :last_modified_date => [:last_modified_date, DateTime],
-        :description => [:description, String],
-        :authority => [:authority, String],
-        :lodgement_date => [:lodgement_date, DateTime],
-        :determination_date => [:determination_date, DateTime],
-        :status => [:status, String],
-        :notification_start_date => [:notification_start_date, DateTime],
-        :notification_end_date => [:notification_end_date, DateTime],
-        :officer => [:officer, String],
-        :estimated_cost => [:estimated_cost, String]
-      },
-      :reference => {
-        :more_info_url => [:more_info_url, URI],
-        :comments_url => [:comments_url, URI]
-      },
-      :location => [:location, Location],
-      :events => [:events, Event],
-      :documents => [:documents, Document],
-      :people => [:people, Person],
-      :extended => [:extended, Object]
-    }
-  
+    # TODO When we remove support for Ruby 1.8 we can convert field_mappings back to a hash
+    # which is much more readable
+    set_field_mappings [
+      [:application, [
+        [:info, [
+          [:dat_id,                   [:dat_id, String, {:level => 1}]],
+          [:last_modified_date,       [:last_modified_date, DateTime, {:level => 1}]],
+          [:description,              [:description, String, {:level => 1}]],
+          [:authority,                [:authority, String, {:level => 1}]],
+          [:lodgement_date,           [:lodgement_date, DateTime, {:level => 1}]],
+          [:determination_date,       [:determination_date, DateTime, {:level => 1}]],
+          [:status,                   [:status, String, {:level => 1}]],
+          [:notification_start_date,  [:notification_start_date, DateTime, {:level => 1}]],
+          [:notification_end_date,    [:notification_end_date, DateTime, {:level => 1}]],
+          [:officer,                  [:officer, String, {:level => 1}]],
+          [:estimated_cost,           [:estimated_cost, String, {:level => 1}]]
+        ]],
+        [:reference, [
+          [:more_info_url,            [:more_info_url, URI, {:level => 1}]],
+          [:comments_url,             [:comments_url, URI, {:level => 1}]]
+        ]],
+        [:location,                   [:location, Location, {:level => 1}]],
+        [:events,                     [:events, Event, {:level => 1}]],
+        [:documents,                  [:documents, Document, {:level => 1}]],
+        [:people,                     [:people, Person, {:level => 2}]],
+        [:extended,                   [:extended, Object, {:level => 3}]]
+      ]]
+    ]
+
     # Mandatory parameters
-    validates :dat_id, :last_modified_date, :description, :authority, :lodgement_date, :determination_date, :status, 
-      :more_info_url, :location, :presence_before_type_cast => true
+    validates :dat_id, :last_modified_date, :description, :authority, :lodgement_date, :determination_date, :status,
+      :presence_before_type_cast => {:spec_section => "4.3.1"}
+    validates :more_info_url, :presence_before_type_cast => {:spec_section => "4.3.2"}
+    validates :location, :presence_before_type_cast => {:spec_section => "4.3.3"}
+    validates :events, :presence_before_type_cast => {:spec_section => "4.3.4"}
+    validates :documents, :presence_before_type_cast => {:spec_section => "4.3.5"}
 
     # Other validations
-    validates :notification_start_date, :notification_end_date, :last_modified_date, :lodgement_date, :determination_date,
-      :date_time => true
-    validates :more_info_url, :http_url => true
-    validates :location, :valid => true
+    validates :last_modified_date, :lodgement_date, :date_time => {:spec_section => "4.3.8"}
+    validates :determination_date, :notification_start_date, :notification_end_date, :date_time_or_none => {:spec_section => "4.3.1"}
+    validates :more_info_url, :http_url => {:spec_section => "4.3.2"}
+    validates :location, :events, :documents, :people, :valid => true
+    validates :events, :documents, :array => {:spec_section => "4.3.4"}
+    # TODO people should be an array if it's included
+
+    validate :notification_dates_consistent!
+
+    def notification_dates_consistent!
+      if notification_start_date_before_type_cast == "none" && notification_end_date_before_type_cast != "none"
+        errors.add(:notification_start_date, ErrorMessage["can't be none unless notification_end_date is none as well", "4.3.1"])
+      end
+      if notification_start_date_before_type_cast != "none" && notification_end_date_before_type_cast == "none"
+        errors.add(:notification_end_date, ErrorMessage["can't be none unless notification_start_date is none as well", "4.3.1"])
+      end
+      if notification_start_date_before_type_cast && notification_end_date_before_type_cast.blank?
+        errors.add(:notification_end_date, ErrorMessage["can not be blank if notification_start_date is set", "4.3.1"])
+      end
+      if notification_start_date_before_type_cast.blank? && notification_end_date_before_type_cast
+        errors.add(:notification_start_date, ErrorMessage["can not be blank if notification_end_date is set", "4.3.1"])
+      end
+      if notification_start_date && notification_end_date && notification_start_date > notification_end_date
+        errors.add(:notification_end_date, ErrorMessage["can not be earlier than notification_start_date", "4.3.1"])
+      end
+    end
 
+    # TODO Validate contents of estimated_cost
     # TODO Validate associated like locations, events, documents, people
     # TODO Do we need to do extra checking to ensure that events, documents and people are arrays?
     # TODO Separate validation for L2 and L3 compliance?

data/lib/atdis/document.rb

@@ -1,12 +1,14 @@
 module ATDIS
   class Document < Model
-    field_mappings :ref => [:ref, String],
-      :title => [:title, String],
-      :document_url => [:document_url, URI]
+    set_field_mappings [
+      [:ref, [:ref, String, {:level => 1}]],
+      [:title, [:title, String, {:level => 1}]],
+      [:document_url, [:document_url, URI, {:level => 1}]]
+    ]
 
     # Mandatory parameters
-    validates :ref, :title, :document_url, :presence_before_type_cast => true
+    validates :ref, :title, :document_url, :presence_before_type_cast => {:spec_section => "4.3.5"}
     # Other validations
-    validates :document_url, :http_url => true
+    validates :document_url, :http_url => {:spec_section => "4.3.5"}
   end
 end

data/lib/atdis/event.rb

@@ -1,12 +1,17 @@
 module ATDIS
   class Event < Model
-    field_mappings :id => [:id, String],
-      :date => [:date, DateTime],
-      :description => [:description, String],
-      :event_type => [:event_type, String],
-      :status => [:status, String]
-
+    set_field_mappings [
+      [:id, [:id, String, {:level => 1}]],
+      [:date, [:date, DateTime, {:level => 1}]],
+      [:description, [:description, String, {:level => 1}]],
+      [:event_type, [:event_type, String, {:level => 1}]],
+      [:status, [:status, String, {:level => 1}]]
+    ]
+    
     # Mandatory parameters
-    validates :id, :date, :description, :presence_before_type_cast => true
+    validates :id, :date, :description, :presence_before_type_cast => {:spec_section => "4.3.4"}
   end
+
+  # TODO Check that :id is unique within an authority
+  
 end

data/lib/atdis/location.rb

@@ -2,17 +2,20 @@ require "rgeo/geo_json"
 
 module ATDIS
   class Location < Model
-    field_mappings :address => [:address, String],
-      :land_title_ref => {
-        :lot => [:lot, String],
-        :section => [:section, String],
-        :dpsp_id => [:dpsp_id, String]
-      },
-      :geometry => [:geometry, RGeo::GeoJSON]
-
+    set_field_mappings [
+      [:address, [:address, String, {:level => 1}]],
+      [:land_title_ref, [
+        [:lot, [:lot, String, {:level => 1}]],
+        [:section, [:section, String, {:none_is_nil => true, :level => 1}]],
+        [:dpsp_id, [:dpsp_id, String, {:level => 1}]]
+      ]],
+      [:geometry, [:geometry, RGeo::GeoJSON, {:level => 1}]]
+    ]
     # Mandatory parameters
-    validates :address, :lot, :section, :dpsp_id, :presence_before_type_cast => true
+    validates :address, :lot, :section, :dpsp_id, :presence_before_type_cast => {:spec_section => "4.3.3"}
+
+    validates :geometry, :geo_json => {:spec_section => "4.3.3"}
 
-    validates :geometry, :geo_json => true
+    # TODO: Provide warning if dpsp_id doesn't start with "DP" or "SP"
   end
 end

data/lib/atdis/model.rb

@@ -7,30 +7,39 @@ module ATDIS
 
     included do
       class_attribute :attribute_types
-      class_attribute :valid_fields
+      class_attribute :field_mappings
     end
 
     module ClassMethods
+      # of the form {:section=>[String, {:none_is_nil=>true}], :address=>[String]}
       def casting_attributes(p)
         define_attribute_methods(p.keys.map{|k| k.to_s})
         self.attribute_types = p
       end
 
-      def field_mappings(p)
+      def set_field_mappings(p)
         a, b = translate_field_mappings(p)
-        self.valid_fields = a
+        # field_mappings is of the form {:pagination=>{:previous=>:previous_page_no, :pages=>:total_no_pages}}
+        self.field_mappings = a
         casting_attributes(b)
       end
 
       private
+
+      def leaf_array?(v)
+        if !v.kind_of?(Array)
+          return false
+        end
+        v.all?{|a| !a.kind_of?(Array)}
+      end
       
       def translate_field_mappings(p)
-        f = {}
-        ca = {}
+        f = ActiveSupport::OrderedHash.new
+        ca = ActiveSupport::OrderedHash.new
         p.each do |k,v|
-          if v.kind_of?(Array)
+          if leaf_array?(v)
             f[k] = v[0]
-            ca[v.first] = v[1]
+            ca[v.first] = v[1..-1]
           else
             f2, ca2 = translate_field_mappings(v)
             f[k] = f2
@@ -42,6 +51,17 @@ module ATDIS
     end
   end
 
+  ErrorMessage = Struct.new :message, :spec_section do
+    def empty?
+      message.empty?
+    end
+
+    # Make this behave pretty much like a string
+    def to_s
+      message
+    end
+  end
+
   class Model
     include ActiveModel::Validations
     include Validators
@@ -50,17 +70,128 @@ module ATDIS
     attribute_method_suffix '_before_type_cast'
     attribute_method_suffix '='
 
-    attr_reader :attributes
+    attr_reader :attributes, :attributes_before_type_cast
     # Stores any part of the json that could not be interpreted. Usually
     # signals an error if it isn't empty.
-    attr_accessor :json_left_overs
+    attr_accessor :json_left_overs, :json_load_error
     
     validate :json_left_overs_is_empty
 
+    def self.level_attribute_names(level)
+      attribute_types.find_all{|k,v| (v[1] || {})[:level] == level }.map{|k,v| k.to_s}
+    end
+
+    def json_attribute(a, new_value, mappings = field_mappings)
+      mappings.each do |attribute, v|
+        if v == a
+          return {attribute => new_value}
+        end
+        if v.kind_of?(Hash)
+          r = json_attribute(a, new_value, v)
+          if r
+            return {attribute => r}
+          end
+        end
+      end
+      nil
+    end
+
+    def self.map_field(key, data, mappings)
+      mappings.each do |k, v|
+        if v == key
+          return data[k]
+        elsif v.kind_of?(Hash) && data.has_key?(k)
+          r = map_field(key, data[k], mappings[k])
+          if r
+            return r
+          end
+        end
+      end
+      nil
+    end
+
+    def self.unused_data(data, mappings = field_mappings)
+      json_left_overs = {}
+      data.each_key do |key|
+        if mappings[key]
+          if mappings[key].kind_of?(Hash)
+            l2 = unused_data(data[key], mappings[key])
+            json_left_overs[key] = l2 unless l2.empty?
+          end
+        else
+          json_left_overs[key] = data[key]
+        end
+      end
+      json_left_overs
+    end
+
+    def self.attribute_names_from_mappings(mappings)
+      result = []
+      mappings.each do |k, v|
+        if v.kind_of?(Hash)
+          result += attribute_names_from_mappings(v)
+        else
+          result << v
+        end
+      end
+      result
+    end
+
+    # Map json structure to our values
+    def self.map_fields(data, mappings = field_mappings)
+      values = {}
+      attribute_names_from_mappings(mappings).each do |attribute|
+        values[attribute] = map_field(attribute, data, mappings)
+      end
+      values
+    end
+
+    def json_errors
+      r = []
+      errors.messages.each do |attribute, e|
+        value = attributes[attribute.to_s]
+        if (value.respond_to?(:valid?) && !value.valid?)
+          r += value.json_errors.map{|a, b| [json_attribute(attribute, a), b]}
+        elsif (value && !value.respond_to?(:valid?) && value.respond_to?(:all?) && !value.all?{|v| v.valid?})
+          f = value.find{|v| !v.valid?}
+          r += f.json_errors.map{|a, b| [json_attribute(attribute, a), b]}
+        else
+          r << [json_attribute(attribute, attributes_before_type_cast[attribute.to_s]), e]
+        end
+      end
+      r
+    end
+
+    # TODO This is doing a similar stepping down into the children that json_errors is doing. Would be nice
+    # to extract the commond code to make this less horrible and arbitrary
+    def level_used_in_children?(level)
+      attributes.each_value do |a|
+        if a.respond_to?(:level_used?) && a.level_used?(level)
+          return true
+        elsif a.kind_of?(Array) && a.any?{|b| b.level_used?(level)}
+          return true
+        end
+      end
+      false
+    end
+
+    # Have we tried to use this attribute?
+    def used_attribute?(a)
+      !attributes_before_type_cast[a].nil?
+    end
+
+    def level_used_locally?(level)
+      self.class.level_attribute_names(level).any?{|a| used_attribute?(a)}
+    end
+
+    def level_used?(level)
+      level_used_locally?(level) || level_used_in_children?(level)
+    end
+
     def json_left_overs_is_empty
       if json_left_overs && !json_left_overs.empty?
         # We have extra parameters that shouldn't be there
-        errors.add(:json, "Unexpected parameters in json data: #{MultiJson.dump(json_left_overs)}")
+        errors.add(:json, ErrorMessage["Unexpected parameters in json data: #{MultiJson.dump(json_left_overs)}", "4"])
       end
     end
 
@@ -77,36 +208,18 @@ module ATDIS
     end
 
     def self.interpret(*params)
-      new(map_fields(valid_fields, *params))
-    end
-
-    # Map json structure to our values
-    def self.map_fields(valid_fields, data)
-      values = {:json_left_overs => {}}
-      data.each_key do |key|
-        if valid_fields[key]
-          if valid_fields[key].kind_of?(Hash)
-            v2 = map_fields(valid_fields[key], data[key])
-            l2 = v2.delete(:json_left_overs)
-            values = values.merge(v2)
-            values[:json_left_overs][key] = l2 unless l2.empty?
-          else
-            values[valid_fields[key]] = data[key]
-          end
-        else
-          values[:json_left_overs][key] = data[key]
-        end
-      end
-      values
+      new(map_fields(*params).merge(:json_left_overs => unused_data(*params)))
     end
 
-    def self.cast(value, type)
+    def self.cast(value, type, options = {})
+      if options[:none_is_nil] && value == "none"
+        nil
       # If it's already the correct type then we don't need to do anything
-      if value.kind_of?(type)
+      elsif value.kind_of?(type)
         value
       # Special handling for arrays. When we typecast arrays we actually typecast each member of the array
       elsif value.kind_of?(Array)
-        value.map {|v| cast(v, type)}
+        value.map {|v| cast(v, type, options)}
       elsif type == DateTime
         cast_datetime(value)
       elsif type == URI
@@ -137,7 +250,7 @@ module ATDIS
 
     def attribute=(attr, value)
       @attributes_before_type_cast[attr] = value
-      @attributes[attr] = Model.cast(value, attribute_types[attr.to_sym])
+      @attributes[attr] = Model.cast(value, attribute_types[attr.to_sym][0], attribute_types[attr.to_sym][1] || {})
     end
 
     def self.cast_datetime(value)

data/lib/atdis/page.rb

@@ -2,39 +2,48 @@ module ATDIS
   class Page < Model
     attr_accessor :url
 
-    field_mappings :response => [:results, Application],
-      :count => [:count, Fixnum],
-      :pagination => {
-        :previous => [:previous_page_no, Fixnum],
-        :next => [:next_page_no, Fixnum],
-        :current => [:current_page_no, Fixnum],
-        :per_page => [:no_results_per_page, Fixnum],
-        :count => [:total_no_results, Fixnum],
-        :pages => [:total_no_pages, Fixnum]
-      }
+    set_field_mappings [
+      [:response, [:results, Application, {:level => 1}]],
+      [:count, [:count, Fixnum, {:level => 2}]],
+      [:pagination, [
+        [:previous, [:previous_page_no, Fixnum, {:level => 2}]],
+        [:next, [:next_page_no, Fixnum, {:level => 2}]],
+        [:current, [:current_page_no, Fixnum, {:level => 2}]],
+        [:per_page, [:no_results_per_page, Fixnum, {:level => 2}]],
+        [:count, [:total_no_results, Fixnum, {:level => 2}]],
+        [:pages, [:total_no_pages, Fixnum, {:level => 2}]]
+      ]]
+    ]
 
     # Mandatory parameters
-    validates :results, :presence_before_type_cast => true
+    validates :results, :presence_before_type_cast => {:spec_section => "4.3"}
     validates :results, :valid => true
     validate :count_is_consistent, :all_pagination_is_present, :previous_page_no_is_consistent, :next_page_no_is_consistent
     validate :current_page_no_is_consistent, :total_no_results_is_consistent
+    validate :json_loaded_correctly!
+
+    def json_loaded_correctly!
+      if json_load_error
+        errors.add(:json, ErrorMessage["Invalid JSON: #{json_load_error}", nil])
+      end
+    end
 
     # If some of the pagination fields are present all of the required ones should be present
     def all_pagination_is_present
       if count || previous_page_no || next_page_no || current_page_no || no_results_per_page ||
         total_no_results || total_no_pages
-        errors.add(:count, "should be present if pagination is being used") if count.nil?
-        errors.add(:current_page_no, "should be present if pagination is being used") if current_page_no.nil?
-        errors.add(:no_results_per_page, "should be present if pagination is being used") if no_results_per_page.nil?
-        errors.add(:total_no_results, "should be present if pagination is being used") if total_no_results.nil?
-        errors.add(:total_no_pages, "should be present if pagination is being used") if total_no_pages.nil?
+        errors.add(:count, ErrorMessage["should be present if pagination is being used", "6.5"]) if count.nil?
+        errors.add(:current_page_no, ErrorMessage["should be present if pagination is being used", "6.5"]) if current_page_no.nil?
+        errors.add(:no_results_per_page, ErrorMessage["should be present if pagination is being used", "6.5"]) if no_results_per_page.nil?
+        errors.add(:total_no_results, ErrorMessage["should be present if pagination is being used", "6.5"]) if total_no_results.nil?
+        errors.add(:total_no_pages, ErrorMessage["should be present if pagination is being used", "6.5"]) if total_no_pages.nil?
       end
     end
 
     def count_is_consistent
       if count
-        errors.add(:count, "is not the same as the number of applications returned") if count != results.count        
-        errors.add(:count, "should not be larger than the number of results per page") if count > no_results_per_page
+        errors.add(:count, ErrorMessage["is not the same as the number of applications returned", "6.5"]) if count != results.count        
+        errors.add(:count, ErrorMessage["should not be larger than the number of results per page", "6.5"]) if count > no_results_per_page
       end
     end
 
@@ -42,14 +51,14 @@ module ATDIS
       if current_page_no
         if previous_page_no
           if previous_page_no != current_page_no - 1
-            errors.add(:previous_page_no, "should be one less than current page number or null if first page")
+            errors.add(:previous_page_no, ErrorMessage["should be one less than current page number or null if first page", "6.5"])
           end
           if current_page_no == 1
-            errors.add(:previous_page_no, "should be null if on the first page")
+            errors.add(:previous_page_no, ErrorMessage["should be null if on the first page", "6.5"])
           end            
         else
           if current_page_no > 1
-            errors.add(:previous_page_no, "can't be null if not on the first page")
+            errors.add(:previous_page_no, ErrorMessage["can't be null if not on the first page", "6.5"])
           end
         end
       end 
@@ -57,29 +66,29 @@ module ATDIS
 
     def next_page_no_is_consistent
       if next_page_no && next_page_no != current_page_no + 1
-        errors.add(:next_page_no, "should be one greater than current page number or null if last page")
+        errors.add(:next_page_no, ErrorMessage["should be one greater than current page number or null if last page", "6.5"])
       end
       if next_page_no.nil? && current_page_no != total_no_pages
-        errors.add(:next_page_no, "can't be null if not on the last page")
+        errors.add(:next_page_no, ErrorMessage["can't be null if not on the last page", "6.5"])
       end
       if next_page_no && current_page_no == total_no_pages
-        errors.add(:next_page_no, "should be null if on the last page")
+        errors.add(:next_page_no, ErrorMessage["should be null if on the last page", "6.5"])
       end
     end
 
     def current_page_no_is_consistent
       if current_page_no
-        errors.add(:current_page_no, "is larger than the number of pages") if current_page_no > total_no_pages        
-        errors.add(:current_page_no, "can not be less than 1") if current_page_no < 1
+        errors.add(:current_page_no, ErrorMessage["is larger than the number of pages", "6.5"]) if current_page_no > total_no_pages        
+        errors.add(:current_page_no, ErrorMessage["can not be less than 1", "6.5"]) if current_page_no < 1
       end
     end
 
     def total_no_results_is_consistent
       if total_no_pages && total_no_results > total_no_pages * no_results_per_page
-        errors.add(:total_no_results, "is larger than can be retrieved through paging")
+        errors.add(:total_no_results, ErrorMessage["is larger than can be retrieved through paging", "6.5"])
       end
       if total_no_pages && total_no_results <= (total_no_pages - 1) * no_results_per_page
-        errors.add(:total_no_results, "could fit into a smaller number of pages")
+        errors.add(:total_no_results, ErrorMessage["could fit into a smaller number of pages", "6.5"])
       end
     end
 
@@ -90,7 +99,14 @@ module ATDIS
     end
 
     def self.read_json(text)
-      interpret(MultiJson.load(text, :symbolize_keys => true))
+      begin
+        data = MultiJson.load(text, :symbolize_keys => true)
+        interpret(data)
+      rescue MultiJson::LoadError => e
+        a = interpret({:response => []})
+        a.json_load_error = e.to_s
+        a
+      end
     end
 
     def previous_url