atdis 0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.ruby-version

@@ -0,0 +1 @@
+ruby-1.8.7-p370

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - "1.8.7"
+  - "1.9.3"
+  - "2.0.0"
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+group :development do
+  gem "rspec"
+  gem 'guard'
+  gem 'guard-rspec'
+  gem 'growl'
+  gem 'rb-fsevent', '~> 0.9'
+  # Probably required on OS X. See https://github.com/guard/guard/wiki/Add-Readline-support-to-Ruby-on-Mac-OS-X
+  gem 'rb-readline'
+  gem 'coveralls', :require => false
+end
+
+# Specify your gem's dependencies in atdis.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,7 @@
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec' do
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^spec/.+_spec\.rb$})
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 OpenAustralia Foundation Limited
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Atdis
+
+[![Build Status](https://travis-ci.org/openaustralia/atdis.png?branch=master)](https://travis-ci.org/openaustralia/atdis) [![Coverage Status](https://coveralls.io/repos/openaustralia/atdis/badge.png?branch=master)](https://coveralls.io/r/openaustralia/atdis?branch=master) [![Code Climate](https://codeclimate.com/github/openaustralia/atdis.png)](https://codeclimate.com/github/openaustralia/atdis)
+
+A ruby interface to the application tracking data interchange specification (ATDIS) API
+
+We're developing this against version ATDIS 1.0.4.
+
+This is **highly alpha** software that probably doesn't yet do what it says on the tin. It is very much a work in progress.
+
+Source code is available on GitHub at https://github.com/openaustralia/atdis
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'atdis'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install atdis
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+# If you want to make this the default task
+task :default => :spec

data/atdis.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'atdis/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "atdis"
+  spec.version       = Atdis::VERSION
+  spec.authors       = ["Matthew Landauer"]
+  spec.email         = ["matthew@openaustraliafoundation.org.au"]
+  spec.description   = %q{A ruby interface to the application tracking data interchange specification (ATDIS) API}
+  spec.summary       = spec.description
+  spec.homepage      = "http://github.com/openaustralia/atdis"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency "multi_json", "~> 1.7"
+  spec.add_dependency "rest-client"
+  spec.add_dependency "rgeo-geojson"
+  spec.add_dependency "activemodel", "~> 3"
+end

data/lib/atdis/application.rb

@@ -0,0 +1,47 @@
+require 'multi_json'
+
+module ATDIS
+  class Application < Model
+    field_mappings :application => {
+      :info => {
+        :dat_id => [:dat_id, String],
+        :last_modified_date => [:last_modified_date, DateTime],
+        :description => [:description, String],
+        :authority => [:authority, String],
+        :lodgement_date => [:lodgement_date, DateTime],
+        :determination_date => [:determination_date, DateTime],
+        :status => [:status, String],
+        :notification_start_date => [:notification_start_date, DateTime],
+        :notification_end_date => [:notification_end_date, DateTime],
+        :officer => [:officer, String],
+        :estimated_cost => [:estimated_cost, String]
+      },
+      :reference => {
+        :more_info_url => [:more_info_url, URI],
+        :comments_url => [:comments_url, URI]
+      },
+      :location => [:location, Location],
+      :events => [:events, Event],
+      :documents => [:documents, Document],
+      :people => [:people, Person],
+      :extended => [:extended, Object]
+    }
+  
+    # Mandatory parameters
+    validates :dat_id, :last_modified_date, :description, :authority, :lodgement_date, :determination_date, :status, 
+      :more_info_url, :location, :presence_before_type_cast => true
+
+    # Other validations
+    validates :notification_start_date, :notification_end_date, :last_modified_date, :lodgement_date, :determination_date,
+      :date_time => true
+    validates :more_info_url, :http_url => true
+    validates :location, :valid => true
+
+    # TODO Validate associated like locations, events, documents, people
+    # TODO Do we need to do extra checking to ensure that events, documents and people are arrays?
+    # TODO Separate validation for L2 and L3 compliance?
+    # TODO Validate date orders. i.e. determination_date >= lodgement_date and notification_end_date >= notification_start_date
+    # TODO also last_modified_date >= lodgement_date and all the other dates. In other words we can't put a future date in. That
+    # doesn't make sense in this context. Also should check dates in things like Events (to see that they're not in the future)
+  end
+end

data/lib/atdis/document.rb

@@ -0,0 +1,12 @@
+module ATDIS
+  class Document < Model
+    field_mappings :ref => [:ref, String],
+      :title => [:title, String],
+      :document_url => [:document_url, URI]
+
+    # Mandatory parameters
+    validates :ref, :title, :document_url, :presence_before_type_cast => true
+    # Other validations
+    validates :document_url, :http_url => true
+  end
+end

data/lib/atdis/event.rb

@@ -0,0 +1,12 @@
+module ATDIS
+  class Event < Model
+    field_mappings :id => [:id, String],
+      :date => [:date, DateTime],
+      :description => [:description, String],
+      :event_type => [:event_type, String],
+      :status => [:status, String]
+
+    # Mandatory parameters
+    validates :id, :date, :description, :presence_before_type_cast => true
+  end
+end

data/lib/atdis/feed.rb

@@ -0,0 +1,21 @@
+require "rest-client"
+
+module ATDIS
+  class Feed
+    attr_reader :base_url
+
+    # base_url - the base url from which the urls for all atdis urls are made
+    # It is the concatenation of the protocol and web address as defined in section 4.2 of specification
+    # For example if the base_url is "http://www.council.nsw.gov.au" then the url for listing all the
+    # applications is "http://www.council.nsw.gov.au/atdis/1.0/applications.json"
+    def initialize(base_url)
+      @base_url = base_url.kind_of?(URI) ? base_url : URI.parse(base_url)
+    end
+
+    def applications(page = 1)
+      url = base_url + "atdis/1.0/applications.json"
+      url += "?page=#{page}" if page > 1
+      Page.read_url(url)
+    end
+  end
+end

data/lib/atdis/location.rb

@@ -0,0 +1,18 @@
+require "rgeo/geo_json"
+
+module ATDIS
+  class Location < Model
+    field_mappings :address => [:address, String],
+      :land_title_ref => {
+        :lot => [:lot, String],
+        :section => [:section, String],
+        :dpsp_id => [:dpsp_id, String]
+      },
+      :geometry => [:geometry, RGeo::GeoJSON]
+
+    # Mandatory parameters
+    validates :address, :lot, :section, :dpsp_id, :presence_before_type_cast => true
+
+    validates :geometry, :geo_json => true
+  end
+end

data/lib/atdis/model.rb

@@ -0,0 +1,190 @@
+require 'active_model'
+require 'date'
+
+module ATDIS
+  module TypeCastAttributes
+    extend ActiveSupport::Concern
+
+    included do
+      class_attribute :attribute_types
+      class_attribute :valid_fields
+    end
+
+    module ClassMethods
+      def casting_attributes(p)
+        define_attribute_methods(p.keys.map{|k| k.to_s})
+        self.attribute_types = p
+      end
+
+      def field_mappings(p)
+        a, b = translate_field_mappings(p)
+        self.valid_fields = a
+        casting_attributes(b)
+      end
+
+      private
+      
+      def translate_field_mappings(p)
+        f = {}
+        ca = {}
+        p.each do |k,v|
+          if v.kind_of?(Array)
+            f[k] = v[0]
+            ca[v.first] = v[1]
+          else
+            f2, ca2 = translate_field_mappings(v)
+            f[k] = f2
+            ca = ca.merge(ca2)
+          end
+        end
+        [f, ca]
+      end
+    end
+  end
+
+  class Model
+    include ActiveModel::Validations
+    include Validators
+    include ActiveModel::AttributeMethods
+    include TypeCastAttributes
+    attribute_method_suffix '_before_type_cast'
+    attribute_method_suffix '='
+
+    attr_reader :attributes
+    # Stores any part of the json that could not be interpreted. Usually
+    # signals an error if it isn't empty.
+    attr_accessor :json_left_overs
+    
+    validate :json_left_overs_is_empty
+
+    def json_left_overs_is_empty
+      if json_left_overs && !json_left_overs.empty?
+        # We have extra parameters that shouldn't be there
+        errors.add(:json, "Unexpected parameters in json data: #{MultiJson.dump(json_left_overs)}")
+      end
+    end
+
+    def initialize(params={})
+      @attributes, @attributes_before_type_cast = {}, {}
+      params.each do |attr, value|
+        self.send("#{attr}=", value)
+      end if params
+    end
+
+    # Does what the equivalent on Activerecord does
+    def self.attribute_names
+      attribute_types.keys.map{|k| k.to_s}
+    end
+
+    def self.interpret(*params)
+      new(map_fields(valid_fields, *params))
+    end
+
+    # Map json structure to our values
+    def self.map_fields(valid_fields, data)
+      values = {:json_left_overs => {}}
+      data.each_key do |key|
+        if valid_fields[key]
+          if valid_fields[key].kind_of?(Hash)
+            v2 = map_fields(valid_fields[key], data[key])
+            l2 = v2.delete(:json_left_overs)
+            values = values.merge(v2)
+            values[:json_left_overs][key] = l2 unless l2.empty?
+          else
+            values[valid_fields[key]] = data[key]
+          end
+        else
+          values[:json_left_overs][key] = data[key]
+        end
+      end
+      values
+    end
+
+    def self.cast(value, type)
+      # If it's already the correct type then we don't need to do anything
+      if value.kind_of?(type)
+        value
+      # Special handling for arrays. When we typecast arrays we actually typecast each member of the array
+      elsif value.kind_of?(Array)
+        value.map {|v| cast(v, type)}
+      elsif type == DateTime
+        cast_datetime(value)
+      elsif type == URI
+        cast_uri(value)
+      elsif type == String
+        cast_string(value)
+      elsif type == Fixnum
+        cast_fixnum(value)
+      elsif type == RGeo::GeoJSON
+        cast_geojson(value)
+      # Otherwise try to use Type.interpret to do the typecasting
+      elsif type.respond_to?(:interpret)
+        type.interpret(value) if value
+      else
+        raise
+      end
+    end
+
+    private
+
+    def attribute(attr)
+      @attributes[attr]
+    end
+
+    def attribute_before_type_cast(attr)
+      @attributes_before_type_cast[attr]
+    end
+
+    def attribute=(attr, value)
+      @attributes_before_type_cast[attr] = value
+      @attributes[attr] = Model.cast(value, attribute_types[attr.to_sym])
+    end
+
+    def self.cast_datetime(value)
+      # This would be much easier if we knew we only had to support Ruby 1.9 or greater because it has
+      # an implementation built in. Because for the time being we need to support Ruby 1.8 as well
+      # we'll build an implementation of parsing by hand. Ugh.
+      # Referencing http://www.w3.org/TR/NOTE-datetime
+      # In section 4.3.1 of ATDIS 1.0.4 it shows two variants of iso 8601, either the full date
+      # or the full date with hours, seconds, minutes and timezone. We'll assume that these
+      # are the two variants that are allowed.
+      if value.respond_to?(:match) && value.match(/^\d\d\d\d-\d\d-\d\d(T\d\d:\d\d:\d\d(Z|(\+|-)\d\d:\d\d))?$/)
+        DateTime.parse(value)
+      end
+    end
+
+    def self.cast_uri(value)
+      begin
+        URI.parse(value)
+      rescue URI::InvalidURIError
+        nil
+      end
+    end
+
+    def self.cast_string(value)
+      value.to_s
+    end
+
+    # This casting allows nil values
+    def self.cast_fixnum(value)
+      value.to_i if value
+    end
+
+    def self.cast_geojson(value)
+      RGeo::GeoJSON.decode(hash_symbols_to_string(value))
+    end
+
+    # Converts {:foo => {:bar => "yes"}} to {"foo" => {"bar" => "yes"}}
+    def self.hash_symbols_to_string(hash)
+      if hash.respond_to?(:each_pair)
+        result = {}
+        hash.each_pair do |key, value|
+          result[key.to_s] = hash_symbols_to_string(value)
+        end
+        result
+      else
+        hash
+      end
+    end
+  end
+end

data/lib/atdis/page.rb

@@ -0,0 +1,114 @@
+module ATDIS
+  class Page < Model
+    attr_accessor :url
+
+    field_mappings :response => [:results, Application],
+      :count => [:count, Fixnum],
+      :pagination => {
+        :previous => [:previous_page_no, Fixnum],
+        :next => [:next_page_no, Fixnum],
+        :current => [:current_page_no, Fixnum],
+        :per_page => [:no_results_per_page, Fixnum],
+        :count => [:total_no_results, Fixnum],
+        :pages => [:total_no_pages, Fixnum]
+      }
+
+    # Mandatory parameters
+    validates :results, :presence_before_type_cast => true
+    validates :results, :valid => true
+    validate :count_is_consistent, :all_pagination_is_present, :previous_page_no_is_consistent, :next_page_no_is_consistent
+    validate :current_page_no_is_consistent, :total_no_results_is_consistent
+
+    # If some of the pagination fields are present all of the required ones should be present
+    def all_pagination_is_present
+      if count || previous_page_no || next_page_no || current_page_no || no_results_per_page ||
+        total_no_results || total_no_pages
+        errors.add(:count, "should be present if pagination is being used") if count.nil?
+        errors.add(:current_page_no, "should be present if pagination is being used") if current_page_no.nil?
+        errors.add(:no_results_per_page, "should be present if pagination is being used") if no_results_per_page.nil?
+        errors.add(:total_no_results, "should be present if pagination is being used") if total_no_results.nil?
+        errors.add(:total_no_pages, "should be present if pagination is being used") if total_no_pages.nil?
+      end
+    end
+
+    def count_is_consistent
+      if count
+        errors.add(:count, "is not the same as the number of applications returned") if count != results.count        
+        errors.add(:count, "should not be larger than the number of results per page") if count > no_results_per_page
+      end
+    end
+
+    def previous_page_no_is_consistent
+      if current_page_no
+        if previous_page_no
+          if previous_page_no != current_page_no - 1
+            errors.add(:previous_page_no, "should be one less than current page number or null if first page")
+          end
+          if current_page_no == 1
+            errors.add(:previous_page_no, "should be null if on the first page")
+          end            
+        else
+          if current_page_no > 1
+            errors.add(:previous_page_no, "can't be null if not on the first page")
+          end
+        end
+      end 
+    end
+
+    def next_page_no_is_consistent
+      if next_page_no && next_page_no != current_page_no + 1
+        errors.add(:next_page_no, "should be one greater than current page number or null if last page")
+      end
+      if next_page_no.nil? && current_page_no != total_no_pages
+        errors.add(:next_page_no, "can't be null if not on the last page")
+      end
+      if next_page_no && current_page_no == total_no_pages
+        errors.add(:next_page_no, "should be null if on the last page")
+      end
+    end
+
+    def current_page_no_is_consistent
+      if current_page_no
+        errors.add(:current_page_no, "is larger than the number of pages") if current_page_no > total_no_pages        
+        errors.add(:current_page_no, "can not be less than 1") if current_page_no < 1
+      end
+    end
+
+    def total_no_results_is_consistent
+      if total_no_pages && total_no_results > total_no_pages * no_results_per_page
+        errors.add(:total_no_results, "is larger than can be retrieved through paging")
+      end
+      if total_no_pages && total_no_results <= (total_no_pages - 1) * no_results_per_page
+        errors.add(:total_no_results, "could fit into a smaller number of pages")
+      end
+    end
+
+    def self.read_url(url)
+      r = read_json(RestClient.get(url.to_s).to_str)
+      r.url = url.to_s
+      r
+    end
+
+    def self.read_json(text)
+      interpret(MultiJson.load(text, :symbolize_keys => true))
+    end
+
+    def previous_url
+      raise "Can't use previous_url when loaded with read_json" if url.nil?
+      ATDIS::SeparatedURL.merge(url, :page => previous_page_no) if previous_page_no
+    end
+
+    def next_url
+      raise "Can't use next_url when loaded with read_json" if url.nil?
+      ATDIS::SeparatedURL.merge(url, :page => next_page_no) if next_page_no
+    end
+
+    def previous
+      Page.read_url(previous_url) if previous_url
+    end
+
+    def next
+      Page.read_url(next_url) if next_url
+    end
+  end
+end

data/lib/atdis/person.rb

@@ -0,0 +1,10 @@
+module ATDIS
+  class Person < Model
+    field_mappings :name => [:name, String],
+      :role => [:role, String],
+      :contact => [:contact, String]
+
+    # Mandatory parameters
+    validates :name, :role, :presence_before_type_cast => true    
+  end
+end

data/lib/atdis/separated_url.rb

@@ -0,0 +1,37 @@
+module ATDIS
+  class SeparatedURL
+
+    def self.merge(full_url, params)
+      url, url_params = split(full_url)
+      combine(url, url_params.merge(params))
+    end
+
+    private
+    
+    def self.combine(url, url_params)
+      # Doing this jiggery pokery to ensure the params are sorted alphabetically (even on Ruby 1.8)
+      query = url_params.map{|k,v| [k.to_s, v]}.sort.map{|k,v| "#{CGI.escape(k)}=#{CGI.escape(v.to_s)}"}.join("&")
+      if url_params.empty?
+        url
+      else
+        url + "?" + query
+      end
+    end
+
+    def self.split(full_url)
+      uri = URI.parse(full_url)
+      if (uri.scheme == "http" && uri.port == 80) || (uri.scheme == "https" && uri.port == 443)
+        url = "#{uri.scheme}://#{uri.host}#{uri.path}"
+      else
+        url = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
+      end
+      if uri.query
+        url_params = Hash[*CGI::parse(uri.query).map{|k,v| [k.to_sym,v.first]}.flatten]
+      else
+        url_params = {}
+      end
+      [url, url_params]
+    end
+
+  end
+end

data/lib/atdis/validators.rb

@@ -0,0 +1,51 @@
+require 'active_model'
+
+module ATDIS
+  module Validators
+    class GeoJsonValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        raw_value = record.send("#{attribute}_before_type_cast")
+        if raw_value.present? && value.nil?
+          record.errors.add(attribute, "is not valid GeoJSON")
+        end
+      end
+    end
+
+    class DateTimeValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        raw_value = record.send("#{attribute}_before_type_cast")
+        if raw_value.present? && !value.kind_of?(DateTime)
+          record.errors.add(attribute, "is not a valid date")
+        end
+      end
+    end
+
+    class HttpUrlValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        raw_value = record.send("#{attribute}_before_type_cast")
+        if raw_value.present? && !value.kind_of?(URI::HTTP) && !value.kind_of?(URI::HTTPS) 
+          record.errors.add(attribute, "is not a valid URL")
+        end
+      end
+    end
+
+    # Take into account the value before type casting
+    class PresenceBeforeTypeCastValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        raw_value = record.send("#{attribute}_before_type_cast")
+        unless raw_value.present?
+          record.errors.add(attribute, "can't be blank")
+        end
+      end
+    end
+
+    # This attribute itself needs to be valid
+    class ValidValidator < ActiveModel::EachValidator
+      def validate_each(record, attribute, value)
+        if (value.respond_to?(:valid?) && !value.valid?) || (value && !value.respond_to?(:valid?) && !value.all?{|v| v.valid?})
+          record.errors.add(attribute, "is not valid")
+        end
+      end
+    end
+  end
+end