rets-hack 0.11

data/lib/rets/metadata/root.rb

@@ -0,0 +1,155 @@
+module Rets
+  module Metadata
+    METADATA_TYPES = %w(SYSTEM RESOURCE CLASS TABLE LOOKUP LOOKUP_TYPE OBJECT)
+
+    # It's useful when dealing with the Rets standard to represent their
+    # relatively flat namespace of interweived components as a Tree. With
+    # a collection of resources at the top, and their various, classes,
+    # tables, lookups, and lookup types underneath.
+    #
+    # It looks something like ...
+    #
+    #    Resource
+    #     |
+    #    Class
+    #     |
+    #     `-- Table
+    #     |
+    #     `-- Lookups
+    #           |
+    #           `-- LookupType
+    #
+    # For our purposes it was helpful to denormalize some of the more deeply
+    # nested branches. In particular by relating Lookups to LookupTypes, and
+    # Tables to lookups with can simplify this diagram.
+    #
+    #
+    #    Resource
+    #     |
+    #    Class
+    #     |
+    #     `-- Table
+    #          |
+    #          `-- Lookups
+    #
+    # By associating Tables and lookups when we parse this structure. It allows
+    # us to seemlessly map Lookup values to their Long or Short value forms.
+    class Root
+      # Metadata_types is the low level parsed representation of the raw xml
+      # sources. Just one level up, they contain Containers, consisting of
+      # SystemContainers or RowContainers
+      attr_writer :metadata_types
+
+      # the tree is the high level represenation of the metadata heiarchy
+      # it begins with root. Stored as a list of Metadata::Resources
+      attr_writer :tree
+
+      # Sources are the raw xml documents fetched for each metadata type
+      # they are stored as a hash with the type names as their keys
+      # and the raw xml as the values
+      attr_reader :sources
+
+      # Metadata can be unmarshalled from cache. @logger is not set during that process, constructor is not called.
+      # Client code must set it after unmarshalling.
+      attr_reader :logger
+
+      # fetcher is a proc that inverts control to the client to retrieve metadata
+      # types
+      def initialize(logger, sources)
+        @logger = logger
+        @tree = nil
+        @metadata_types = nil # TODO think up a better name ... containers?
+        @sources = sources
+      end
+
+      def marshal_dump
+        sources
+      end
+
+      def version
+        metadata_types[:system].first.version
+      end
+
+      def date
+        metadata_types[:system].first.date
+      end
+
+      # Wether there exists a more up to date version of the metadata to fetch
+      # is dependant on either a timestamp indicating when the most recent
+      # version was published, or a version number. These values may or may
+      # not exist on any given rets server.
+      def current?(current_timestamp, current_version)
+        if !current_version.to_s.empty? && !version.to_s.empty?
+          current_version == version
+        else
+          current_timestamp ? current_timestamp == date : true
+        end
+      end
+
+      def build_tree
+        tree = Hash.new { |h, k| h.key?(k.downcase) ? h[k.downcase] : nil }
+
+        resource_containers = metadata_types[:resource]
+
+        resource_containers.each do |resource_container|
+          resource_container.rows.each do |resource_fragment|
+            resource = Resource.build(resource_fragment, metadata_types, @logger)
+            #some mlses list resource types without an associated data, throw those away
+            tree[resource.id.downcase] = resource if resource
+          end
+        end
+
+        tree
+      end
+
+      def tree
+        @tree ||= build_tree
+      end
+
+      # Print the tree to a file
+      #
+      # [out] The file to print to.  Defaults to $stdout.
+      def print_tree(out = $stdout)
+        tree.each do |name, value|
+          value.print_tree(out)
+        end
+      end
+
+      def metadata_types
+        return @metadata_types if @metadata_types
+
+        h = {}
+
+        sources.each do |name, source|
+          h[name.downcase.to_sym] = build_containers(Nokogiri.parse(source))
+        end
+
+        @metadata_types = h
+      end
+
+      # Returns an array of container classes that represents
+      # the metadata stored in the document provided.
+      def build_containers(doc)
+        # find all tags that match /RETS/METADATA-*
+        fragments = doc.xpath("/RETS/*[starts-with(name(), 'METADATA-')]")
+
+        fragments.map { |fragment| build_container(fragment) }
+      end
+
+      def build_container(fragment)
+        tag  = fragment.name             # METADATA-RESOURCE
+        type = tag.sub(/^METADATA-/, "") # RESOURCE
+
+        class_name = type.capitalize.gsub(/_(\w)/) { $1.upcase }
+        container_name = "#{class_name}Container"
+
+        if ::RUBY_VERSION < '1.9'
+          container_class = Containers.const_defined?(container_name) ? Containers.const_get(container_name) : Containers::Container
+        else
+          container_class = Containers.const_defined?(container_name, true) ? Containers.const_get(container_name, true) : Containers::Container
+        end
+        container_class.new(fragment)
+      end
+    end
+  end
+end

data/lib/rets/metadata/table.rb

@@ -0,0 +1,33 @@
+module Rets
+  module Metadata
+    class Table
+      attr_reader :table_fragment, :resource_id, :name, :long_name
+
+      def initialize(table_fragment, resource_id)
+        @table_fragment = table_fragment
+        @resource_id = resource_id
+
+        @name = table_fragment["SystemName"]
+        @long_name = table_fragment["LongName"]
+      end
+
+      # Print the tree to a file
+      #
+      # [out] The file to print to.  Defaults to $stdout.
+      def print_tree(out = $stdout)
+        out.puts "### Table: #{name}"
+        out.puts "      Resource: #{resource_id}"
+        out.puts "      ShortName: #{ table_fragment["ShortName"] }"
+        out.puts "      LongName: #{ long_name }"
+        out.puts "      StandardName: #{ table_fragment["StandardName"] }"
+        out.puts "      Units: #{ table_fragment["Units"] }"
+        out.puts "      Searchable: #{ table_fragment["Searchable"] }"
+        out.puts "      Required: #{table_fragment['Required']}"
+      end
+
+      def resolve(value)
+        value.to_s.strip
+      end
+    end
+  end
+end

data/lib/rets/metadata/table_factory.rb

@@ -0,0 +1,19 @@
+module Rets
+  module Metadata
+    class TableFactory
+
+      def self.build(table_fragment, resource_id, lookup_types)
+        if table_fragment["LookupName"].empty?
+          Table.new(table_fragment, resource_id)
+        else
+          if table_fragment["Interpretation"] == "LookupMulti"
+            MultiLookupTable.build(table_fragment, resource_id, lookup_types)
+          else
+            LookupTable.build(table_fragment, resource_id, lookup_types)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rets/metadata/yaml_serializer.rb

@@ -0,0 +1,27 @@
+require 'yaml'
+
+module Rets
+  module Metadata
+
+    # Serialize/Deserialize metadata using Marshal.
+    class YamlSerializer
+
+      # Serialize to a file.  The library reserves the right to change
+      # the type or contents of o, so don't depend on it being
+      # anything in particular.
+      def save(file, o)
+        YAML.dump(o, file)
+      end
+
+      # Deserialize from a file.  If the metadata cannot be
+      # deserialized, return nil.
+      def load(file)
+        YAML.load(file)
+      rescue Psych::SyntaxError
+        nil
+      end
+
+    end
+    
+  end
+end

data/lib/rets/metadata.rb

@@ -0,0 +1,18 @@
+require 'rets/metadata/caching'
+require 'rets/metadata/null_cache'
+require 'rets/metadata/file_cache'
+require 'rets/metadata/json_serializer'
+require 'rets/metadata/yaml_serializer'
+require 'rets/metadata/marshal_serializer'
+
+require 'rets/metadata/containers'
+
+require 'rets/metadata/root'
+require 'rets/metadata/resource'
+require 'rets/metadata/lookup_type'
+require 'rets/metadata/table_factory'
+require 'rets/metadata/table'
+require 'rets/metadata/lookup_table'
+require 'rets/metadata/multi_lookup_table'
+require 'rets/metadata/rets_class'
+require 'rets/metadata/rets_object'

data/lib/rets/parser/compact.rb

@@ -0,0 +1,117 @@
+# coding: utf-8
+require 'cgi'
+
+module Rets
+  module Parser
+    class Compact
+      DEFAULT_DELIMITER = "\t"
+
+      INCLUDE_NULL_FIELDS = -1
+
+      InvalidDelimiter = Class.new(ArgumentError)
+
+      def self.parse_document(xml)
+        doc = SaxParser.new
+        parser = Nokogiri::XML::SAX::Parser.new(doc)
+        io = StringIO.new(xml.to_s)
+
+        parser.parse(io)
+        doc.results.map {|r| parse(doc.columns, r, doc.delimiter) }
+      end
+
+      class SaxParser < Nokogiri::XML::SAX::Document
+        attr_reader :results, :columns, :delimiter
+
+        def initialize
+          @results = []
+          @columns = ''
+          @result_index = nil
+          @delimiter = nil
+          @columns_start = false
+          @data_start = false
+        end
+
+        def start_element name, attrs=[]
+          case name
+          when 'DELIMITER'
+            @delimiter = attrs.last.last.to_i.chr
+          when 'COLUMNS'
+            @columns_start = true
+          when 'DATA'
+            @result_index = @results.size
+          end
+        end
+
+        def end_element name
+          case name
+          when 'COLUMNS'
+            @columns_start = false
+          when 'DATA'
+            @result_index = nil
+          end
+        end
+
+        def characters string
+          if @columns_start
+            @columns << string
+          end
+
+          if @result_index
+            @results[@result_index] ||= ''
+            @results[@result_index] << string
+          end
+        end
+      end
+
+      # Parses a single row of RETS-COMPACT data.
+      #
+      # Delimiter must be a regexp because String#split behaves differently when
+      # given a string pattern. (It removes leading spaces).
+      #
+      def self.parse(columns, data, delimiter = nil)
+        delimiter ||= DEFAULT_DELIMITER
+        delimiter = Regexp.new(Regexp.escape(delimiter))
+
+        if delimiter == // || delimiter == /,/
+          raise Rets::Parser::Compact::InvalidDelimiter, "Empty or invalid delimiter found, unable to parse."
+        end
+
+        column_names = columns.split(delimiter)
+        data_values = data.split(delimiter, INCLUDE_NULL_FIELDS).map do |x|
+          safely_decode_character_references!(x)
+          CGI.unescape_html(x)
+        end
+
+        zipped_key_values = column_names.zip(data_values).map { |k, v| [k.freeze, v.to_s] }
+
+        hash = Hash[*zipped_key_values.flatten]
+        hash.reject { |key, value| key.empty? && value.to_s.empty? }
+      end
+
+      def self.safely_decode_character_references!(string)
+        string.gsub!(/&#(x)?([\h]+);/) do
+          if $2
+            base = $1 == "x" ? 16 : 10
+            int = Integer($2, base)
+            begin
+              int.chr(Encoding::UTF_8)
+            rescue RangeError
+              ""
+            end
+          end
+        end
+        string
+      end
+
+      def self.get_count(xml)
+        doc = Nokogiri.parse(xml.to_s)
+        if node = doc.at("//COUNT")
+          node.attr('Records').to_i
+        else
+          0
+        end
+      end
+
+    end
+  end
+end

data/lib/rets/parser/error_checker.rb

@@ -0,0 +1,56 @@
+module Rets
+  module Parser
+    class ErrorChecker
+      INVALID_REQUEST_ERROR_MAPPING = Hash[Rets.constants.map {|c| Rets.const_get(c) }.select { |klass| klass.is_a?(Class) && klass < Rets::InvalidRequest }.map {|klass| [klass.const_get('ERROR_CODE'), klass] }]
+
+      def self.check(response)
+        # some RETS servers returns HTTP code 412 when session cookie expired, yet the response body
+        # passes XML check. We need to special case for this situation.
+        # This method is also called from multipart.rb where there are headers and body but no status_code
+        if response.respond_to?(:status_code) && response.status_code == 412
+          raise HttpError, "HTTP status: #{response.status_code}, body: #{response.body}"
+        end
+
+        # some RETS servers return success code in XML body but failure code 4xx in http status
+        # If xml body is present we ignore http status
+
+        if !response.body.empty?
+          begin
+            xml = Nokogiri::XML.parse(response.body, nil, nil, Nokogiri::XML::ParseOptions::STRICT)
+
+            rets_element = xml.xpath("/RETS")
+            unless rets_element.empty?
+              reply_text = (rets_element.attr("ReplyText") || rets_element.attr("replyText")).value
+              reply_code = (rets_element.attr("ReplyCode") || rets_element.attr("replyCode")).value.to_i
+
+              if reply_code == NoRecordsFound::ERROR_CODE
+                raise NoRecordsFound.new(reply_text)
+              elsif reply_code == NoObjectFound::ERROR_CODE
+                raise NoObjectFound.new(reply_text)
+              elsif reply_code.nonzero?
+                error_class = INVALID_REQUEST_ERROR_MAPPING[reply_code]
+                if error_class
+                  raise error_class.new(reply_code, reply_text)
+                else
+                  raise InvalidRequest.new(reply_code, reply_text)
+                end
+              else
+                return
+              end
+            end
+          rescue Nokogiri::XML::SyntaxError
+            #Not xml
+          end
+        end
+
+        if response.respond_to?(:ok?) && ! response.ok?
+          if response.status_code == 401
+            raise AuthorizationFailure.new(response.status_code, response.body)
+          else
+            raise HttpError, "HTTP status: #{response.status_code}, body: #{response.body}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rets/parser/multipart.rb

@@ -0,0 +1,39 @@
+module Rets
+  module Parser
+    # Inspired by Mail.
+    class Multipart
+      CRLF = "\r\n"
+      WSP  = "\s"
+      HEADER_LINE = /^([!-9;-~]+:\s*.+)$/
+
+      Part = Struct.new(:headers, :body)
+
+      def self.parse(raw, boundary)
+        parts = []
+        boundary_regexp = /--#{Regexp.quote(boundary)}(--)?#{CRLF}/
+
+        # WTF some RETS servers declare response body including jpeg binary is encoded in utf8
+        raw.force_encoding 'ascii-8bit' if raw.respond_to?(:force_encoding)
+
+        raw.split(boundary_regexp).each do |chunk|
+          header_part, body_part = chunk.split(/#{CRLF}#{WSP}*#{CRLF}/m, 2)
+
+          if header_part =~ HEADER_LINE
+            headers = header_part.split(/\r\n/).map { |kv| p = kv.split(/:\s?/); [p[0].downcase, p[1..-1].join(':')] }
+            headers = Hash[*headers.flatten]
+            parts << Part.new(headers, body_part)
+          else
+            next # not a valid chunk.
+          end
+        end
+        check_for_invalids_parts!(parts)
+        parts
+      end
+
+      def self.check_for_invalids_parts!(parts)
+        return unless parts.length == 1 && parts.first.headers['content-type'] == 'text/xml'
+        ErrorChecker.check(parts.first)
+      end
+    end
+  end
+end