wku-ruby-rets 2.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,63 @@
1
+ # SAX parser for the GetMetadata call.
2
+ class RETS::Base::SAXMetadata < Nokogiri::XML::SAX::Document
3
+ attr_accessor :rets_data
4
+
5
+ def initialize(block)
6
+ @rets_data = {:delimiter => "\t"}
7
+ @block = block
8
+ @parent = {}
9
+ end
10
+
11
+ def start_element(tag, attrs)
12
+ @current_tag = nil
13
+
14
+ # Figure out if the request is a success
15
+ if tag == "RETS"
16
+ @rets_data[:code], @rets_data[:text] = attrs.first.last, attrs.last.last
17
+ if @rets_data[:code] != "0" and @rets_data[:code] != "20201"
18
+ raise RETS::APIError.new("#{@rets_data[:code]}: #{@rets_data[:text]}", @rets_data[:code], @rets_data[:text])
19
+ end
20
+
21
+ elsif tag == "SYSTEM"
22
+ @rets_data[:system_id] = attrs.first.last
23
+
24
+ # Parsing data
25
+ elsif tag == "COLUMNS" or tag == "DATA"
26
+ @buffer = ""
27
+ @current_tag = tag
28
+
29
+ # Start of the parent we're working with
30
+ elsif tag =~ /^METADATA-(.+)/
31
+ @parent[:tag] = tag
32
+ @parent[:name] = $1
33
+ @parent[:data] = []
34
+ @parent[:attrs] = {}
35
+ attrs.each {|attr| @parent[:attrs][attr[0]] = attr[1] }
36
+ end
37
+ end
38
+
39
+ def characters(string)
40
+ @buffer << string if @current_tag
41
+ end
42
+
43
+ def end_element(tag)
44
+ return unless @current_tag
45
+
46
+ if @current_tag == "COLUMNS"
47
+ @columns = @buffer.split(@rets_data[:delimiter])
48
+ elsif tag == "DATA"
49
+ data = {}
50
+
51
+ list = @buffer.split(@rets_data[:delimiter])
52
+ list.each_index do |index|
53
+ next if @columns[index].nil? or @columns[index] == ""
54
+ data[@columns[index]] = list[index]
55
+ end
56
+
57
+ @parent[:data].push(data)
58
+ elsif tag == @parent[:tag]
59
+ @block.call(@parent[:name], @parent[:attrs], @parent[:data])
60
+ @parent[:tag] = nil
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,58 @@
1
+ # SAX parser for the Search API call.
2
+ class RETS::Base::SAXSearch < Nokogiri::XML::SAX::Document
3
+ attr_reader :rets_data
4
+
5
+ def initialize(rets_data, block)
6
+ @block = block
7
+ @rets_data = rets_data
8
+ end
9
+
10
+ def start_element(tag, attrs)
11
+ @current_tag = nil
12
+
13
+ # Figure out if the request is a success
14
+ if tag == "RETS"
15
+ @rets_data[:code], @rets_data[:text] = attrs.first.last, attrs.last.last
16
+ if @rets_data[:code] != "0" and @rets_data[:code] != "20201"
17
+ raise RETS::APIError.new("#{@rets_data[:code]}: #{@rets_data[:text]}", @rets_data[:code], @rets_data[:text])
18
+ end
19
+
20
+ # Determine the separator for data
21
+ elsif tag == "DELIMITER"
22
+ @rets_data[:delimiter] = attrs.first.last.to_i.chr
23
+
24
+ # Total records returned
25
+ elsif tag == "COUNT"
26
+ @rets_data[:count] = attrs.first.last.to_i
27
+
28
+ # Parsing data
29
+ elsif tag == "COLUMNS" or tag == "DATA"
30
+ @buffer = ""
31
+ @current_tag = tag
32
+ end
33
+ end
34
+
35
+ def characters(string)
36
+ @buffer << string if @current_tag
37
+ end
38
+
39
+ def end_element(tag)
40
+ return unless @current_tag
41
+
42
+ if @current_tag == "COLUMNS"
43
+ @columns = @buffer.split(@rets_data[:delimiter])
44
+
45
+ # Finalize data and send it off
46
+ elsif tag == "DATA"
47
+ data = {}
48
+
49
+ list = @buffer.split(@rets_data[:delimiter])
50
+ list.each_index do |index|
51
+ next if @columns[index].nil? or @columns[index] == ""
52
+ data[@columns[index]] = list[index]
53
+ end
54
+
55
+ @block.call(data)
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,75 @@
1
+ require "nokogiri"
2
+
3
+ module RETS
4
+ class Client
5
+ URL_KEYS = {:getobject => true, :login => true, :logout => true, :search => true, :getmetadata => true}
6
+
7
+ ##
8
+ # Attempts to login to a RETS server.
9
+ # @param [Hash] args
10
+ # @option args [String] :url Login URL for the RETS server
11
+ # @option args [String] :username Username to authenticate with
12
+ # @option args [String] :password Password to authenticate with
13
+ # @option args [Symbol, Optional] :auth_mode When set to *:basic* will automatically use HTTP Basic authentication, skips a discovery request when initially connecting
14
+ # @option args [Hash, Optional] :useragent Only necessary for User Agent authentication
15
+ # * :name [String, Optional] - Name to set the User-Agent to
16
+ # * :password [String, Optional] - Password to use for RETS-UA-Authorization
17
+ # @option args [String, Optional] :rets_version Forces RETS-UA-Authorization on the first request if this and useragent name/password are set. Can be auto detected, but usually lets you bypass 1 - 2 additional authentication requests initially.
18
+ # @option args [Hash, Optional] :http Additional configuration for the HTTP requests
19
+ # * :verify_mode [Integer, Optional] How to verify the SSL certificate when connecting through HTTPS, either OpenSSL::SSL::VERIFY_PEER or OpenSSL::SSL::VERIFY_NONE, defaults to OpenSSL::SSL::VERIFY_NONE
20
+ # * :ca_file [String, Optional] Path to the CA certification file in PEM format
21
+ # * :ca_path [String, Optional] Path to the directory containing CA certifications in PEM format
22
+ # * :proxy [Hash, Optional] Use a HTTP proxy to communicate with the RETS server
23
+ # * :proxy :address [String, Optional] Address to connect to the proxy
24
+ # * :proxy :port [Integer, Optional] Port to use
25
+ # * :proxy :username [String, Optional] Username to authenticate to the proxy with if authentication is required
26
+ # * :proxy :password [String, Optional] Password to authenticate to the proxy with if authentication is required
27
+ #
28
+ # @raise [ArgumentError]
29
+ # @raise [RETS::APIError]
30
+ # @raise [RETS::HTTPError]
31
+ # @raise [RETS::Unauthorized]
32
+ # @raise [RETS::ResponseError]
33
+ #
34
+ # @return [RETS::Base::Core]
35
+ def self.login(args)
36
+ raise ArgumentError, "No URL passed" unless args[:url]
37
+
38
+ urls = {:login => URI.parse(args[:url])}
39
+ raise ArgumentError, "Invalid URL passed" unless urls[:login].is_a?(URI::HTTP)
40
+
41
+ base_url = urls[:login].to_s
42
+ base_url.gsub!(urls[:login].path, "") if urls[:login].path
43
+
44
+ http = RETS::HTTP.new(args)
45
+ http.request(:url => urls[:login], :check_response => true, :read_timeout => args[:read_timeout]) do |response|
46
+ rets_attr = Nokogiri::XML(response.body).xpath("//RETS")
47
+ if rets_attr.empty?
48
+ raise RETS::ResponseError, "Does not seem to be a RETS server."
49
+ end
50
+
51
+ rets_attr.first.content.split("\n").each do |row|
52
+ key, value = row.split("=", 2)
53
+ next unless key and value
54
+
55
+ key, value = key.downcase.strip.to_sym, value.strip
56
+
57
+ if URL_KEYS[key]
58
+ # In case it's a relative path and doesn't include the domain
59
+ if value =~ /(http|www)/
60
+ urls[key] = URI.parse(value)
61
+ else
62
+ key_url = URI.parse(urls[:login].to_s)
63
+ key_url.path = value
64
+ urls[key] = key_url
65
+ end
66
+ end
67
+ end
68
+ end
69
+
70
+ http.login_uri = urls[:login]
71
+
72
+ RETS::Base::Core.new(http, urls)
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,37 @@
1
+ module RETS
2
+ ##
3
+ # Generic module that provides access to the code and text separately of the exception
4
+ module ReplyErrors
5
+ attr_reader :reply_text, :reply_code
6
+
7
+ def initialize(msg, reply_code=nil, reply_text=nil)
8
+ super(msg)
9
+ @reply_code, @reply_text = reply_code, reply_text
10
+ end
11
+ end
12
+
13
+ ##
14
+ # RETS server replied to a request with an error of some sort.
15
+ class APIError < StandardError
16
+ include ReplyErrors
17
+ end
18
+
19
+ ##
20
+ # Server responded with bad data.
21
+ class ResponseError < StandardError
22
+ end
23
+
24
+ ##
25
+ # HTTP errors related to a request.
26
+ class HTTPError < StandardError
27
+ include ReplyErrors
28
+ end
29
+
30
+ ##
31
+ # Cannot login
32
+ class Unauthorized < RuntimeError; end
33
+
34
+ ##
35
+ # Account does not have access to the requested API.
36
+ class CapabilityNotFound < RuntimeError; end
37
+ end
data/lib/rets/http.rb ADDED
@@ -0,0 +1,346 @@
1
+ require "net/https"
2
+ require "digest"
3
+
4
+ module RETS
5
+ class HTTP
6
+ attr_accessor :login_uri
7
+
8
+ ##
9
+ # Creates a new HTTP instance which will automatically handle authenting to the RETS server.
10
+ def initialize(args)
11
+ @headers = {"User-Agent" => "Ruby RETS/v#{RETS::VERSION}"}
12
+ @request_count = 0
13
+ @config = {:http => {}}.merge(args)
14
+ @rets_data, @cookie_list = {}, {}
15
+
16
+ if @config[:useragent] and @config[:useragent][:name]
17
+ @headers["User-Agent"] = @config[:useragent][:name]
18
+ end
19
+
20
+ if @config[:rets_version]
21
+ @rets_data[:version] = @config[:rets_version]
22
+ self.setup_ua_authorization(:version => @config[:rets_version])
23
+ end
24
+
25
+ if @config[:auth_mode] == :basic
26
+ @auth_mode = @config.delete(:auth_mode)
27
+ end
28
+ end
29
+
30
+ def url_encode(str)
31
+ encoded_string = ""
32
+ str.each_char do |char|
33
+ case char
34
+ when "+"
35
+ encoded_string << "%2b"
36
+ when "="
37
+ encoded_string << "%3d"
38
+ when "?"
39
+ encoded_string << "%3f"
40
+ when "&"
41
+ encoded_string << "%26"
42
+ when "%"
43
+ encoded_string << "%25"
44
+ when ","
45
+ encoded_string << "%2C"
46
+ else
47
+ encoded_string << char
48
+ end
49
+ end
50
+ encoded_string
51
+ end
52
+
53
+ def get_digest(header)
54
+ return unless header
55
+
56
+ header.each do |text|
57
+ mode, text = text.split(" ", 2)
58
+ return text if mode == "Digest"
59
+ end
60
+
61
+ nil
62
+ end
63
+
64
+ ##
65
+ # Creates and manages the HTTP digest auth
66
+ # if the WWW-Authorization header is passed, then it will overwrite what it knows about the auth data.
67
+ def save_digest(header)
68
+ @request_count = 0
69
+
70
+ @digest = {}
71
+ header.split(",").each do |line|
72
+ k, v = line.strip.split("=", 2)
73
+ @digest[k] = (k != "algorithm" and k != "stale") && v[1..-2] || v
74
+ end
75
+
76
+ @digest_type = @digest["qop"] ? @digest["qop"].split(",") : []
77
+ end
78
+
79
+ ##
80
+ # Creates a HTTP digest header.
81
+ def create_digest(method, request_uri)
82
+ # http://en.wikipedia.org/wiki/Digest_access_authentication
83
+ first = Digest::MD5.hexdigest("#{@config[:username]}:#{@digest["realm"]}:#{@config[:password]}")
84
+ second = Digest::MD5.hexdigest("#{method}:#{request_uri}")
85
+
86
+ # Using the "newer" authentication QOP
87
+ if @digest_type.include?("auth")
88
+ cnonce = Digest::MD5.hexdigest("#{@headers["User-Agent"]}:#{@config[:password]}:#{@request_count}:#{@digest["nonce"]}")
89
+ hash = Digest::MD5.hexdigest("#{first}:#{@digest["nonce"]}:#{"%08X" % @request_count}:#{cnonce}:#{@digest["qop"]}:#{second}")
90
+ # Nothing specified, so default to the old one
91
+ elsif @digest_type.empty?
92
+ hash = Digest::MD5.hexdigest("#{first}:#{@digest["nonce"]}:#{second}")
93
+ else
94
+ raise RETS::HTTPError, "Cannot determine auth type for server (#{@digest_type.join(",")})"
95
+ end
96
+
97
+ http_digest = "Digest username=\"#{@config[:username]}\", "
98
+ http_digest << "realm=\"#{@digest["realm"]}\", "
99
+ http_digest << "nonce=\"#{@digest["nonce"]}\", "
100
+ http_digest << "uri=\"#{request_uri}\", "
101
+ http_digest << "algorithm=MD5, " unless @digest_type.empty?
102
+ http_digest << "response=\"#{hash}\", "
103
+ http_digest << "opaque=\"#{@digest["opaque"]}\""
104
+
105
+ unless @digest_type.empty?
106
+ http_digest << ", "
107
+ http_digest << "qop=\"#{@digest["qop"]}\", "
108
+ http_digest << "nc=#{"%08X" % @request_count}, "
109
+ http_digest << "cnonce=\"#{cnonce}\""
110
+ end
111
+
112
+ http_digest
113
+ end
114
+
115
+ ##
116
+ # Creates a HTTP basic header.
117
+ def create_basic
118
+ "Basic " << ["#{@config[:username]}:#{@config[:password]}"].pack("m").delete("\r\n")
119
+ end
120
+
121
+ ##
122
+ # Finds the ReplyText and ReplyCode attributes in the response
123
+ #
124
+ # @param [Nokogiri::XML::NodeSet] rets <RETS> attributes found
125
+ #
126
+ # @return [String] RETS ReplyCode
127
+ # @return [String] RETS ReplyText
128
+ def get_rets_response(rets)
129
+ code, text = nil, nil
130
+ rets.attributes.each do |attr|
131
+ key = attr.first.downcase
132
+ if key == "replycode"
133
+ code = attr.last.value
134
+ elsif key == "replytext"
135
+ text = attr.last.value
136
+ end
137
+ end
138
+
139
+ return code, text
140
+ end
141
+
142
+ ##
143
+ # Handles managing the relevant RETS-UA-Authorization headers
144
+ #
145
+ # @param [Hash] args
146
+ # @option args [String] :version RETS Version
147
+ # @option args [String, Optional] :session_id RETS Session ID
148
+ def setup_ua_authorization(args)
149
+ # Most RETS implementations don't care about RETS-Version for RETS-UA-Authorization, they don't require RETS-Version in general.
150
+ # Rapattoni require RETS-Version even without RETS-UA-Authorization, so will try and set the header when possible from the HTTP request rather than implying it.
151
+ # Interealty requires RETS-Version for RETS-UA-Authorization, so will fake it when we get an 20037 error
152
+ @headers["RETS-Version"] = args[:version] if args[:version]
153
+
154
+ if @headers["RETS-Version"] and @config[:useragent] and @config[:useragent][:password]
155
+ login = Digest::MD5.hexdigest("#{@config[:useragent][:name]}:#{@config[:useragent][:password]}")
156
+ @headers.merge!("RETS-UA-Authorization" => "Digest #{Digest::MD5.hexdigest("#{login}::#{args[:session_id]}:#{@headers["RETS-Version"]}")}")
157
+ end
158
+ end
159
+
160
+ ##
161
+ # Sends a request to the RETS server.
162
+ #
163
+ # @param [Hash] args
164
+ # @option args [URI] :url URI to request data from
165
+ # @option args [Hash, Optional] :params Query string to include with the request
166
+ # @option args [Integer, Optional] :read_timeout How long to wait for the socket to return data before timing out
167
+ #
168
+ # @raise [RETS::APIError]
169
+ # @raise [RETS::HTTPError]
170
+ # @raise [RETS::Unauthorized]
171
+ def request(args, &block)
172
+ if args[:params]
173
+ url_terminator = (args[:url].request_uri.include?("?")) ? "&" : "?"
174
+ request_uri = "#{args[:url].request_uri}#{url_terminator}"
175
+ args[:params].each do |k, v|
176
+ request_uri << "#{k}=#{url_encode(v.to_s)}&" if v
177
+ end
178
+ else
179
+ request_uri = args[:url].request_uri
180
+ end
181
+
182
+ headers = args[:headers]
183
+ if args[:disable_compression]
184
+ headers ||= {}
185
+ headers["Accept-Encoding"] = "identity"
186
+ end
187
+
188
+ # Digest will change every time due to how its setup
189
+ @request_count += 1
190
+ if @auth_mode == :digest
191
+ if headers
192
+ headers["Authorization"] = create_digest("GET", request_uri)
193
+ else
194
+ headers = {"Authorization" => create_digest("GET", request_uri)}
195
+ end
196
+ end
197
+
198
+ headers = headers ? @headers.merge(headers) : @headers
199
+
200
+ if !@config[:http][:proxy]
201
+ http = ::Net::HTTP.new(args[:url].host, args[:url].port)
202
+ else
203
+ http = ::Net::HTTP.new(args[:url].host, args[:url].port, @config[:http][:proxy][:address], @config[:http][:proxy][:port], @config[:http][:proxy][:username], @config[:http][:proxy][:password])
204
+ end
205
+
206
+ http.read_timeout = args[:read_timeout] if args[:read_timeout]
207
+ http.set_debug_output(@config[:debug_output]) if @config[:debug_output]
208
+
209
+ if args[:url].scheme == "https"
210
+ http.use_ssl = true
211
+ http.verify_mode = @config[:http][:verify_mode] || OpenSSL::SSL::VERIFY_NONE
212
+ http.ca_file = @config[:http][:ca_file] if @config[:http][:ca_file]
213
+ http.ca_path = @config[:http][:ca_path] if @config[:http][:ca_path]
214
+ end
215
+
216
+ http.start do
217
+ http.request_get(request_uri, headers) do |response|
218
+ # Pass along the cookies
219
+ # Some servers will continually call Set-Cookie with the same value for every single request
220
+ # to avoid authentication problems from cookies being stomped over (which is sad, nobody likes having their cookies crushed).
221
+ # We keep a hash of every cookie set and only update it if something changed
222
+ if response.header["set-cookie"]
223
+ cookies_changed = nil
224
+
225
+ response.header.get_fields("set-cookie").each do |cookie|
226
+ key, value = cookie.split(";").first.split("=")
227
+ key.strip!
228
+
229
+ # Sometimes we can get a nil value from raprets
230
+ unless value
231
+ cookies_changed = true if @cookie_list[key]
232
+ @cookie_list.delete(key)
233
+ next
234
+ end
235
+
236
+ value.strip!
237
+
238
+ # If it's a RETS-Session-ID, it needs to be shoved into the RETS-UA-Authorization field
239
+ # Save the RETS-Session-ID so it can be used with RETS-UA-Authorization
240
+ if key.downcase == "rets-session-id"
241
+ @rets_data[:session_id] = value
242
+ self.setup_ua_authorization(@rets_data) if @rets_data[:version]
243
+ end
244
+
245
+ cookies_changed = true if @cookie_list[key] != value
246
+ @cookie_list[key] = value
247
+ end
248
+
249
+ if cookies_changed
250
+ @headers.merge!("Cookie" => @cookie_list.map {|k, v| "#{k}=#{v}"}.join("; "))
251
+ end
252
+ end
253
+
254
+ # Rather than returning HTTP 401 when User-Agent authentication is needed, Retsiq returns HTTP 200
255
+ # with RETS error 20037. If we get a 20037, will let it pass through and handle it as if it was a HTTP 401.
256
+ # Retsiq apparently returns a 20041 now instead of a 20037 for the same use case.
257
+ # StratusRETS returns 20052 for an expired season
258
+ rets_code = nil
259
+ if response.code != "401" and ( response.code != "200" or args[:check_response] )
260
+ if response.body =~ /<RETS/i
261
+ rets_code, text = self.get_rets_response(Nokogiri::XML(response.body).xpath("//RETS").first)
262
+ unless rets_code == "20037" or rets_code == "20041" or rets_code == "20052" or rets_code == "0"
263
+ raise RETS::APIError.new("#{rets_code}: #{text}", rets_code, text)
264
+ end
265
+
266
+ elsif !args[:check_response]
267
+ raise RETS::HTTPError.new("#{response.code}: #{response.message}", response.code, response.message)
268
+ end
269
+ end
270
+
271
+ # Strictly speaking, we do not need to set a RETS-Version in most cases, if RETS-UA-Authorization is not used
272
+ # It makes more sense to be safe and set it. Innovia at least does not set this until authentication is successful
273
+ # which is why this check is also here for HTTP 200s and not just 401s
274
+ if response.code == "200" and !@rets_data[:version] and response.header["rets-version"] != ""
275
+ @rets_data[:version] = response.header["rets-version"]
276
+ end
277
+
278
+ # Digest can become stale requiring us to reload data
279
+ if @auth_mode == :digest and response.header["www-authenticate"] =~ /stale=true/i
280
+ save_digest(get_digest(response.header.get_fields("www-authenticate")))
281
+
282
+ args[:block] ||= block
283
+ return self.request(args)
284
+
285
+ elsif response.code == "401" or rets_code == "20037" or rets_code == "20041" or rets_code == "20052"
286
+ raise RETS::Unauthorized, "Cannot login, check credentials" if ( @auth_mode and @retried_request ) or ( @retried_request and rets_code == "20037" )
287
+ @retried_request = true
288
+
289
+ # We already have an auth mode, and the request wasn't retried.
290
+ # Meaning we know that we had a successful authentication but something happened so we should relogin.
291
+ if @auth_mode
292
+ @headers.delete("Cookie")
293
+ @cookie_list = {}
294
+
295
+ self.request(:url => login_uri)
296
+ return self.request(args.merge(:block => block))
297
+ end
298
+
299
+ # Find a valid way of authenticating to the server as some will support multiple methods
300
+ if response.header.get_fields("www-authenticate") and !response.header.get_fields("www-authenticate").empty?
301
+ digest = get_digest(response.header.get_fields("www-authenticate"))
302
+ if digest
303
+ save_digest(digest)
304
+ @auth_mode = :digest
305
+ else
306
+ @headers.merge!("Authorization" => create_basic)
307
+ @auth_mode = :basic
308
+ end
309
+
310
+ unless @auth_mode
311
+ raise RETS::HTTPError.new("Cannot authenticate, no known mode found", response.code)
312
+ end
313
+ end
314
+
315
+ # Check if we need to deal with User-Agent authorization
316
+ if response.header["rets-version"] and response.header["rets-version"] != ""
317
+ @rets_data[:version] = response.header["rets-version"]
318
+
319
+ # If we get a 20037 error, it could be due to not having a RETS-Version set
320
+ # Under Innovia, passing RETS/1.7 will cause some errors
321
+ # because they don't pass the RETS-Version header until a successful login which is a HTTP 200
322
+ # They also don't use RETS-UA-Authorization, and it's better to not imply the RETS-Version header
323
+ # unless necessary, so will only do it for 20037 errors now.
324
+ elsif !@rets_data[:version] and rets_code == "20037"
325
+ @rets_data[:version] = "RETS/1.7"
326
+ end
327
+
328
+ self.setup_ua_authorization(@rets_data)
329
+
330
+ args[:block] ||= block
331
+ return self.request(args)
332
+
333
+ # We just tried to auth and don't have access to the original block in yieldable form
334
+ elsif args[:block]
335
+ @retried_request = nil
336
+ args.delete(:block).call(response)
337
+
338
+ elsif block_given?
339
+ @retried_request = nil
340
+ yield response
341
+ end
342
+ end
343
+ end
344
+ end
345
+ end
346
+ end