ruby-rets 2.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +94 -0
- data/MIT-LICENSE +19 -0
- data/README.md +43 -0
- data/Rakefile +12 -0
- data/lib/rets/base/core.rb +290 -0
- data/lib/rets/base/sax_metadata.rb +63 -0
- data/lib/rets/base/sax_search.rb +58 -0
- data/lib/rets/client.rb +70 -0
- data/lib/rets/exceptions.rb +37 -0
- data/lib/rets/http.rb +329 -0
- data/lib/rets/stream_http.rb +159 -0
- data/lib/rets/version.rb +3 -0
- data/lib/ruby-rets.rb +9 -0
- data/ruby-rets.gemspec +24 -0
- metadata +100 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7b4fb43a879323ee4c589b274cb52c7f3f6d10b2
|
4
|
+
data.tar.gz: 8bdf9ae5be3b1da6aac9a203c29f63ddec811993
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: dc338096a89979e92136e00b857e7f9ec12597bff78c16d4e424d87230fffdc34fe6f7a9037dc31b0dd3a3fb563cc131bec596c4a2006addd4cc631cca988f9a
|
7
|
+
data.tar.gz: 74c09c0271b537c61e2c629215fe37c189ce3db9300a2d55174c3747fc1e4551439fd2cca5d11218e6eb9c78fc8144c5a75a53bf4b3adec78fe25640bb266c0e
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
# Overview
|
2
|
+
|
3
|
+
## 2.0.7
|
4
|
+
|
5
|
+
Confirmed to work with Ruby 2.0.0-p0
|
6
|
+
|
7
|
+
### Fixes
|
8
|
+
* `RETS-Version` is no longer set to `RETS/1.7` unless an error `20037` is returned and no other one is specified
|
9
|
+
* `RETS-Version` can be set based on a HTTP 200 response for systems which only pass it after authentication (Innovia)
|
10
|
+
|
11
|
+
## 2.0.6
|
12
|
+
|
13
|
+
### Features
|
14
|
+
* System name added to `rets_data[:system_id]` when pulling metadata out
|
15
|
+
* `client.request_size` and `client.request_hash` can be used when streaming is disabled in `client.search`
|
16
|
+
* Added `client.request_time` which gives the time taken in seconds for a `client.search` or `client.get_object` to run
|
17
|
+
|
18
|
+
### Fixes
|
19
|
+
* Fixed an authentication issue when using Retsiq where it would return 20041 in place of HTTP 401
|
20
|
+
* Fixed errors when streaming with an unknown Content-Length and chunked encoding
|
21
|
+
* Fixed a block being required when making a `client.search` call with `:count_mode => :only`
|
22
|
+
* Fixed character encoding that was causing issues on some RETS servers (Paul Trippett)
|
23
|
+
* Fixed an exception when Rapattoni RETS servers returned a "RETS-STATUS" XML tag on a multipart `client.get_object` call
|
24
|
+
* Fixed a `RETS::APIError` code 0 error under some RETS systems when making a call to `client.get_object` with `:location`
|
25
|
+
|
26
|
+
## 2.0.5
|
27
|
+
|
28
|
+
### Features
|
29
|
+
* Make `client.rets_data` available immediately when calling `client.search` rather than having to wait until it finishes (Paul Trippett)
|
30
|
+
* `:rets_version` can be passed to `client.login` without the User-Agent fields being set, for RETS servers that require the version to be passed initially
|
31
|
+
|
32
|
+
### Fixes
|
33
|
+
* Default to HTTP Digest authentication when Digest and Basic are provided (Paul Trippett)
|
34
|
+
* Fixed authentication header parser if a server returns Basic/Digest and the Digest data goes stale (Paul Trippett)
|
35
|
+
* Fixed query strings from the initial login request not being saved when discovery service URLs (Paul Trippett)
|
36
|
+
|
37
|
+
## 2.0.4
|
38
|
+
|
39
|
+
### Features
|
40
|
+
* Added support for RETS servers that use digest authentication without the quality of protection flag (MRIS)
|
41
|
+
* Added SSL support (Paul Trippett)
|
42
|
+
|
43
|
+
### Fixes
|
44
|
+
* Fixed metadata parsing breaking if a field wasn't filled out (Paul Trippett)
|
45
|
+
* Fixed multipart parsing for `client.get_object` if a part is blank
|
46
|
+
|
47
|
+
## 2.0.3
|
48
|
+
|
49
|
+
### Fixes
|
50
|
+
* Fixed a stack overflow due to how Interealty handles User-Agent authentication errors
|
51
|
+
|
52
|
+
## 2.0.2
|
53
|
+
|
54
|
+
### Features
|
55
|
+
* Dropped support for TimeoutSeconds, instead if an HTTP 401 is received after a successful request then a reauthentication is forced. Provides better compatibility with how some RETS implementations handle sessions
|
56
|
+
|
57
|
+
### Fixes
|
58
|
+
* Client methods no longer return the HTTP request
|
59
|
+
* Requests will correctly be called after a HTTP digest becomes stale
|
60
|
+
|
61
|
+
## 2.0.1
|
62
|
+
|
63
|
+
### API Changes
|
64
|
+
* `client.login` will now raise `ResponseError` errors if the RETS tag cannot be found in the response
|
65
|
+
* `client.login` added the ability to pass `:rets_version` to force the RETS Version used in HTTP requests. Provides a small speedup as it can skip one HTTP request depending on the RETS implementation
|
66
|
+
* `client.get_object` can return both Content-Description or Description rather than just Description. Also will return Preferred
|
67
|
+
|
68
|
+
### Features
|
69
|
+
* Added support for TimeoutSeconds, after the timeout passes the gem seamlessly reauthenticates
|
70
|
+
* Improved the edge case handling for authentication requests to greatly increase compatability with logging into any RETS based system
|
71
|
+
|
72
|
+
### Fixes
|
73
|
+
* Object multipart parsing no longer fails if the boundary is wrapped in quotes
|
74
|
+
* Response parsing won't fail if the RETS server uses odd casing for the "ReplyText" and "ReplyCode" args in RETS
|
75
|
+
|
76
|
+
## 2.0.0
|
77
|
+
|
78
|
+
### API Changes
|
79
|
+
* `client.logout` will now raise `CapabilityNotFound` errors if it's unsupported
|
80
|
+
* `client.get_object` now requires a block which is yielded to rather than returning an array of the content
|
81
|
+
* `client.get_object` headers are now returned in lowercase form ("content-id" not "Content-ID" and so on)
|
82
|
+
* `RETS::Client.login` now uses `:useragent => {:name => "Foo", :password => "Bar"}` to pass User Agent data
|
83
|
+
* `RETS::Client.login` no longer implies the User-Agent username or password by the primary username and password
|
84
|
+
|
85
|
+
### Features
|
86
|
+
* Added support for Count, Offset, Select and RestrictedIndicators in `client.search`
|
87
|
+
* Added support for Location in `client.get_object`
|
88
|
+
* RETS reply code, text and other data such as count or delimiter can be gotten through `client.rets_data` after the call is finished
|
89
|
+
|
90
|
+
### Fixes
|
91
|
+
* Redid how authentication is handled, no longer implies HTTP Basic auth when using RETS-UA-Authorization
|
92
|
+
* RETS-Version is now used for RETS-UA-Authorization when available with "RETS/1.7" as a fallback
|
93
|
+
* Exceptions are now raised consistently and have been simplifed to `APIError`, `HTTPError`, `Unauthorized` and `CapabilityNotFound`
|
94
|
+
* `HTTPError` and `APIError` now include the reply text and code in `reply_code` and `reply_text`
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
Copyright (c) 2011 Placester, Inc
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
5
|
+
in the Software without restriction, including without limitation the rights
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
8
|
+
furnished to do so, subject to the following conditions:
|
9
|
+
|
10
|
+
The above copyright notice and this permission notice shall be included in
|
11
|
+
all copies or substantial portions of the Software.
|
12
|
+
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
19
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
Overview
|
2
|
+
===
|
3
|
+
Simplifies the process of pulling data from RETS servers without having to worry about various authentication setups, should support all 1.x implementations. Parsing uses SAX to stream data as it comes rather than having to pull the entire document down and parse it all at once as some servers can return quite a lot of data.
|
4
|
+
|
5
|
+
Compability
|
6
|
+
-
|
7
|
+
Tested against Ruby 1.8.7, 1.9.2, 2.0.0, RBX and JRuby, build history is available [here](http://travis-ci.org/Placester/ruby-rets).
|
8
|
+
|
9
|
+
<img src="https://secure.travis-ci.org/Placester/ruby-rets.png?branch=master&.png"/>
|
10
|
+
|
11
|
+
Documentation
|
12
|
+
-
|
13
|
+
See http://rubydoc.info/github/Placester/ruby-rets/master/frames for full documentation.
|
14
|
+
|
15
|
+
Examples
|
16
|
+
-
|
17
|
+
|
18
|
+
client = RETS::Client.login(:url => "http://foobar.com/rets/Login", :username => "foo", :password => "bar")
|
19
|
+
client.search(:search_type => :Property, :class => :RES, :query => "(ListPrice=50000-)") do |data|
|
20
|
+
# RETS data in key/value format, as COMPACT-DECODED
|
21
|
+
end
|
22
|
+
|
23
|
+
client.get_object(:resource => :Property, :type => :Photo, :location => false, :id => "1:0:*") do |headers, content|
|
24
|
+
puts "Object-ID #{headers"object-id"]}, Content-ID #{headers["content-id"]}, Description #{["description"]}"
|
25
|
+
puts "Data"
|
26
|
+
puts content
|
27
|
+
end
|
28
|
+
|
29
|
+
VCR / WebMock
|
30
|
+
-
|
31
|
+
Due to the streaming parser, the search features won't work with a library like VCR or Ephemeral Response. For WebMock, you can use the below patch to enable support for saving the HTTP requests to speed up your own tests.
|
32
|
+
|
33
|
+
module Net
|
34
|
+
module WebMockHTTPResponse
|
35
|
+
def self.extended(response)
|
36
|
+
response.instance_variable_set(:@socket, StringIO.new(response.body))
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
License
|
42
|
+
-
|
43
|
+
Licensed under MIT
|
data/Rakefile
ADDED
@@ -0,0 +1,290 @@
|
|
1
|
+
# For more information on what the possible values of fields that are passed to the RETS server can be, see {http://www.rets.org/documentation}.
|
2
|
+
module RETS
|
3
|
+
module Base
|
4
|
+
class Core
|
5
|
+
GET_OBJECT_DATA = ["object-id", "description", "content-id", "content-description", "location", "content-type", "preferred"]
|
6
|
+
|
7
|
+
# Can be called after any {RETS::Base::Core} call that hits the RETS Server.
|
8
|
+
# @return [String] How big the request was
|
9
|
+
attr_reader :request_size
|
10
|
+
|
11
|
+
# Can be called after any {RETS::Base::Core} call that hits the RETS Server.
|
12
|
+
# @return [String] SHA1 hash of the request
|
13
|
+
attr_reader :request_hash
|
14
|
+
|
15
|
+
# Can be called after any {#get_object} or {#search} call that hits the RETS Server.
|
16
|
+
# @return [Float] How long the request took
|
17
|
+
attr_reader :request_time
|
18
|
+
|
19
|
+
# Can be called after any {RETS::Base::Core} call that hits the RETS Server.
|
20
|
+
# @return [Hash]
|
21
|
+
# Gives access to the miscellaneous RETS data, such as reply text, code, delimiter, count and so on depending on the API call made.
|
22
|
+
# * *text* (String) - Reply text from the server
|
23
|
+
# * *code* (String) - Reply code from the server
|
24
|
+
attr_reader :rets_data
|
25
|
+
|
26
|
+
def initialize(http, urls)
|
27
|
+
@http = http
|
28
|
+
@urls = urls
|
29
|
+
end
|
30
|
+
|
31
|
+
##
|
32
|
+
# Attempts to logout of the RETS server.
|
33
|
+
#
|
34
|
+
# @raise [RETS::CapabilityNotFound]
|
35
|
+
# @raise [RETS::APIError]
|
36
|
+
# @raise [RETS::HTTPError]
|
37
|
+
def logout
|
38
|
+
unless @urls[:logout]
|
39
|
+
raise RETS::CapabilityNotFound.new("No Logout capability found for given user.")
|
40
|
+
end
|
41
|
+
|
42
|
+
@http.request(:url => @urls[:logout])
|
43
|
+
|
44
|
+
nil
|
45
|
+
end
|
46
|
+
|
47
|
+
##
|
48
|
+
# Whether the RETS server has the requested capability.
|
49
|
+
#
|
50
|
+
# @param [Symbol] type Lowercase of the capability, "getmetadata", "getobject" and so on
|
51
|
+
# @return [Boolean]
|
52
|
+
def has_capability?(type)
|
53
|
+
@urls.has_key?(type)
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# Requests metadata from the RETS server.
|
58
|
+
#
|
59
|
+
# @param [Hash] args
|
60
|
+
# @option args [String] :type Metadata to request, the same value if you were manually making the request, "METADATA-SYSTEM", "METADATA-CLASS" and so on
|
61
|
+
# @option args [String] :id Filter the data returned by ID, "*" would return all available data
|
62
|
+
# @option args [Integer, Optional] :read_timeout How many seconds to wait before giving up
|
63
|
+
#
|
64
|
+
# @yield For every group of metadata downloaded
|
65
|
+
# @yieldparam [String] :type Type of data that was parsed with "METADATA-" stripped out, for "METADATA-SYSTEM" this will be "SYSTEM"
|
66
|
+
# @yieldparam [Hash] :attrs Attributes of the data, generally *Version*, *Date* and *Resource* but can vary depending on what metadata you requested
|
67
|
+
# @yieldparam [Array] :metadata Array of hashes with metadata info
|
68
|
+
#
|
69
|
+
# @raise [RETS::CapabilityNotFound]
|
70
|
+
# @raise [RETS::APIError]
|
71
|
+
# @raise [RETS::HTTPError]
|
72
|
+
# @see #rets_data
|
73
|
+
# @see #request_size
|
74
|
+
# @see #request_hash
|
75
|
+
def get_metadata(args, &block)
|
76
|
+
raise ArgumentError, "No block passed" unless block_given?
|
77
|
+
|
78
|
+
unless @urls[:getmetadata]
|
79
|
+
raise RETS::CapabilityNotFound.new("No GetMetadata capability found for given user.")
|
80
|
+
end
|
81
|
+
|
82
|
+
@request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, nil
|
83
|
+
@http.request(:url => @urls[:getmetadata], :read_timeout => args[:read_timeout], :params => {:Format => :COMPACT, :Type => args[:type], :ID => args[:id]}) do |response|
|
84
|
+
stream = RETS::StreamHTTP.new(response)
|
85
|
+
sax = RETS::Base::SAXMetadata.new(block)
|
86
|
+
|
87
|
+
Nokogiri::XML::SAX::Parser.new(sax).parse_io(stream)
|
88
|
+
|
89
|
+
@request_size, @request_hash = stream.size, stream.hash
|
90
|
+
@rets_data = sax.rets_data
|
91
|
+
end
|
92
|
+
|
93
|
+
nil
|
94
|
+
end
|
95
|
+
|
96
|
+
##
|
97
|
+
# Requests an object from the RETS server.
|
98
|
+
#
|
99
|
+
# @param [Hash] args
|
100
|
+
# @option args [String] :resource Resource to load, typically *Property*
|
101
|
+
# @option args [String] :type Type of object you want, usually *Photo*
|
102
|
+
# @option args [String] :id What objects to return
|
103
|
+
# @option args [Array, Optional] :accept Array of MIME types to accept, by default this is *image/png*, *image/gif* and *image/jpeg*
|
104
|
+
# @option args [Boolean, Optional] :location Return the location of the object rather than the contents of it
|
105
|
+
# @option args [Integer, Optional] :read_timeout How many seconds to wait before timing out
|
106
|
+
#
|
107
|
+
# @yield For every object downloaded
|
108
|
+
# @yieldparam [Hash] :headers Object headers
|
109
|
+
# * *object-id* (String) - Objects ID
|
110
|
+
# * *content-id* (String) - Content ID
|
111
|
+
# * *content-type* (String) - MIME type of the content
|
112
|
+
# * *description* (String, Optional) - A description of the object
|
113
|
+
# * *location* (String, Optional) - Where the file is located, only returned is *location* is true
|
114
|
+
# @yieldparam [String, Optional] :content Content for the object, not called when *location* is set
|
115
|
+
#
|
116
|
+
# @raise [RETS::CapabilityNotFound]
|
117
|
+
# @raise [RETS::APIError]
|
118
|
+
# @raise [RETS::HTTPError]
|
119
|
+
# @see #rets_data
|
120
|
+
# @see #request_size
|
121
|
+
# @see #request_hash
|
122
|
+
def get_object(args, &block)
|
123
|
+
raise ArgumentError, "No block passed" unless block_given?
|
124
|
+
|
125
|
+
unless @urls[:getobject]
|
126
|
+
raise RETS::CapabilityNotFound.new("No GetObject capability found for given user.")
|
127
|
+
end
|
128
|
+
|
129
|
+
req = {:url => @urls[:getobject], :read_timeout => args[:read_timeout], :headers => {}}
|
130
|
+
req[:params] = {:Resource => args[:resource], :Type => args[:type], :Location => (args[:location] ? 1 : 0), :ID => args[:id]}
|
131
|
+
if args[:accept].is_a?(Array)
|
132
|
+
req[:headers]["Accept"] = args[:accept].join(",")
|
133
|
+
else
|
134
|
+
req[:headers]["Accept"] = "image/png,image/gif,image/jpeg"
|
135
|
+
end
|
136
|
+
|
137
|
+
# Will get swapped to a streaming call rather than a download-and-parse later, easy to do as it's called with a block now
|
138
|
+
start = Time.now.utc.to_f
|
139
|
+
|
140
|
+
@request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, nil
|
141
|
+
@http.request(req) do |response|
|
142
|
+
body = response.read_body
|
143
|
+
|
144
|
+
@request_time = Time.now.utc.to_f - start
|
145
|
+
@request_size, @request_hash = body.length, Digest::SHA1.hexdigest(body)
|
146
|
+
|
147
|
+
# Make sure we aren't erroring
|
148
|
+
if body =~ /(<RETS (.+)\>)/
|
149
|
+
# RETSIQ (and probably others) return a <RETS> tag on a location request without any error inside
|
150
|
+
# since parsing errors out of full image data calls is a tricky pain. We're going to keep the
|
151
|
+
# loose error checking, but will confirm that it has an actual error code
|
152
|
+
code, text = @http.get_rets_response(Nokogiri::XML($1).xpath("//RETS").first)
|
153
|
+
unless code == "0"
|
154
|
+
@rets_data = {:code => code, :text => text}
|
155
|
+
|
156
|
+
if code == "20403"
|
157
|
+
return
|
158
|
+
else
|
159
|
+
raise RETS::APIError.new("#{code}: #{text}", code, text)
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
# Using a wildcard somewhere
|
165
|
+
if response.content_type == "multipart/parallel"
|
166
|
+
boundary = response.type_params["boundary"]
|
167
|
+
boundary.gsub!(/^"|"$/, "")
|
168
|
+
|
169
|
+
parts = body.split("--#{boundary}\r\n")
|
170
|
+
parts.last.gsub!("\r\n--#{boundary}--", "")
|
171
|
+
parts.each do |part|
|
172
|
+
part.strip!
|
173
|
+
next if part == ""
|
174
|
+
|
175
|
+
headers, content = part.split("\r\n\r\n", 2)
|
176
|
+
|
177
|
+
parsed_headers = {}
|
178
|
+
headers.split("\r\n").each do |line|
|
179
|
+
name, value = line.split(":", 2)
|
180
|
+
next unless value and value != ""
|
181
|
+
|
182
|
+
parsed_headers[name.downcase] = value.strip
|
183
|
+
end
|
184
|
+
|
185
|
+
# Check off the first children because some Rap Rets seems to use RETS-Status
|
186
|
+
# and it will include it with an object while returning actual data.
|
187
|
+
# It only does this for multipart requests, single image pulls will use <RETS> like it should.
|
188
|
+
if parsed_headers["content-type"] == "text/xml"
|
189
|
+
code, text = @http.get_rets_response(Nokogiri::XML(content).children.first)
|
190
|
+
next if code == "20403"
|
191
|
+
end
|
192
|
+
|
193
|
+
if block.arity == 1
|
194
|
+
yield parsed_headers
|
195
|
+
else
|
196
|
+
yield parsed_headers, content
|
197
|
+
end
|
198
|
+
|
199
|
+
end
|
200
|
+
|
201
|
+
# Either text (error) or an image of some sorts, which is irrelevant for this
|
202
|
+
else
|
203
|
+
headers = {}
|
204
|
+
GET_OBJECT_DATA.each do |field|
|
205
|
+
next unless response.header[field] and response.header[field] != ""
|
206
|
+
headers[field] = response.header[field].strip
|
207
|
+
end
|
208
|
+
|
209
|
+
if block.arity == 1
|
210
|
+
yield headers
|
211
|
+
else
|
212
|
+
yield headers, body
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
nil
|
218
|
+
end
|
219
|
+
|
220
|
+
##
|
221
|
+
# Searches the RETS server for data.
|
222
|
+
#
|
223
|
+
# @param [Hash] args
|
224
|
+
# @option args [String] :search_type What to search on, typically *Property*, *Office* or *Agent*
|
225
|
+
# @option args [String] :class What class of data to return, varies between RETS implementations and can be anything from *1* to *ResidentialProperty*
|
226
|
+
# @option args [String] :query How to filter data, should be unescaped as CGI::escape will be called on the string
|
227
|
+
# @option args [Symbol, Optional] :count_mode Either *:only* to return just the total records found or *:both* to get count and records returned
|
228
|
+
# @option args [Integer, Optional] :limit Limit total records returned
|
229
|
+
# @option args [Integer, Optional] :offset Offset to start returning records from
|
230
|
+
# @option args [Array, Optional] :select Restrict the fields the RETS server returns
|
231
|
+
# @option args [Boolean, Optional] :standard_names Whether to use standard names for all fields
|
232
|
+
# @option args [String, Optional] :restricted String to show in place of a field value for any restricted fields the user cannot see
|
233
|
+
# @option args [Integer, Optional] :read_timeout How long to wait for data from the socket before giving up
|
234
|
+
# @option args [Boolean, Optional] :disable_stream Disables the streaming setup for data and instead loads it all and then parses
|
235
|
+
#
|
236
|
+
# @yield Called for every <DATA></DATA> group from the RETS server
|
237
|
+
# @yieldparam [Hash] :data One record of data from the RETS server
|
238
|
+
#
|
239
|
+
# @raise [RETS::CapabilityNotFound]
|
240
|
+
# @raise [RETS::APIError]
|
241
|
+
# @raise [RETS::HTTPError]
|
242
|
+
# @see #rets_data
|
243
|
+
# @see #request_size
|
244
|
+
# @see #request_hash
|
245
|
+
def search(args, &block)
|
246
|
+
if !block_given? and args[:count_mode] != :only
|
247
|
+
raise ArgumentError, "No block found"
|
248
|
+
end
|
249
|
+
|
250
|
+
unless @urls[:search]
|
251
|
+
raise RETS::CapabilityNotFound.new("Cannot find URL for Search call")
|
252
|
+
end
|
253
|
+
|
254
|
+
req = {:url => @urls[:search], :read_timeout => args[:read_timeout]}
|
255
|
+
req[:params] = {:Format => "COMPACT-DECODED", :SearchType => args[:search_type], :QueryType => "DMQL2", :Query => args[:query], :Class => args[:class], :Limit => args[:limit], :Offset => args[:offset], :RestrictedIndicator => args[:restricted]}
|
256
|
+
req[:params][:Select] = args[:select].join(",") if args[:select].is_a?(Array)
|
257
|
+
req[:params][:StandardNames] = 1 if args[:standard_names]
|
258
|
+
|
259
|
+
if args[:count_mode] == :only
|
260
|
+
req[:params][:Count] = 2
|
261
|
+
elsif args[:count_mode] == :both
|
262
|
+
req[:params][:Count] = 1
|
263
|
+
end
|
264
|
+
|
265
|
+
@request_size, @request_hash, @request_time, @rets_data = nil, nil, nil, {}
|
266
|
+
|
267
|
+
start = Time.now.utc.to_f
|
268
|
+
@http.request(req) do |response|
|
269
|
+
if args[:disable_stream]
|
270
|
+
stream = StringIO.new(response.body)
|
271
|
+
@request_time = Time.now.utc.to_f - start
|
272
|
+
else
|
273
|
+
stream = RETS::StreamHTTP.new(response)
|
274
|
+
end
|
275
|
+
|
276
|
+
sax = RETS::Base::SAXSearch.new(@rets_data, block)
|
277
|
+
Nokogiri::XML::SAX::Parser.new(sax).parse_io(stream)
|
278
|
+
|
279
|
+
if args[:disable_stream]
|
280
|
+
@request_size, @request_hash = response.body.length, Digest::SHA1.hexdigest(response.body)
|
281
|
+
else
|
282
|
+
@request_size, @request_hash = stream.size, stream.hash
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
nil
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# SAX parser for the GetMetadata call.
|
2
|
+
class RETS::Base::SAXMetadata < Nokogiri::XML::SAX::Document
|
3
|
+
attr_accessor :rets_data
|
4
|
+
|
5
|
+
def initialize(block)
|
6
|
+
@rets_data = {:delimiter => "\t"}
|
7
|
+
@block = block
|
8
|
+
@parent = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def start_element(tag, attrs)
|
12
|
+
@current_tag = nil
|
13
|
+
|
14
|
+
# Figure out if the request is a success
|
15
|
+
if tag == "RETS"
|
16
|
+
@rets_data[:code], @rets_data[:text] = attrs.first.last, attrs.last.last
|
17
|
+
if @rets_data[:code] != "0" and @rets_data[:code] != "20201"
|
18
|
+
raise RETS::APIError.new("#{@rets_data[:code]}: #{@rets_data[:text]}", @rets_data[:code], @rets_data[:text])
|
19
|
+
end
|
20
|
+
|
21
|
+
elsif tag == "SYSTEM"
|
22
|
+
@rets_data[:system_id] = attrs.first.last
|
23
|
+
|
24
|
+
# Parsing data
|
25
|
+
elsif tag == "COLUMNS" or tag == "DATA"
|
26
|
+
@buffer = ""
|
27
|
+
@current_tag = tag
|
28
|
+
|
29
|
+
# Start of the parent we're working with
|
30
|
+
elsif tag =~ /^METADATA-(.+)/
|
31
|
+
@parent[:tag] = tag
|
32
|
+
@parent[:name] = $1
|
33
|
+
@parent[:data] = []
|
34
|
+
@parent[:attrs] = {}
|
35
|
+
attrs.each {|attr| @parent[:attrs][attr[0]] = attr[1] }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def characters(string)
|
40
|
+
@buffer << string if @current_tag
|
41
|
+
end
|
42
|
+
|
43
|
+
def end_element(tag)
|
44
|
+
return unless @current_tag
|
45
|
+
|
46
|
+
if @current_tag == "COLUMNS"
|
47
|
+
@columns = @buffer.split(@rets_data[:delimiter])
|
48
|
+
elsif tag == "DATA"
|
49
|
+
data = {}
|
50
|
+
|
51
|
+
list = @buffer.split(@rets_data[:delimiter])
|
52
|
+
list.each_index do |index|
|
53
|
+
next if @columns[index].nil? or @columns[index] == ""
|
54
|
+
data[@columns[index]] = list[index]
|
55
|
+
end
|
56
|
+
|
57
|
+
@parent[:data].push(data)
|
58
|
+
elsif tag == @parent[:tag]
|
59
|
+
@block.call(@parent[:name], @parent[:attrs], @parent[:data])
|
60
|
+
@parent[:tag] = nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# SAX parser for the Search API call.
|
2
|
+
class RETS::Base::SAXSearch < Nokogiri::XML::SAX::Document
|
3
|
+
attr_reader :rets_data
|
4
|
+
|
5
|
+
def initialize(rets_data, block)
|
6
|
+
@block = block
|
7
|
+
@rets_data = rets_data
|
8
|
+
end
|
9
|
+
|
10
|
+
def start_element(tag, attrs)
|
11
|
+
@current_tag = nil
|
12
|
+
|
13
|
+
# Figure out if the request is a success
|
14
|
+
if tag == "RETS"
|
15
|
+
@rets_data[:code], @rets_data[:text] = attrs.first.last, attrs.last.last
|
16
|
+
if @rets_data[:code] != "0" and @rets_data[:code] != "20201"
|
17
|
+
raise RETS::APIError.new("#{@rets_data[:code]}: #{@rets_data[:text]}", @rets_data[:code], @rets_data[:text])
|
18
|
+
end
|
19
|
+
|
20
|
+
# Determine the separator for data
|
21
|
+
elsif tag == "DELIMITER"
|
22
|
+
@rets_data[:delimiter] = attrs.first.last.to_i.chr
|
23
|
+
|
24
|
+
# Total records returned
|
25
|
+
elsif tag == "COUNT"
|
26
|
+
@rets_data[:count] = attrs.first.last.to_i
|
27
|
+
|
28
|
+
# Parsing data
|
29
|
+
elsif tag == "COLUMNS" or tag == "DATA"
|
30
|
+
@buffer = ""
|
31
|
+
@current_tag = tag
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def characters(string)
|
36
|
+
@buffer << string if @current_tag
|
37
|
+
end
|
38
|
+
|
39
|
+
def end_element(tag)
|
40
|
+
return unless @current_tag
|
41
|
+
|
42
|
+
if @current_tag == "COLUMNS"
|
43
|
+
@columns = @buffer.split(@rets_data[:delimiter])
|
44
|
+
|
45
|
+
# Finalize data and send it off
|
46
|
+
elsif tag == "DATA"
|
47
|
+
data = {}
|
48
|
+
|
49
|
+
list = @buffer.split(@rets_data[:delimiter])
|
50
|
+
list.each_index do |index|
|
51
|
+
next if @columns[index].nil? or @columns[index] == ""
|
52
|
+
data[@columns[index]] = list[index]
|
53
|
+
end
|
54
|
+
|
55
|
+
@block.call(data)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/rets/client.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
require "nokogiri"
|
2
|
+
|
3
|
+
module RETS
|
4
|
+
class Client
|
5
|
+
URL_KEYS = {:getobject => true, :login => true, :logout => true, :search => true, :getmetadata => true}
|
6
|
+
|
7
|
+
##
|
8
|
+
# Attempts to login to a RETS server.
|
9
|
+
# @param [Hash] args
|
10
|
+
# @option args [String] :url Login URL for the RETS server
|
11
|
+
# @option args [String] :username Username to authenticate with
|
12
|
+
# @option args [String] :password Password to authenticate with
|
13
|
+
# @option args [Symbol, Optional] :auth_mode When set to *:basic* will automatically use HTTP Basic authentication, skips a discovery request when initially connecting
|
14
|
+
# @option args [Hash, Optional] :useragent Only necessary for User Agent authentication
|
15
|
+
# * :name [String, Optional] - Name to set the User-Agent to
|
16
|
+
# * :password [String, Optional] - Password to use for RETS-UA-Authorization
|
17
|
+
# @option args [String, Optional] :rets_version Forces RETS-UA-Authorization on the first request if this and useragent name/password are set. Can be auto detected, but usually lets you bypass 1 - 2 additional authentication requests initially.
|
18
|
+
# @option args [Hash, Optional] :http Additional configuration for the HTTP requests
|
19
|
+
# * :verify_mode [Integer, Optional] How to verify the SSL certificate when connecting through HTTPS, either OpenSSL::SSL::VERIFY_PEER or OpenSSL::SSL::VERIFY_NONE, defaults to OpenSSL::SSL::VERIFY_NONE
|
20
|
+
# * :ca_file [String, Optional] Path to the CA certification file in PEM format
|
21
|
+
# * :ca_path [String, Optional] Path to the directory containing CA certifications in PEM format
|
22
|
+
#
|
23
|
+
# @raise [ArgumentError]
|
24
|
+
# @raise [RETS::APIError]
|
25
|
+
# @raise [RETS::HTTPError]
|
26
|
+
# @raise [RETS::Unauthorized]
|
27
|
+
# @raise [RETS::ResponseError]
|
28
|
+
#
|
29
|
+
# @return [RETS::Base::Core]
|
30
|
+
def self.login(args)
|
31
|
+
raise ArgumentError, "No URL passed" unless args[:url]
|
32
|
+
|
33
|
+
urls = {:login => URI.parse(args[:url])}
|
34
|
+
raise ArgumentError, "Invalid URL passed" unless urls[:login].is_a?(URI::HTTP)
|
35
|
+
|
36
|
+
base_url = urls[:login].to_s
|
37
|
+
base_url.gsub!(urls[:login].path, "") if urls[:login].path
|
38
|
+
|
39
|
+
http = RETS::HTTP.new(args)
|
40
|
+
http.request(:url => urls[:login], :check_response => true) do |response|
|
41
|
+
rets_attr = Nokogiri::XML(response.body).xpath("//RETS")
|
42
|
+
if rets_attr.empty?
|
43
|
+
raise RETS::ResponseError, "Does not seem to be a RETS server."
|
44
|
+
end
|
45
|
+
|
46
|
+
rets_attr.first.content.split("\n").each do |row|
|
47
|
+
key, value = row.split("=", 2)
|
48
|
+
next unless key and value
|
49
|
+
|
50
|
+
key, value = key.downcase.strip.to_sym, value.strip
|
51
|
+
|
52
|
+
if URL_KEYS[key]
|
53
|
+
# In case it's a relative path and doesn't include the domain
|
54
|
+
if value =~ /(http|www)/
|
55
|
+
urls[key] = URI.parse(value)
|
56
|
+
else
|
57
|
+
key_url = URI.parse(urls[:login].to_s)
|
58
|
+
key_url.path = value
|
59
|
+
urls[key] = key_url
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
http.login_uri = urls[:login]
|
66
|
+
|
67
|
+
RETS::Base::Core.new(http, urls)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module RETS
|
2
|
+
##
|
3
|
+
# Generic module that provides access to the code and text separately of the exception
|
4
|
+
module ReplyErrors
|
5
|
+
attr_reader :reply_text, :reply_code
|
6
|
+
|
7
|
+
def initialize(msg, reply_code=nil, reply_text=nil)
|
8
|
+
super(msg)
|
9
|
+
@reply_code, @reply_text = reply_code, reply_text
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
##
|
14
|
+
# RETS server replied to a request with an error of some sort.
|
15
|
+
class APIError < StandardError
|
16
|
+
include ReplyErrors
|
17
|
+
end
|
18
|
+
|
19
|
+
##
|
20
|
+
# Server responded with bad data.
|
21
|
+
class ResponseError < StandardError
|
22
|
+
end
|
23
|
+
|
24
|
+
##
|
25
|
+
# HTTP errors related to a request.
|
26
|
+
class HTTPError < StandardError
|
27
|
+
include ReplyErrors
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# Cannot login
|
32
|
+
class Unauthorized < RuntimeError; end
|
33
|
+
|
34
|
+
##
|
35
|
+
# Account does not have access to the requested API.
|
36
|
+
class CapabilityNotFound < RuntimeError; end
|
37
|
+
end
|
data/lib/rets/http.rb
ADDED
@@ -0,0 +1,329 @@
|
|
1
|
+
require "net/https"
|
2
|
+
require "digest"
|
3
|
+
|
4
|
+
module RETS
|
5
|
+
class HTTP
|
6
|
+
attr_accessor :login_uri
|
7
|
+
|
8
|
+
##
|
9
|
+
# Creates a new HTTP instance which will automatically handle authenting to the RETS server.
|
10
|
+
def initialize(args)
|
11
|
+
@headers = {"User-Agent" => "Ruby RETS/v#{RETS::VERSION}"}
|
12
|
+
@request_count = 0
|
13
|
+
@config = {:http => {}}.merge(args)
|
14
|
+
@rets_data, @cookie_list = {}, {}
|
15
|
+
|
16
|
+
if @config[:useragent] and @config[:useragent][:name]
|
17
|
+
@headers["User-Agent"] = @config[:useragent][:name]
|
18
|
+
end
|
19
|
+
|
20
|
+
if @config[:rets_version]
|
21
|
+
@rets_data[:version] = @config[:rets_version]
|
22
|
+
self.setup_ua_authorization(:version => @config[:rets_version])
|
23
|
+
end
|
24
|
+
|
25
|
+
if @config[:auth_mode] == :basic
|
26
|
+
@auth_mode = @config.delete(:auth_mode)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def url_encode(str)
|
31
|
+
encoded_string = ""
|
32
|
+
str.each_char do |char|
|
33
|
+
case char
|
34
|
+
when "+"
|
35
|
+
encoded_string << "%2b"
|
36
|
+
when "="
|
37
|
+
encoded_string << "%3d"
|
38
|
+
when "?"
|
39
|
+
encoded_string << "%3f"
|
40
|
+
when "&"
|
41
|
+
encoded_string << "%26"
|
42
|
+
when "%"
|
43
|
+
encoded_string << "%25"
|
44
|
+
when ","
|
45
|
+
encoded_string << "%2C"
|
46
|
+
else
|
47
|
+
encoded_string << char
|
48
|
+
end
|
49
|
+
end
|
50
|
+
encoded_string
|
51
|
+
end
|
52
|
+
|
53
|
+
def get_digest(header)
|
54
|
+
return unless header
|
55
|
+
|
56
|
+
header.each do |text|
|
57
|
+
mode, text = text.split(" ", 2)
|
58
|
+
return text if mode == "Digest"
|
59
|
+
end
|
60
|
+
|
61
|
+
nil
|
62
|
+
end
|
63
|
+
|
64
|
+
##
|
65
|
+
# Creates and manages the HTTP digest auth
|
66
|
+
# if the WWW-Authorization header is passed, then it will overwrite what it knows about the auth data.
|
67
|
+
def save_digest(header)
|
68
|
+
@request_count = 0
|
69
|
+
|
70
|
+
@digest = {}
|
71
|
+
header.split(",").each do |line|
|
72
|
+
k, v = line.strip.split("=", 2)
|
73
|
+
@digest[k] = (k != "algorithm" and k != "stale") && v[1..-2] || v
|
74
|
+
end
|
75
|
+
|
76
|
+
@digest_type = @digest["qop"] ? @digest["qop"].split(",") : []
|
77
|
+
end
|
78
|
+
|
79
|
+
##
|
80
|
+
# Creates a HTTP digest header.
|
81
|
+
def create_digest(method, request_uri)
|
82
|
+
# http://en.wikipedia.org/wiki/Digest_access_authentication
|
83
|
+
first = Digest::MD5.hexdigest("#{@config[:username]}:#{@digest["realm"]}:#{@config[:password]}")
|
84
|
+
second = Digest::MD5.hexdigest("#{method}:#{request_uri}")
|
85
|
+
|
86
|
+
# Using the "newer" authentication QOP
|
87
|
+
if @digest_type.include?("auth")
|
88
|
+
cnonce = Digest::MD5.hexdigest("#{@headers["User-Agent"]}:#{@config[:password]}:#{@request_count}:#{@digest["nonce"]}")
|
89
|
+
hash = Digest::MD5.hexdigest("#{first}:#{@digest["nonce"]}:#{"%08X" % @request_count}:#{cnonce}:#{@digest["qop"]}:#{second}")
|
90
|
+
# Nothing specified, so default to the old one
|
91
|
+
elsif @digest_type.empty?
|
92
|
+
hash = Digest::MD5.hexdigest("#{first}:#{@digest["nonce"]}:#{second}")
|
93
|
+
else
|
94
|
+
raise RETS::HTTPError, "Cannot determine auth type for server (#{@digest_type.join(",")})"
|
95
|
+
end
|
96
|
+
|
97
|
+
http_digest = "Digest username=\"#{@config[:username]}\", "
|
98
|
+
http_digest << "realm=\"#{@digest["realm"]}\", "
|
99
|
+
http_digest << "nonce=\"#{@digest["nonce"]}\", "
|
100
|
+
http_digest << "uri=\"#{request_uri}\", "
|
101
|
+
http_digest << "algorithm=MD5, " unless @digest_type.empty?
|
102
|
+
http_digest << "response=\"#{hash}\", "
|
103
|
+
http_digest << "opaque=\"#{@digest["opaque"]}\""
|
104
|
+
|
105
|
+
unless @digest_type.empty?
|
106
|
+
http_digest << ", "
|
107
|
+
http_digest << "qop=\"#{@digest["qop"]}\", "
|
108
|
+
http_digest << "nc=#{"%08X" % @request_count}, "
|
109
|
+
http_digest << "cnonce=\"#{cnonce}\""
|
110
|
+
end
|
111
|
+
|
112
|
+
http_digest
|
113
|
+
end
|
114
|
+
|
115
|
+
##
|
116
|
+
# Creates a HTTP basic header.
|
117
|
+
def create_basic
|
118
|
+
"Basic " << ["#{@config[:username]}:#{@config[:password]}"].pack("m").delete("\r\n")
|
119
|
+
end
|
120
|
+
|
121
|
+
##
|
122
|
+
# Finds the ReplyText and ReplyCode attributes in the response
|
123
|
+
#
|
124
|
+
# @param [Nokogiri::XML::NodeSet] rets <RETS> attributes found
|
125
|
+
#
|
126
|
+
# @return [String] RETS ReplyCode
|
127
|
+
# @return [String] RETS ReplyText
|
128
|
+
def get_rets_response(rets)
|
129
|
+
code, text = nil, nil
|
130
|
+
rets.attributes.each do |attr|
|
131
|
+
key = attr.first.downcase
|
132
|
+
if key == "replycode"
|
133
|
+
code = attr.last.value
|
134
|
+
elsif key == "replytext"
|
135
|
+
text = attr.last.value
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
return code, text
|
140
|
+
end
|
141
|
+
|
142
|
+
##
|
143
|
+
# Handles managing the relevant RETS-UA-Authorization headers
|
144
|
+
#
|
145
|
+
# @param [Hash] args
|
146
|
+
# @option args [String] :version RETS Version
|
147
|
+
# @option args [String, Optional] :session_id RETS Session ID
|
148
|
+
def setup_ua_authorization(args)
|
149
|
+
# Most RETS implementations don't care about RETS-Version for RETS-UA-Authorization, they don't require RETS-Version in general.
|
150
|
+
# Rapattoni require RETS-Version even without RETS-UA-Authorization, so will try and set the header when possible from the HTTP request rather than implying it.
|
151
|
+
# Interealty requires RETS-Version for RETS-UA-Authorization, so will fake it when we get an 20037 error
|
152
|
+
@headers["RETS-Version"] = args[:version] if args[:version]
|
153
|
+
|
154
|
+
if @headers["RETS-Version"] and @config[:useragent] and @config[:useragent][:password]
|
155
|
+
login = Digest::MD5.hexdigest("#{@config[:useragent][:name]}:#{@config[:useragent][:password]}")
|
156
|
+
@headers.merge!("RETS-UA-Authorization" => "Digest #{Digest::MD5.hexdigest("#{login}::#{args[:session_id]}:#{@headers["RETS-Version"]}")}")
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
##
|
161
|
+
# Sends a request to the RETS server.
|
162
|
+
#
|
163
|
+
# @param [Hash] args
|
164
|
+
# @option args [URI] :url URI to request data from
|
165
|
+
# @option args [Hash, Optional] :params Query string to include with the request
|
166
|
+
# @option args [Integer, Optional] :read_timeout How long to wait for the socket to return data before timing out
|
167
|
+
#
|
168
|
+
# @raise [RETS::APIError]
|
169
|
+
# @raise [RETS::HTTPError]
|
170
|
+
# @raise [RETS::Unauthorized]
|
171
|
+
def request(args, &block)
|
172
|
+
if args[:params]
|
173
|
+
url_terminator = (args[:url].request_uri.include?("?")) ? "&" : "?"
|
174
|
+
request_uri = "#{args[:url].request_uri}#{url_terminator}"
|
175
|
+
args[:params].each do |k, v|
|
176
|
+
request_uri << "#{k}=#{url_encode(v.to_s)}&" if v
|
177
|
+
end
|
178
|
+
else
|
179
|
+
request_uri = args[:url].request_uri
|
180
|
+
end
|
181
|
+
|
182
|
+
headers = args[:headers]
|
183
|
+
|
184
|
+
# Digest will change every time due to how its setup
|
185
|
+
@request_count += 1
|
186
|
+
if @auth_mode == :digest
|
187
|
+
if headers
|
188
|
+
headers["Authorization"] = create_digest("GET", request_uri)
|
189
|
+
else
|
190
|
+
headers = {"Authorization" => create_digest("GET", request_uri)}
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
headers = headers ? @headers.merge(headers) : @headers
|
195
|
+
|
196
|
+
http = ::Net::HTTP.new(args[:url].host, args[:url].port)
|
197
|
+
http.read_timeout = args[:read_timeout] if args[:read_timeout]
|
198
|
+
http.set_debug_output(@config[:debug_output]) if @config[:debug_output]
|
199
|
+
|
200
|
+
if args[:url].scheme == "https"
|
201
|
+
http.use_ssl = true
|
202
|
+
http.verify_mode = @config[:http][:verify_mode] || OpenSSL::SSL::VERIFY_NONE
|
203
|
+
http.ca_file = @config[:http][:ca_file] if @config[:http][:ca_file]
|
204
|
+
http.ca_path = @config[:http][:ca_path] if @config[:http][:ca_path]
|
205
|
+
end
|
206
|
+
|
207
|
+
http.start do
|
208
|
+
http.request_get(request_uri, headers) do |response|
|
209
|
+
# Pass along the cookies
|
210
|
+
# Some servers will continually call Set-Cookie with the same value for every single request
|
211
|
+
# to avoid authentication problems from cookies being stomped over (which is sad, nobody likes having their cookies crushed).
|
212
|
+
# We keep a hash of every cookie set and only update it if something changed
|
213
|
+
if response.header["set-cookie"]
|
214
|
+
cookies_changed = nil
|
215
|
+
|
216
|
+
response.header.get_fields("set-cookie").each do |cookie|
|
217
|
+
key, value = cookie.split(";").first.split("=")
|
218
|
+
key.strip!
|
219
|
+
value.strip!
|
220
|
+
|
221
|
+
# If it's a RETS-Session-ID, it needs to be shoved into the RETS-UA-Authorization field
|
222
|
+
# Save the RETS-Session-ID so it can be used with RETS-UA-Authorization
|
223
|
+
if key.downcase == "rets-session-id"
|
224
|
+
@rets_data[:session_id] = value
|
225
|
+
self.setup_ua_authorization(@rets_data) if @rets_data[:version]
|
226
|
+
end
|
227
|
+
|
228
|
+
cookies_changed = true if @cookie_list[key] != value
|
229
|
+
@cookie_list[key] = value
|
230
|
+
end
|
231
|
+
|
232
|
+
if cookies_changed
|
233
|
+
@headers.merge!("Cookie" => @cookie_list.map {|k, v| "#{k}=#{v}"}.join("; "))
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Rather than returning HTTP 401 when User-Agent authentication is needed, Retsiq returns HTTP 200
|
238
|
+
# with RETS error 20037. If we get a 20037, will let it pass through and handle it as if it was a HTTP 401.
|
239
|
+
# Retsiq apparently returns a 20041 now instead of a 20037 for the same use case.
|
240
|
+
# StratusRETS returns 20052 for an expired season
|
241
|
+
rets_code = nil
|
242
|
+
if response.code != "401" and ( response.code != "200" or args[:check_response] )
|
243
|
+
if response.body =~ /<RETS/i
|
244
|
+
rets_code, text = self.get_rets_response(Nokogiri::XML(response.body).xpath("//RETS").first)
|
245
|
+
unless rets_code == "20037" or rets_code == "20041" or rets_code == "20052" or rets_code == "0"
|
246
|
+
raise RETS::APIError.new("#{rets_code}: #{text}", rets_code, text)
|
247
|
+
end
|
248
|
+
|
249
|
+
elsif !args[:check_response]
|
250
|
+
raise RETS::HTTPError.new("#{response.code}: #{response.message}", response.code, response.message)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
|
254
|
+
# Strictly speaking, we do not need to set a RETS-Version in most cases, if RETS-UA-Authorization is not used
|
255
|
+
# It makes more sense to be safe and set it. Innovia at least does not set this until authentication is successful
|
256
|
+
# which is why this check is also here for HTTP 200s and not just 401s
|
257
|
+
if response.code == "200" and !@rets_data[:version] and response.header["rets-version"] != ""
|
258
|
+
@rets_data[:version] = response.header["rets-version"]
|
259
|
+
end
|
260
|
+
|
261
|
+
# Digest can become stale requiring us to reload data
|
262
|
+
if @auth_mode == :digest and response.header["www-authenticate"] =~ /stale=true/i
|
263
|
+
save_digest(get_digest(response.header.get_fields("www-authenticate")))
|
264
|
+
|
265
|
+
args[:block] ||= block
|
266
|
+
return self.request(args)
|
267
|
+
|
268
|
+
elsif response.code == "401" or rets_code == "20037" or rets_code == "20041" or rets_code == "20052"
|
269
|
+
raise RETS::Unauthorized, "Cannot login, check credentials" if ( @auth_mode and @retried_request ) or ( @retried_request and rets_code == "20037" )
|
270
|
+
@retried_request = true
|
271
|
+
|
272
|
+
# We already have an auth mode, and the request wasn't retried.
|
273
|
+
# Meaning we know that we had a successful authentication but something happened so we should relogin.
|
274
|
+
if @auth_mode
|
275
|
+
@headers.delete("Cookie")
|
276
|
+
@cookie_list = {}
|
277
|
+
|
278
|
+
self.request(:url => login_uri)
|
279
|
+
return self.request(args.merge(:block => block))
|
280
|
+
end
|
281
|
+
|
282
|
+
# Find a valid way of authenticating to the server as some will support multiple methods
|
283
|
+
if response.header.get_fields("www-authenticate") and !response.header.get_fields("www-authenticate").empty?
|
284
|
+
digest = get_digest(response.header.get_fields("www-authenticate"))
|
285
|
+
if digest
|
286
|
+
save_digest(digest)
|
287
|
+
@auth_mode = :digest
|
288
|
+
else
|
289
|
+
@headers.merge!("Authorization" => create_basic)
|
290
|
+
@auth_mode = :basic
|
291
|
+
end
|
292
|
+
|
293
|
+
unless @auth_mode
|
294
|
+
raise RETS::HTTPError.new("Cannot authenticate, no known mode found", response.code)
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
# Check if we need to deal with User-Agent authorization
|
299
|
+
if response.header["rets-version"] and response.header["rets-version"] != ""
|
300
|
+
@rets_data[:version] = response.header["rets-version"]
|
301
|
+
|
302
|
+
# If we get a 20037 error, it could be due to not having a RETS-Version set
|
303
|
+
# Under Innovia, passing RETS/1.7 will cause some errors
|
304
|
+
# because they don't pass the RETS-Version header until a successful login which is a HTTP 200
|
305
|
+
# They also don't use RETS-UA-Authorization, and it's better to not imply the RETS-Version header
|
306
|
+
# unless necessary, so will only do it for 20037 errors now.
|
307
|
+
elsif !@rets_data[:version] and rets_code == "20037"
|
308
|
+
@rets_data[:version] = "RETS/1.7"
|
309
|
+
end
|
310
|
+
|
311
|
+
self.setup_ua_authorization(@rets_data)
|
312
|
+
|
313
|
+
args[:block] ||= block
|
314
|
+
return self.request(args)
|
315
|
+
|
316
|
+
# We just tried to auth and don't have access to the original block in yieldable form
|
317
|
+
elsif args[:block]
|
318
|
+
@retried_request = nil
|
319
|
+
args.delete(:block).call(response)
|
320
|
+
|
321
|
+
elsif block_given?
|
322
|
+
@retried_request = nil
|
323
|
+
yield response
|
324
|
+
end
|
325
|
+
end
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
# This is a slightly crazy hack, but it's saner if we can just use Net::HTTP and then fallback on the StreamHTTP class when we need to do stream parsing.
|
2
|
+
# If we were to do it ourselves with Sockets, it would be a bigger pain to manage that, and we would have to do roughly the same setup as below anyway.
|
3
|
+
# Essentially, for the hack of using instance_variable_get/instance_variable_set, we get a simple stream parser, without having to write our own HTTP class.
|
4
|
+
module RETS
|
5
|
+
class StreamHTTP
|
6
|
+
ENCODABLE = RUBY_VERSION >= "1.9.0"
|
7
|
+
|
8
|
+
##
|
9
|
+
# Initializes a new HTTP stream which can be passed to Nokogiri for SAX parsing.
|
10
|
+
# @param [Net::HTTPResponse] response
|
11
|
+
# Unused HTTP response, no calls to any of the read_body or other methods can have been called.
|
12
|
+
def initialize(response)
|
13
|
+
@response = response
|
14
|
+
@left_to_read = @response.content_length
|
15
|
+
@content_length = @response.content_length
|
16
|
+
@chunked = @response.chunked?
|
17
|
+
@socket = @response.instance_variable_get(:@socket)
|
18
|
+
|
19
|
+
@digest = Digest::SHA1.new
|
20
|
+
@total_size = 0
|
21
|
+
|
22
|
+
if @response.header.key?("content-type") and @response["content-type"] =~ /.*charset=(.*)/i
|
23
|
+
@encoding = $1.to_s.upcase
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
##
|
28
|
+
# The total size read from the stream, can be called either while reading or at the end.
|
29
|
+
def size
|
30
|
+
@total_size
|
31
|
+
end
|
32
|
+
|
33
|
+
##
|
34
|
+
# SHA1 hash of the data read from the stream
|
35
|
+
def hash
|
36
|
+
@digest.hexdigest
|
37
|
+
end
|
38
|
+
|
39
|
+
##
|
40
|
+
# Detected encoding
|
41
|
+
def encoding
|
42
|
+
@encoding
|
43
|
+
end
|
44
|
+
|
45
|
+
##
|
46
|
+
# Read
|
47
|
+
#
|
48
|
+
# @param [Integer] read_len
|
49
|
+
# How many bytes to read from the HTTP stream
|
50
|
+
def read(read_len)
|
51
|
+
# If we closed the connection, return nil without calling anything again to avoid EOF
|
52
|
+
# or other errors
|
53
|
+
return nil if @closed
|
54
|
+
|
55
|
+
if @left_to_read
|
56
|
+
# We hit the end of what we need to read, if this is a chunked request, then we need to check for the next chunk
|
57
|
+
if @left_to_read <= read_len
|
58
|
+
data = @socket.read(@left_to_read)
|
59
|
+
@total_size += @left_to_read
|
60
|
+
@left_to_read = nil
|
61
|
+
@read_clfr = true
|
62
|
+
# Reading from known buffer still
|
63
|
+
else
|
64
|
+
@left_to_read -= read_len
|
65
|
+
@total_size += read_len
|
66
|
+
data = @socket.read(read_len)
|
67
|
+
end
|
68
|
+
|
69
|
+
elsif @chunked
|
70
|
+
# We finished reading the chunks, read the last 2 to get \r\n out of the way, and then find the next chunk
|
71
|
+
if @read_clfr
|
72
|
+
@read_clfr = nil
|
73
|
+
@socket.read(2)
|
74
|
+
end
|
75
|
+
|
76
|
+
data, chunk_read = "", 0
|
77
|
+
while true
|
78
|
+
# Read first line to get the chunk length
|
79
|
+
line = @socket.readline
|
80
|
+
|
81
|
+
len = line.slice(/[0-9a-fA-F]+/) or raise Net::HTTPBadResponse.new("wrong chunk size line: #{line}")
|
82
|
+
len = len.hex
|
83
|
+
|
84
|
+
# Nothing left, read off the final \r\n
|
85
|
+
if len == 0
|
86
|
+
@socket.read(2)
|
87
|
+
@socket.close
|
88
|
+
@response.instance_variable_set(:@read, true)
|
89
|
+
|
90
|
+
@closed = true
|
91
|
+
break
|
92
|
+
end
|
93
|
+
|
94
|
+
# Reading this chunk will set us over the buffer amount
|
95
|
+
# Read what we can of it (if anything), and send back what we have and queue a read for the rest
|
96
|
+
if ( chunk_read + len ) > read_len
|
97
|
+
can_read = len - ( ( chunk_read + len ) - read_len )
|
98
|
+
|
99
|
+
@left_to_read = len - can_read
|
100
|
+
@total_size += can_read
|
101
|
+
|
102
|
+
data << @socket.read(can_read) if can_read > 0
|
103
|
+
break
|
104
|
+
# We can just return the chunk as-is
|
105
|
+
else
|
106
|
+
@total_size += len
|
107
|
+
chunk_read += len
|
108
|
+
|
109
|
+
data << @socket.read(len)
|
110
|
+
@socket.read(2)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
# If we don't have a content length, then we need to keep reading until we run out of data
|
115
|
+
elsif !@content_length
|
116
|
+
data = @socket.readline
|
117
|
+
|
118
|
+
@total_size += data.length if data
|
119
|
+
end
|
120
|
+
|
121
|
+
# We've finished reading, set this so Net::HTTP doesn't try and read it again
|
122
|
+
if !data or data == ""
|
123
|
+
@response.instance_variable_set(:@read, true)
|
124
|
+
|
125
|
+
nil
|
126
|
+
else
|
127
|
+
if data.length >= @total_size and !@chunked
|
128
|
+
@response.instance_variable_set(:@read, true)
|
129
|
+
end
|
130
|
+
|
131
|
+
if ENCODABLE and @encoding
|
132
|
+
data = data.force_encoding(@encoding) if @encoding
|
133
|
+
data = data.encode("UTF-8")
|
134
|
+
end
|
135
|
+
|
136
|
+
@digest.update(data)
|
137
|
+
data
|
138
|
+
end
|
139
|
+
|
140
|
+
# Mark as read finished, return the last bits of data (if any)
|
141
|
+
rescue EOFError
|
142
|
+
@response.instance_variable_set(:@read, true)
|
143
|
+
@socket.close
|
144
|
+
@closed = true
|
145
|
+
|
146
|
+
if data and data != ""
|
147
|
+
@digest.update(data)
|
148
|
+
data
|
149
|
+
else
|
150
|
+
nil
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
##
|
155
|
+
# Does nothing, only used because Nokogiri requires it in a SAX parser.
|
156
|
+
def close
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
data/lib/rets/version.rb
ADDED
data/lib/ruby-rets.rb
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
path = File.expand_path("../rets", __FILE__)
|
2
|
+
require "#{path}/version"
|
3
|
+
require "#{path}/exceptions"
|
4
|
+
require "#{path}/client"
|
5
|
+
require "#{path}/http"
|
6
|
+
require "#{path}/stream_http"
|
7
|
+
require "#{path}/base/core"
|
8
|
+
require "#{path}/base/sax_search"
|
9
|
+
require "#{path}/base/sax_metadata"
|
data/ruby-rets.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'rets/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "ruby-rets"
|
8
|
+
spec.version = RETS::VERSION
|
9
|
+
spec.authors = ["Placester"]
|
10
|
+
spec.email = ["sean@seancoleman.net"]
|
11
|
+
spec.summary = %q{Mirror of the github ruby-rets gem}
|
12
|
+
spec.homepage = ""
|
13
|
+
spec.license = "MIT"
|
14
|
+
|
15
|
+
spec.files = `git ls-files -z`.split("\x0")
|
16
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
17
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
|
+
spec.require_paths = ["lib"]
|
19
|
+
|
20
|
+
spec.add_development_dependency "bundler", "~> 1.6"
|
21
|
+
spec.add_development_dependency "rake"
|
22
|
+
|
23
|
+
spec.add_runtime_dependency 'nokogiri', '>= 1.5.0'
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,100 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ruby-rets
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 2.0.8
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Placester
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.6'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.6'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: nokogiri
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 1.5.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.5.0
|
55
|
+
description:
|
56
|
+
email:
|
57
|
+
- sean@seancoleman.net
|
58
|
+
executables: []
|
59
|
+
extensions: []
|
60
|
+
extra_rdoc_files: []
|
61
|
+
files:
|
62
|
+
- CHANGELOG.md
|
63
|
+
- MIT-LICENSE
|
64
|
+
- README.md
|
65
|
+
- Rakefile
|
66
|
+
- lib/rets/base/core.rb
|
67
|
+
- lib/rets/base/sax_metadata.rb
|
68
|
+
- lib/rets/base/sax_search.rb
|
69
|
+
- lib/rets/client.rb
|
70
|
+
- lib/rets/exceptions.rb
|
71
|
+
- lib/rets/http.rb
|
72
|
+
- lib/rets/stream_http.rb
|
73
|
+
- lib/rets/version.rb
|
74
|
+
- lib/ruby-rets.rb
|
75
|
+
- ruby-rets.gemspec
|
76
|
+
homepage: ''
|
77
|
+
licenses:
|
78
|
+
- MIT
|
79
|
+
metadata: {}
|
80
|
+
post_install_message:
|
81
|
+
rdoc_options: []
|
82
|
+
require_paths:
|
83
|
+
- lib
|
84
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0'
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
requirements:
|
91
|
+
- - ">="
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
requirements: []
|
95
|
+
rubyforge_project:
|
96
|
+
rubygems_version: 2.2.2
|
97
|
+
signing_key:
|
98
|
+
specification_version: 4
|
99
|
+
summary: Mirror of the github ruby-rets gem
|
100
|
+
test_files: []
|