kronk 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.autotest +23 -0
- data/History.txt +6 -0
- data/Manifest.txt +14 -0
- data/README.txt +118 -0
- data/Rakefile +20 -0
- data/bin/kronk +13 -0
- data/lib/kronk.rb +404 -0
- data/lib/kronk/data_set.rb +230 -0
- data/lib/kronk/diff.rb +210 -0
- data/lib/kronk/plist_parser.rb +15 -0
- data/lib/kronk/request.rb +191 -0
- data/lib/kronk/response.rb +226 -0
- data/lib/kronk/xml_parser.rb +108 -0
- data/test/test_data_set.rb +410 -0
- data/test/test_diff.rb +427 -0
- data/test/test_helper.rb +37 -0
- data/test/test_kronk.rb +192 -0
- data/test/test_request.rb +225 -0
- data/test/test_response.rb +258 -0
- data/test/test_xml_parser.rb +101 -0
- metadata +225 -0
@@ -0,0 +1,191 @@
|
|
1
|
+
class Kronk
|
2
|
+
|
3
|
+
##
|
4
|
+
# Request wrapper class for net/http.
|
5
|
+
|
6
|
+
class Request
|
7
|
+
|
8
|
+
class NotFoundError < Exception; end
|
9
|
+
|
10
|
+
##
|
11
|
+
# Follows the redirect from a 30X response object and decrease the
|
12
|
+
# number of redirects left if it's an Integer.
|
13
|
+
|
14
|
+
def self.follow_redirect resp, options={}
|
15
|
+
rdir = options[:follow_redirects]
|
16
|
+
rdir = rdir - 1 if Integer === rdir && rdir > 0
|
17
|
+
|
18
|
+
retrieve_uri resp['Location'], options.merge(:follow_redirects => rdir)
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
##
|
23
|
+
# Check the rdir value to figure out if redirect should be followed.
|
24
|
+
|
25
|
+
def self.follow_redirect? resp, rdir
|
26
|
+
resp.code.to_s =~ /^30\d$/ &&
|
27
|
+
(rdir == true || Integer === rdir && rdir > 0)
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
##
|
32
|
+
# Returns the value from a url, file, or cache as a String.
|
33
|
+
# Options supported are:
|
34
|
+
# :data:: Hash/String - the data to pass to the http request
|
35
|
+
# :follow_redirects:: Integer/Bool - number of times to follow redirects
|
36
|
+
# :headers:: Hash - extra headers to pass to the request
|
37
|
+
# :http_method:: Symbol - the http method to use; defaults to :get
|
38
|
+
#
|
39
|
+
# TODO: Log request speed.
|
40
|
+
|
41
|
+
def self.retrieve query, options={}
|
42
|
+
resp =
|
43
|
+
if !local?(query)
|
44
|
+
retrieve_uri query, options
|
45
|
+
else
|
46
|
+
retrieve_file query, options
|
47
|
+
end
|
48
|
+
|
49
|
+
begin
|
50
|
+
File.open(options[:cache_response], "w+") do |file|
|
51
|
+
file.write resp.raw
|
52
|
+
end if options[:cache_response]
|
53
|
+
rescue => e
|
54
|
+
$stderr << "#{e.class}: #{e.message}"
|
55
|
+
end
|
56
|
+
|
57
|
+
resp
|
58
|
+
rescue SocketError, Errno::ENOENT
|
59
|
+
raise NotFoundError, "#{query} could not be found"
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
##
|
64
|
+
# Check if a URI should be treated as a local file.
|
65
|
+
|
66
|
+
def self.local? uri
|
67
|
+
!(uri =~ %r{^\w+://})
|
68
|
+
end
|
69
|
+
|
70
|
+
|
71
|
+
##
|
72
|
+
# Read http response from a file and return a HTTPResponse instance.
|
73
|
+
|
74
|
+
def self.retrieve_file path, options={}
|
75
|
+
options = options.dup
|
76
|
+
|
77
|
+
path = Kronk::DEFAULT_CACHE_FILE if path == :cache
|
78
|
+
resp = nil
|
79
|
+
|
80
|
+
File.open(path, "r") do |file|
|
81
|
+
begin
|
82
|
+
resp = Response.read_new file
|
83
|
+
|
84
|
+
rescue Net::HTTPBadResponse
|
85
|
+
file.rewind
|
86
|
+
resp = HeadlessResponse.new file.read
|
87
|
+
resp['Content-Type'] = File.extname path
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
resp = follow_redirect resp, options if
|
92
|
+
follow_redirect? resp, options[:follow_redirects]
|
93
|
+
|
94
|
+
resp
|
95
|
+
end
|
96
|
+
|
97
|
+
|
98
|
+
##
|
99
|
+
# Make an http request to the given uri and return a HTTPResponse instance.
|
100
|
+
# Supports the following options:
|
101
|
+
# :data:: Hash/String - the data to pass to the http request
|
102
|
+
# :follow_redirects:: Integer/Bool - number of times to follow redirects
|
103
|
+
# :headers:: Hash - extra headers to pass to the request
|
104
|
+
# :http_method:: Symbol - the http method to use; defaults to :get
|
105
|
+
#
|
106
|
+
# Note: if no http method is specified and data is given, will default
|
107
|
+
# to using a post request.
|
108
|
+
|
109
|
+
def self.retrieve_uri uri, options={}
|
110
|
+
options = options.dup
|
111
|
+
http_method = options.delete(:http_method)
|
112
|
+
http_method ||= options[:data] ? :post : :get
|
113
|
+
|
114
|
+
resp = self.call http_method, uri, options
|
115
|
+
|
116
|
+
resp = follow_redirect resp, options if
|
117
|
+
follow_redirect? resp, options[:follow_redirects]
|
118
|
+
|
119
|
+
resp
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
##
|
124
|
+
# Make an http request to the given uri and return a HTTPResponse instance.
|
125
|
+
# Supports the following options:
|
126
|
+
# :data:: Hash/String - the data to pass to the http request
|
127
|
+
# :follow_redirects:: Integer/Bool - number of times to follow redirects
|
128
|
+
# :headers:: Hash - extra headers to pass to the request
|
129
|
+
# :http_method:: Symbol - the http method to use; defaults to :get
|
130
|
+
|
131
|
+
def self.call http_method, uri, options={}
|
132
|
+
suffix = options.delete :uri_suffix
|
133
|
+
|
134
|
+
uri = "#{uri}#{suffix}" if suffix
|
135
|
+
uri = URI.parse uri unless URI === uri
|
136
|
+
|
137
|
+
data = options[:data]
|
138
|
+
data &&= Hash === data ? build_query(data) : data.to_s
|
139
|
+
|
140
|
+
socket = socket_io = nil
|
141
|
+
|
142
|
+
resp = Net::HTTP.start uri.host, uri.port do |http|
|
143
|
+
socket = http.instance_variable_get "@socket"
|
144
|
+
socket.debug_output = socket_io = StringIO.new
|
145
|
+
|
146
|
+
http.send_request http_method.to_s.upcase,
|
147
|
+
uri.request_uri,
|
148
|
+
data,
|
149
|
+
options[:headers]
|
150
|
+
end
|
151
|
+
|
152
|
+
resp.extend Response::Helpers
|
153
|
+
|
154
|
+
r_req, r_resp, r_bytes = Response.read_raw_from socket_io
|
155
|
+
resp.instance_variable_set "@raw", r_resp
|
156
|
+
|
157
|
+
resp
|
158
|
+
end
|
159
|
+
|
160
|
+
|
161
|
+
##
|
162
|
+
# Creates a query string from data.
|
163
|
+
|
164
|
+
def self.build_query data, param=nil
|
165
|
+
raise ArgumentError,
|
166
|
+
"Can't convert #{data.class} to query without a param name" unless
|
167
|
+
Hash === data || param
|
168
|
+
|
169
|
+
case data
|
170
|
+
when Array
|
171
|
+
out = data.map do |value|
|
172
|
+
key = "#{param}[]"
|
173
|
+
build_query value, key
|
174
|
+
end
|
175
|
+
|
176
|
+
out.join "&"
|
177
|
+
|
178
|
+
when Hash
|
179
|
+
out = data.sort.map do |key, value|
|
180
|
+
key = param.nil? ? key : "#{param}[#{key}]"
|
181
|
+
build_query value, key
|
182
|
+
end
|
183
|
+
|
184
|
+
out.join "&"
|
185
|
+
|
186
|
+
else
|
187
|
+
"#{param}=#{data}"
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
end
|
@@ -0,0 +1,226 @@
|
|
1
|
+
class Kronk
|
2
|
+
|
3
|
+
##
|
4
|
+
# Wrapper to add a few niceties to the Net::HTTPResponse class.
|
5
|
+
|
6
|
+
class Response < Net::HTTPResponse
|
7
|
+
|
8
|
+
class MissingParser < Exception; end
|
9
|
+
|
10
|
+
##
|
11
|
+
# Create a new Response instance from an IO object.
|
12
|
+
|
13
|
+
def self.read_new io
|
14
|
+
io = Net::BufferedIO.new io unless Net::BufferedIO === io
|
15
|
+
io.debug_output = socket_io = StringIO.new
|
16
|
+
|
17
|
+
begin
|
18
|
+
resp = super io
|
19
|
+
resp.reading_body io, true do;end
|
20
|
+
rescue EOFError
|
21
|
+
end
|
22
|
+
|
23
|
+
resp.extend Helpers
|
24
|
+
|
25
|
+
r_req, r_resp, r_bytes = read_raw_from socket_io
|
26
|
+
resp.instance_variable_set "@raw", r_resp
|
27
|
+
resp.instance_variable_set "@read", true
|
28
|
+
resp.instance_variable_set "@socket", true
|
29
|
+
|
30
|
+
resp.instance_variable_set "@body", resp.raw.split("\r\n\r\n",2)[1] if
|
31
|
+
!resp.body
|
32
|
+
|
33
|
+
resp
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
##
|
38
|
+
# Read the raw response from a debug_output instance and return an array
|
39
|
+
# containing the raw request, response, and number of bytes received.
|
40
|
+
|
41
|
+
def self.read_raw_from debug_io
|
42
|
+
req = nil
|
43
|
+
resp = ""
|
44
|
+
bytes = nil
|
45
|
+
|
46
|
+
debug_io.rewind
|
47
|
+
output = debug_io.read.split "\n"
|
48
|
+
|
49
|
+
if output.first =~ %r{<-\s(.*)}
|
50
|
+
req = instance_eval $1
|
51
|
+
output.delete_at 0
|
52
|
+
end
|
53
|
+
|
54
|
+
if output.last =~ %r{read (\d+) bytes}
|
55
|
+
bytes = $1.to_i
|
56
|
+
output.delete_at(-1)
|
57
|
+
end
|
58
|
+
|
59
|
+
output.map do |line|
|
60
|
+
next unless line[0..2] == "-> "
|
61
|
+
resp << instance_eval(line[2..-1])
|
62
|
+
end
|
63
|
+
|
64
|
+
[req, resp, bytes]
|
65
|
+
end
|
66
|
+
|
67
|
+
|
68
|
+
##
|
69
|
+
# Helper methods for Net::HTTPResponse objects.
|
70
|
+
|
71
|
+
module Helpers
|
72
|
+
|
73
|
+
##
|
74
|
+
# Returns the raw http response.
|
75
|
+
|
76
|
+
def raw
|
77
|
+
@raw
|
78
|
+
end
|
79
|
+
|
80
|
+
|
81
|
+
##
|
82
|
+
# Returns the body data parsed according to the content type.
|
83
|
+
# If no parser is given will look for the default parser based on
|
84
|
+
# the Content-Type, or will return the cached parsed body if available.
|
85
|
+
|
86
|
+
def parsed_body parser=nil
|
87
|
+
return @parsed_body if @parsed_body && !parser
|
88
|
+
parser ||= Kronk.parser_for self['Content-Type']
|
89
|
+
|
90
|
+
raise MissingParser,
|
91
|
+
"No parser for Content-Type: #{self['Content-Type']}" unless parser
|
92
|
+
|
93
|
+
@parsed_body = parser.parse self.body
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
##
|
98
|
+
# Returns the parsed header hash.
|
99
|
+
|
100
|
+
def parsed_header include_headers=true
|
101
|
+
headers = self.to_hash.dup
|
102
|
+
|
103
|
+
case include_headers
|
104
|
+
when nil, false
|
105
|
+
nil
|
106
|
+
|
107
|
+
when Array, String
|
108
|
+
include_headers = [*include_headers].map{|h| h.to_s.downcase}
|
109
|
+
|
110
|
+
headers.each do |key, value|
|
111
|
+
headers.delete key unless
|
112
|
+
include_headers.include? key.to_s.downcase
|
113
|
+
end
|
114
|
+
|
115
|
+
headers
|
116
|
+
|
117
|
+
when true
|
118
|
+
headers
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
##
|
124
|
+
# Returns the header portion of the raw http response.
|
125
|
+
|
126
|
+
def raw_header include_headers=true
|
127
|
+
headers = "#{raw.split("\r\n\r\n", 2)[0]}\r\n"
|
128
|
+
|
129
|
+
case include_headers
|
130
|
+
when nil, false
|
131
|
+
nil
|
132
|
+
|
133
|
+
when Array, String
|
134
|
+
includes = [*include_headers].join("|")
|
135
|
+
headers.scan(%r{^((?:#{includes}): [^\n]*\n)}im).flatten.join
|
136
|
+
|
137
|
+
when true
|
138
|
+
headers
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
##
|
144
|
+
# Returns the raw response with selective headers and/or the body of
|
145
|
+
# the response. Supports the following options:
|
146
|
+
# :no_body:: Bool - Don't return the body; default nil
|
147
|
+
# :with_headers:: Bool/String/Array - Return headers; default nil
|
148
|
+
|
149
|
+
def selective_string options={}
|
150
|
+
str = self.body unless options[:no_body]
|
151
|
+
|
152
|
+
if options[:with_headers]
|
153
|
+
header = raw_header(options[:with_headers])
|
154
|
+
str = [header, str].compact.join "\r\n"
|
155
|
+
end
|
156
|
+
|
157
|
+
str
|
158
|
+
end
|
159
|
+
|
160
|
+
|
161
|
+
##
|
162
|
+
# Returns the parsed response with selective headers and/or the body of
|
163
|
+
# the response. Supports the following options:
|
164
|
+
# :no_body:: Bool - Don't return the body; default nil
|
165
|
+
# :with_headers:: Bool/String/Array - Return headers; default nil
|
166
|
+
# :ignore_data:: String/Array - Removes the data from given data paths
|
167
|
+
# :only_data:: String/Array - Extracts the data from given data paths
|
168
|
+
|
169
|
+
def selective_data options={}
|
170
|
+
data = nil
|
171
|
+
|
172
|
+
unless options[:no_body]
|
173
|
+
ds = DataSet.new parsed_body
|
174
|
+
|
175
|
+
ds.collect_data_points options[:only_data] if options[:only_data]
|
176
|
+
ds.delete_data_points options[:ignore_data] if options[:ignore_data]
|
177
|
+
|
178
|
+
data = ds.data
|
179
|
+
end
|
180
|
+
|
181
|
+
if options[:with_headers]
|
182
|
+
data = [parsed_header(options[:with_headers]), data].compact
|
183
|
+
end
|
184
|
+
|
185
|
+
data
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
|
191
|
+
##
|
192
|
+
# Mock response object without a header for body-only http responses.
|
193
|
+
|
194
|
+
class HeadlessResponse
|
195
|
+
|
196
|
+
include Response::Helpers
|
197
|
+
|
198
|
+
attr_accessor :body, :code
|
199
|
+
|
200
|
+
def initialize body
|
201
|
+
@body = body
|
202
|
+
@raw = body
|
203
|
+
@header = {}
|
204
|
+
end
|
205
|
+
|
206
|
+
|
207
|
+
##
|
208
|
+
# Interface method only. Returns nil for all but content type.
|
209
|
+
|
210
|
+
def [] key
|
211
|
+
@header[key]
|
212
|
+
end
|
213
|
+
|
214
|
+
def []= key, value
|
215
|
+
@header[key] = value
|
216
|
+
end
|
217
|
+
|
218
|
+
|
219
|
+
##
|
220
|
+
# Interface method only. Returns empty hash.
|
221
|
+
|
222
|
+
def to_hash
|
223
|
+
Hash.new
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
class Kronk
|
2
|
+
|
3
|
+
##
|
4
|
+
# Wrapper class for Nokogiri parser.
|
5
|
+
|
6
|
+
class XMLParser
|
7
|
+
|
8
|
+
##
|
9
|
+
# Takes an xml string and returns a data hash.
|
10
|
+
# Ignores blank spaces between tags.
|
11
|
+
|
12
|
+
def self.parse str
|
13
|
+
root_node = Nokogiri.XML str do |config|
|
14
|
+
config.default_xml.noblanks
|
15
|
+
end
|
16
|
+
|
17
|
+
hash = node_value root_node.children
|
18
|
+
hash.values.first
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
##
|
23
|
+
# Build a hash from a nokogiri xml node.
|
24
|
+
|
25
|
+
def self.node_value xml_node, as_array=false
|
26
|
+
case xml_node
|
27
|
+
when Nokogiri::XML::Text then xml_node.text
|
28
|
+
|
29
|
+
# Returns hash or array
|
30
|
+
when Nokogiri::XML::NodeSet then node_set_value(xml_node, as_array)
|
31
|
+
|
32
|
+
# Returns node name and value
|
33
|
+
when Nokogiri::XML::Element then element_node_value(xml_node)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
##
|
39
|
+
# Returns the value for an xml node set.
|
40
|
+
# Can be a Hash, Array, or String
|
41
|
+
|
42
|
+
def self.node_set_value xml_node, as_array=false
|
43
|
+
orig = {}
|
44
|
+
data = as_array ? Array.new : Hash.new
|
45
|
+
|
46
|
+
xml_node.each do |node|
|
47
|
+
node_data, name = node_value node
|
48
|
+
return node_data unless name
|
49
|
+
|
50
|
+
case data
|
51
|
+
when Array
|
52
|
+
data << node_data
|
53
|
+
when Hash
|
54
|
+
orig[name] ||= node_data
|
55
|
+
|
56
|
+
if data.has_key?(name)
|
57
|
+
data[name] = [data[name]] if data[name] == orig[name]
|
58
|
+
data[name] << node_data
|
59
|
+
else
|
60
|
+
data[name] = node_data
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
data
|
66
|
+
end
|
67
|
+
|
68
|
+
|
69
|
+
##
|
70
|
+
# Returns an Array containing the value of an element node
|
71
|
+
# and its name.
|
72
|
+
|
73
|
+
def self.element_node_value xml_node
|
74
|
+
name = xml_node.name
|
75
|
+
datatype = xml_node.attr :type
|
76
|
+
is_array = array? xml_node.children, name
|
77
|
+
data = node_value xml_node.children, is_array
|
78
|
+
|
79
|
+
data = case datatype
|
80
|
+
when 'array' then data.to_a
|
81
|
+
when 'symbol' then data.to_sym
|
82
|
+
when 'integer' then data.to_i
|
83
|
+
when 'float' then data.to_f
|
84
|
+
when 'boolean'
|
85
|
+
data == 'true' ? true : false
|
86
|
+
else
|
87
|
+
data
|
88
|
+
end
|
89
|
+
|
90
|
+
[data, name]
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
##
|
95
|
+
# Checks if a given node set should be interpreted as an Array.
|
96
|
+
|
97
|
+
def self.array? node_set, parent_name=nil
|
98
|
+
names = node_set.map do |n|
|
99
|
+
return unless Nokogiri::XML::Element === n
|
100
|
+
n.name
|
101
|
+
end
|
102
|
+
|
103
|
+
names.uniq.length == 1 && (names.length > 1 ||
|
104
|
+
parent_name && (names.first == parent_name ||
|
105
|
+
names.first.pluralize == parent_name))
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|