mocrata 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +3 -0
- data/README.md +25 -4
- data/Rakefile +5 -0
- data/lib/mocrata.rb +2 -1
- data/lib/mocrata/dataset.rb +72 -36
- data/lib/mocrata/dataset_url.rb +26 -10
- data/lib/mocrata/request.rb +56 -15
- data/lib/mocrata/response.rb +30 -4
- data/lib/mocrata/version.rb +1 -1
- data/mocrata.gemspec +0 -1
- data/spec/lib/mocrata/dataset_spec.rb +50 -2
- data/spec/lib/mocrata/dataset_url_spec.rb +2 -1
- data/spec/lib/mocrata/request_spec.rb +86 -10
- data/spec/lib/mocrata/response_spec.rb +51 -4
- metadata +3 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc529931b56e0de47b1a4ade8a83c05e4dc31b7e
|
4
|
+
data.tar.gz: c28c9d3d688b4129694fbf20c96d52168b1889e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2a8e8142718c0b96126f2975e7108463667ce9a679b65fdab7ed1fe55c0c6f9915e5c53e088a5e1aab085c3f80cd3d2bda59c0819640e1f251b5a71c58093d9a
|
7
|
+
data.tar.gz: 2787a4131eabc7ee3fc12a612d03b4228fc2623a092f4d976bbbd971e2dee59ba6720f12659bc34d35ac064583c2fbbd51e31ae4551f5f10042b46dff10b6e10
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Mocrata
|
2
2
|
|
3
|
+
[![Build Status](https://travis-ci.org/mode/mocrata.svg?branch=master)](https://travis-ci.org/mode/mocrata)
|
4
|
+
[![Code Climate](https://codeclimate.com/repos/53d16a75695680764e01ea68/badges/c93756788e4438e90512/gpa.png)](https://codeclimate.com/repos/53d16a75695680764e01ea68/feed)
|
5
|
+
[![Gem Version](https://badge.fury.io/rb/mocrata.svg)](http://badge.fury.io/rb/mocrata)
|
6
|
+
|
3
7
|
Mocrata is a [SODA](http://dev.socrata.com/) (Socrata Open Data API) client
|
4
8
|
developed by [Mode Analytics](https://modeanalytics.com).
|
5
9
|
|
@@ -30,16 +34,29 @@ end
|
|
30
34
|
### Accessing data
|
31
35
|
|
32
36
|
```
|
33
|
-
dataset = Mocrata::Dataset.new(
|
37
|
+
dataset = Mocrata::Dataset.new("http://opendata.socrata.com/resource/mnkm-8ram")
|
38
|
+
|
39
|
+
dataset.name
|
40
|
+
=> "Country List ISO 3166 Codes Latitude Longitude"
|
41
|
+
|
42
|
+
dataset.csv_header
|
43
|
+
=> ["Country", "Alpha code", "Numeric code", "Latitude", "Longitude"]
|
34
44
|
|
35
45
|
dataset.csv
|
36
|
-
=> [["
|
46
|
+
=> [["Albania", "AL", "8", "41", "20"],
|
47
|
+
["Algeria", "DZ", "12", "28", "3"], ...]
|
37
48
|
|
38
49
|
dataset.json
|
39
|
-
=> [{"
|
50
|
+
=> [{"longitude_average"=>"20",
|
51
|
+
"latitude_average"=>"41",
|
52
|
+
"alpha_2_code"=>"AL",
|
53
|
+
"numeric_code"=>"8",
|
54
|
+
"country"=>"Albania"}, ...]
|
40
55
|
|
41
56
|
dataset.fields
|
42
|
-
=> {"
|
57
|
+
=> {":created_at"=>"meta_data", ":id"=>"meta_data", ":updated_at"=>"meta_data",
|
58
|
+
"alpha_2_code"=>"text", "country"=>"text", "latitude_average"=>"number",
|
59
|
+
"longitude_average"=>"number", "numeric_code"=>"number"}
|
43
60
|
```
|
44
61
|
|
45
62
|
### Iterating through rows
|
@@ -52,6 +69,10 @@ end
|
|
52
69
|
dataset.each_row(:json) { |row| ... }
|
53
70
|
```
|
54
71
|
|
72
|
+
## Documentation
|
73
|
+
|
74
|
+
http://rubydoc.info/github/mode/mocrata/master/frames
|
75
|
+
|
55
76
|
## Contributing
|
56
77
|
|
57
78
|
1. Fork it
|
data/Rakefile
CHANGED
data/lib/mocrata.rb
CHANGED
data/lib/mocrata/dataset.rb
CHANGED
@@ -7,20 +7,21 @@ module Mocrata
|
|
7
7
|
class Dataset
|
8
8
|
# Construct a new Dataset instance
|
9
9
|
#
|
10
|
-
# @param
|
10
|
+
# @param original_url [String] valid {http://dev.socrata.com SODA} resource
|
11
|
+
# url
|
11
12
|
#
|
12
13
|
# @return [Mocrata::Dataset] the instance
|
13
14
|
#
|
14
15
|
# @example
|
15
16
|
# dataset = Mocrata::Dataset.new('http://data.sfgov.org/resource/funx-qxxn')
|
16
17
|
#
|
17
|
-
def initialize(
|
18
|
-
@
|
18
|
+
def initialize(original_url)
|
19
|
+
@original_url = original_url
|
19
20
|
end
|
20
21
|
|
21
22
|
# Iterate through each row of the dataset
|
22
23
|
#
|
23
|
-
# @param format [
|
24
|
+
# @param format [optional, Symbol] the format, `:json` or `:csv`
|
24
25
|
#
|
25
26
|
# @yield [Array<Array>] row of values
|
26
27
|
#
|
@@ -29,7 +30,7 @@ module Mocrata
|
|
29
30
|
# # do something with the row
|
30
31
|
# end
|
31
32
|
#
|
32
|
-
def each_row(format, &block)
|
33
|
+
def each_row(format = :json, &block)
|
33
34
|
each_page(format) do |page|
|
34
35
|
page.each(&block)
|
35
36
|
end
|
@@ -37,8 +38,12 @@ module Mocrata
|
|
37
38
|
|
38
39
|
# Iterate through each page of the dataset
|
39
40
|
#
|
40
|
-
# @param format [
|
41
|
-
# @param
|
41
|
+
# @param format [optional, Symbol] the format, `:json` or `:csv`
|
42
|
+
# @param options [optional, Hash] hash of options
|
43
|
+
#
|
44
|
+
# @option options [Integer] :per_page the number of rows to return for each
|
45
|
+
# page
|
46
|
+
# @option options [Integer] :page the first page
|
42
47
|
#
|
43
48
|
# @yield [Array<Array>] page of rows
|
44
49
|
#
|
@@ -47,56 +52,63 @@ module Mocrata
|
|
47
52
|
# # do something with the page
|
48
53
|
# end
|
49
54
|
#
|
50
|
-
def each_page(format,
|
51
|
-
page
|
52
|
-
per_page
|
55
|
+
def each_page(format = :json, options = {}, &block)
|
56
|
+
page = options.fetch(:page, 1)
|
57
|
+
per_page = options.fetch(:per_page, Mocrata.config.per_page)
|
53
58
|
|
54
59
|
while true
|
55
|
-
rows =
|
60
|
+
rows = get(format, :page => page, :per_page => per_page).body
|
56
61
|
yield rows
|
57
62
|
break if rows.size < per_page
|
58
63
|
page += 1
|
59
64
|
end
|
60
65
|
end
|
61
66
|
|
62
|
-
#
|
63
|
-
#
|
64
|
-
# @param params [optional, Hash] hash of options to pass along to the HTTP request
|
65
|
-
#
|
66
|
-
# @option params [Integer] :page the page to request
|
67
|
-
# @option params [Integer] :per_page the number of rows to return for each page
|
67
|
+
# All rows in the dataset
|
68
68
|
#
|
69
|
-
# @
|
69
|
+
# @param format [optional, Symbol] the format, `:json` or `:csv`
|
70
70
|
#
|
71
|
-
# @
|
72
|
-
# dataset.csv(:page => 2, :per_page => 10)
|
71
|
+
# @return [Array] all rows in the requested format
|
73
72
|
#
|
74
|
-
def
|
75
|
-
|
73
|
+
def rows(format = :json)
|
74
|
+
rows = []
|
75
|
+
each_page(format) { |page| rows += page }
|
76
|
+
rows
|
76
77
|
end
|
77
78
|
|
78
79
|
# The contents of the dataset in JSON format
|
79
80
|
#
|
80
|
-
# @
|
81
|
+
# @return [Array<Hash>] the array of rows
|
81
82
|
#
|
82
|
-
|
83
|
-
|
83
|
+
def json
|
84
|
+
rows(:json)
|
85
|
+
end
|
86
|
+
|
87
|
+
# The contents of the dataset in CSV format
|
84
88
|
#
|
85
|
-
# @return [Array<
|
89
|
+
# @return [Array<Array>] the array of rows
|
86
90
|
#
|
87
|
-
|
88
|
-
|
91
|
+
def csv
|
92
|
+
rows(:csv)
|
93
|
+
end
|
94
|
+
|
95
|
+
# The parsed header of the dataset in CSV format
|
89
96
|
#
|
90
|
-
|
91
|
-
|
97
|
+
# @return [Array<String>] the array of headers
|
98
|
+
#
|
99
|
+
def csv_header
|
100
|
+
options = { :paginate => false, :preserve_header => true }
|
101
|
+
params = { :limit => 0 }
|
102
|
+
|
103
|
+
Mocrata::Request.new(resource_url, :csv, options, params).response.body[0]
|
92
104
|
end
|
93
105
|
|
94
|
-
# Get the headers associated with the dataset
|
106
|
+
# Get the HTTP headers associated with the dataset (SODA doesn't support
|
107
|
+
# HEAD requests)
|
95
108
|
#
|
96
109
|
# @return [Hash] a hash of headers
|
97
110
|
#
|
98
111
|
def headers
|
99
|
-
# SODA doesn't support HEAD requests, unfortunately
|
100
112
|
@headers ||= get(:json, :per_page => 0).headers
|
101
113
|
end
|
102
114
|
|
@@ -108,16 +120,40 @@ module Mocrata
|
|
108
120
|
Hash[field_names.zip(field_types)]
|
109
121
|
end
|
110
122
|
|
123
|
+
# A parsed representation of the dataset's {http://www.odata.org OData}
|
124
|
+
#
|
125
|
+
# @return [REXML::Document] a parsed REXML document
|
126
|
+
#
|
127
|
+
def odata
|
128
|
+
@odata ||= Mocrata::Request.new(odata_url, :xml, :top => 0).response.body
|
129
|
+
end
|
130
|
+
|
131
|
+
# The name of the dataset from OData
|
132
|
+
#
|
133
|
+
# @return [String] the dataset name
|
134
|
+
#
|
135
|
+
def name
|
136
|
+
odata.root.elements['title'].text
|
137
|
+
end
|
138
|
+
|
111
139
|
private
|
112
140
|
|
113
|
-
attr_reader :
|
141
|
+
attr_reader :original_url
|
114
142
|
|
115
143
|
def get(format, params = {})
|
116
|
-
Mocrata::Request.new(
|
144
|
+
Mocrata::Request.new(resource_url, format, params).response
|
145
|
+
end
|
146
|
+
|
147
|
+
def url
|
148
|
+
@url ||= Mocrata::DatasetUrl.new(original_url)
|
149
|
+
end
|
150
|
+
|
151
|
+
def resource_url
|
152
|
+
url.to_resource
|
117
153
|
end
|
118
154
|
|
119
|
-
def
|
120
|
-
|
155
|
+
def odata_url
|
156
|
+
url.to_odata
|
121
157
|
end
|
122
158
|
|
123
159
|
def field_names
|
data/lib/mocrata/dataset_url.rb
CHANGED
@@ -19,19 +19,20 @@ module Mocrata
|
|
19
19
|
@original = original
|
20
20
|
end
|
21
21
|
|
22
|
-
#
|
23
|
-
# string and fragment, if any.
|
22
|
+
# Convert the original URL to a normalized resource URL
|
24
23
|
#
|
25
|
-
# @return [String] the
|
24
|
+
# @return [String] the resource URL
|
26
25
|
#
|
27
|
-
def
|
28
|
-
|
29
|
-
|
30
|
-
uri.scheme = 'https'
|
31
|
-
uri.fragment = nil
|
32
|
-
uri.query = nil
|
26
|
+
def to_resource
|
27
|
+
@to_resource ||= normalize
|
28
|
+
end
|
33
29
|
|
34
|
-
|
30
|
+
# Convert the original URL to a normalized OData URL
|
31
|
+
#
|
32
|
+
# @return [String] the OData URL
|
33
|
+
#
|
34
|
+
def to_odata
|
35
|
+
@to_odata ||= normalize.gsub(/\/resource\//, '/OData.svc/')
|
35
36
|
end
|
36
37
|
|
37
38
|
# Validate the original URL against the expected Socrata dataset URL
|
@@ -79,6 +80,21 @@ module Mocrata
|
|
79
80
|
|
80
81
|
VALID_PATTERN = /\/resource\//
|
81
82
|
|
83
|
+
# Normalize a Socrata dataset URL. Ensures https protocol. Removes query
|
84
|
+
# string and fragment, if any.
|
85
|
+
#
|
86
|
+
# @return [String] the normalized URL
|
87
|
+
#
|
88
|
+
def normalize
|
89
|
+
uri = URI(self.class.ensure_protocol(original))
|
90
|
+
|
91
|
+
uri.scheme = 'https'
|
92
|
+
uri.fragment = nil
|
93
|
+
uri.query = nil
|
94
|
+
|
95
|
+
self.class.strip_format(uri.to_s)
|
96
|
+
end
|
97
|
+
|
82
98
|
class InvalidError < StandardError; end
|
83
99
|
end
|
84
100
|
end
|
data/lib/mocrata/request.rb
CHANGED
@@ -5,26 +5,38 @@ require 'csv'
|
|
5
5
|
require 'json'
|
6
6
|
require 'net/https'
|
7
7
|
|
8
|
+
require 'mocrata/version'
|
9
|
+
|
8
10
|
module Mocrata
|
9
11
|
# @attr_reader [String] url the request URL
|
10
12
|
# @attr_reader [Symbol] format the request format, `:json` or `:csv`
|
11
|
-
# @attr_reader [Hash]
|
13
|
+
# @attr_reader [Hash] options hash of options
|
14
|
+
# @attr_reader [Hash] params the request params
|
12
15
|
#
|
13
16
|
class Request
|
14
|
-
attr_reader :url, :format, :params
|
17
|
+
attr_reader :url, :format, :options, :params
|
15
18
|
|
16
19
|
# Construct a new Request instance
|
17
20
|
#
|
18
21
|
# @param url [String] the request URL
|
19
22
|
# @param format [Symbol] the request format, `:json` or `:csv`
|
20
|
-
# @param
|
23
|
+
# @param options [optional, Hash] hash of options
|
24
|
+
# @param params [optional, Hash] the request params
|
25
|
+
#
|
26
|
+
# @option options [Integer] :page the page to request
|
27
|
+
# @option options [Integer] :per_page the number of rows to return for each
|
28
|
+
# page
|
29
|
+
# @option options [true, false] :paginate whether to add pagination params
|
30
|
+
# @option options [true, false] :preserve_header whether to preserve CSV
|
31
|
+
# header
|
21
32
|
#
|
22
33
|
# @return [Mocrata::Request] the instance
|
23
34
|
#
|
24
|
-
def initialize(url, format, params = {})
|
25
|
-
@url
|
26
|
-
@format
|
27
|
-
@
|
35
|
+
def initialize(url, format, options = {}, params = {})
|
36
|
+
@url = url
|
37
|
+
@format = format
|
38
|
+
@options = options
|
39
|
+
@params = params
|
28
40
|
end
|
29
41
|
|
30
42
|
# Perform the HTTP GET request
|
@@ -34,12 +46,14 @@ module Mocrata
|
|
34
46
|
def response
|
35
47
|
request = Net::HTTP::Get.new(uri.request_uri)
|
36
48
|
|
37
|
-
request
|
38
|
-
request
|
49
|
+
request['accept'] = content_type
|
50
|
+
request['user-agent'] = USER_AGENT
|
51
|
+
|
52
|
+
request.add_field('x-app-token', Mocrata.config.app_token)
|
39
53
|
|
40
54
|
response = http.request(request)
|
41
55
|
|
42
|
-
Mocrata::Response.new(response).tap(&:validate!)
|
56
|
+
Mocrata::Response.new(response, response_options).tap(&:validate!)
|
43
57
|
end
|
44
58
|
|
45
59
|
# @return [String] the content type for the specified format
|
@@ -54,6 +68,9 @@ module Mocrata
|
|
54
68
|
|
55
69
|
private
|
56
70
|
|
71
|
+
USER_AGENT = "mocrata/#{Mocrata::VERSION}"
|
72
|
+
SODA_PARAM_KEYS = %w(top limit)
|
73
|
+
|
57
74
|
def http
|
58
75
|
@http ||= Net::HTTP.new(uri.host, uri.port).tap do |http|
|
59
76
|
http.use_ssl = true
|
@@ -63,16 +80,40 @@ module Mocrata
|
|
63
80
|
|
64
81
|
def soda_params
|
65
82
|
@soda_params ||= {}.tap do |soda|
|
66
|
-
|
67
|
-
|
83
|
+
soda.merge!(pagination_params) if paginate?
|
84
|
+
|
85
|
+
SODA_PARAM_KEYS.each do |key|
|
86
|
+
if params.has_key?(key.to_sym)
|
87
|
+
soda[:"$#{key}"] = params.fetch(key.to_sym)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def pagination_params
|
94
|
+
{}.tap do |result|
|
95
|
+
limit = options.fetch(:per_page, Mocrata.config.per_page)
|
96
|
+
page = options.fetch(:page, 1)
|
97
|
+
|
98
|
+
result[:$limit] = limit
|
99
|
+
result[:$offset] = (page - 1) * limit
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def paginate?
|
104
|
+
options.fetch(:paginate, false) ||
|
105
|
+
options.has_key?(:page) ||
|
106
|
+
options.has_key?(:per_page)
|
107
|
+
end
|
68
108
|
|
69
|
-
|
70
|
-
|
109
|
+
def response_options
|
110
|
+
options.keep_if do |key, value|
|
111
|
+
Mocrata::Response::OPTIONS.include?(key)
|
71
112
|
end
|
72
113
|
end
|
73
114
|
|
74
115
|
def uri
|
75
|
-
@uri ||= URI(url).
|
116
|
+
@uri ||= URI(url).tap do |uri|
|
76
117
|
uri.query = self.class.query_string(soda_params)
|
77
118
|
end
|
78
119
|
end
|
data/lib/mocrata/response.rb
CHANGED
@@ -1,18 +1,26 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
#
|
3
|
-
require 'json'
|
4
3
|
require 'csv'
|
4
|
+
require 'json'
|
5
|
+
require 'rexml/document'
|
5
6
|
|
6
7
|
module Mocrata
|
7
8
|
class Response
|
9
|
+
OPTIONS = [:preserve_header]
|
10
|
+
|
8
11
|
# Construct a new Response instance
|
9
12
|
#
|
10
13
|
# @param http_response [Net::HTTPResponse] the http response
|
14
|
+
# @param options [Hash] hash of options
|
15
|
+
#
|
16
|
+
# @option options [true, false] :preserve_header whether to preserve CSV
|
17
|
+
# header
|
11
18
|
#
|
12
19
|
# @return [Mocrata::Response] the instance
|
13
20
|
#
|
14
|
-
def initialize(http_response)
|
21
|
+
def initialize(http_response, options = {})
|
15
22
|
@http_response = http_response
|
23
|
+
@options = options
|
16
24
|
end
|
17
25
|
|
18
26
|
# Perform certain checks against the HTTP response and raise an exception
|
@@ -29,6 +37,10 @@ module Mocrata
|
|
29
37
|
end
|
30
38
|
end
|
31
39
|
|
40
|
+
unless code == 200
|
41
|
+
raise ResponseError.new("Unexpected response code: #{code}")
|
42
|
+
end
|
43
|
+
|
32
44
|
true
|
33
45
|
end
|
34
46
|
|
@@ -46,6 +58,10 @@ module Mocrata
|
|
46
58
|
end
|
47
59
|
end
|
48
60
|
|
61
|
+
def code
|
62
|
+
http_response.code.to_i
|
63
|
+
end
|
64
|
+
|
49
65
|
# The HTTP response body, processed according to content type
|
50
66
|
#
|
51
67
|
# @return [Array] the parsed body
|
@@ -59,7 +75,7 @@ module Mocrata
|
|
59
75
|
# SODA headers that are always encoded as JSON
|
60
76
|
JSON_HEADERS = %w(x-soda2-fields x-soda2-types)
|
61
77
|
|
62
|
-
attr_reader :http_response
|
78
|
+
attr_reader :http_response, :options
|
63
79
|
|
64
80
|
def content_type
|
65
81
|
type = headers['content-type']
|
@@ -72,13 +88,23 @@ module Mocrata
|
|
72
88
|
end
|
73
89
|
|
74
90
|
def csv
|
75
|
-
CSV.parse(http_response.body)
|
91
|
+
result = CSV.parse(http_response.body)
|
92
|
+
result = result[1..-1] unless preserve_header?
|
93
|
+
result
|
76
94
|
end
|
77
95
|
|
78
96
|
def json
|
79
97
|
JSON.parse(http_response.body)
|
80
98
|
end
|
81
99
|
|
100
|
+
def xml
|
101
|
+
REXML::Document.new(http_response.body)
|
102
|
+
end
|
103
|
+
|
104
|
+
def preserve_header?
|
105
|
+
options.fetch(:preserve_header, false)
|
106
|
+
end
|
107
|
+
|
82
108
|
class ResponseError < StandardError; end
|
83
109
|
end
|
84
110
|
end
|
data/lib/mocrata/version.rb
CHANGED
data/mocrata.gemspec
CHANGED
@@ -31,10 +31,12 @@ describe Mocrata::Dataset do
|
|
31
31
|
|
32
32
|
describe '#each_page' do
|
33
33
|
it 'yields pages' do
|
34
|
-
|
34
|
+
response = double(:response)
|
35
|
+
expect(response).to receive(:body).and_return(*pages)
|
36
|
+
expect(dataset).to receive(:get).and_return(response).at_least(:once)
|
35
37
|
|
36
38
|
expect { |b|
|
37
|
-
dataset.each_page(:json, 4, &b)
|
39
|
+
dataset.each_page(:json, :per_page => 4, &b)
|
38
40
|
}.to yield_successive_args(*pages)
|
39
41
|
end
|
40
42
|
end
|
@@ -53,6 +55,38 @@ describe Mocrata::Dataset do
|
|
53
55
|
end
|
54
56
|
end
|
55
57
|
|
58
|
+
describe '#name' do
|
59
|
+
it 'fetches name from odata' do
|
60
|
+
dataset = Mocrata::Dataset.new(
|
61
|
+
'https://data.sfgov.org/resource/dataset-identifier')
|
62
|
+
|
63
|
+
xml = %{<feed>
|
64
|
+
<title type="text">Test name</title>
|
65
|
+
<id>http://opendata.socrata.com/OData.svc/dataset-identifier</id>
|
66
|
+
<updated>2012-06-15T18:15:19Z</updated>
|
67
|
+
</feed>}
|
68
|
+
|
69
|
+
expect(dataset).to receive(:odata).and_return(REXML::Document.new(xml))
|
70
|
+
expect(dataset.name).to eq('Test name')
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
describe '#odata' do
|
75
|
+
it 'returns odata xml document' do
|
76
|
+
dataset = Mocrata::Dataset.new(
|
77
|
+
'https://data.sfgov.org/resource/funx-qxxn')
|
78
|
+
|
79
|
+
response = Mocrata::Response.new(true)
|
80
|
+
expect(response).to receive(:content_type).and_return(:xml)
|
81
|
+
expect(response).to receive(:http_response).and_return(
|
82
|
+
double(:http_response, :body => ''))
|
83
|
+
expect_any_instance_of(Mocrata::Request).to receive(
|
84
|
+
:response).and_return(response)
|
85
|
+
|
86
|
+
expect(dataset.odata).to be_an_instance_of(REXML::Document)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
56
90
|
describe '#csv' do
|
57
91
|
it 'returns csv body' do
|
58
92
|
response = Mocrata::Response.new(true)
|
@@ -64,6 +98,20 @@ describe Mocrata::Dataset do
|
|
64
98
|
end
|
65
99
|
end
|
66
100
|
|
101
|
+
describe '#csv_header' do
|
102
|
+
it 'returns csv header' do
|
103
|
+
dataset = Mocrata::Dataset.new(
|
104
|
+
'https://data.sfgov.org/resource/funx-qxxn')
|
105
|
+
|
106
|
+
response = double(:response, :body => [['foo', 'bar']])
|
107
|
+
|
108
|
+
expect_any_instance_of(Mocrata::Request).to receive(
|
109
|
+
:response).and_return(response)
|
110
|
+
|
111
|
+
expect(dataset.csv_header).to eq(['foo', 'bar'])
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
67
115
|
describe '#json' do
|
68
116
|
it 'returns json body' do
|
69
117
|
response = Mocrata::Response.new(true)
|
@@ -25,7 +25,8 @@ describe Mocrata::DatasetUrl do
|
|
25
25
|
url = Mocrata::DatasetUrl.new(
|
26
26
|
'data.sfgov.org/resource/funx-qxxn.csv?limit=100#foo')
|
27
27
|
|
28
|
-
expect(url.normalize).to eq(
|
28
|
+
expect(url.send(:normalize)).to eq(
|
29
|
+
'https://data.sfgov.org/resource/funx-qxxn')
|
29
30
|
end
|
30
31
|
end
|
31
32
|
|
@@ -54,22 +54,97 @@ describe Mocrata::Request do
|
|
54
54
|
end
|
55
55
|
|
56
56
|
describe '#soda_params' do
|
57
|
-
|
57
|
+
describe 'with pagination' do
|
58
|
+
it 'has default params' do
|
59
|
+
request = Mocrata::Request.new('', nil, :paginate => true)
|
60
|
+
|
61
|
+
result = request.send(:soda_params)
|
62
|
+
|
63
|
+
expect(result).to eq(:$limit => 1000, :$offset => 0)
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'has custom params' do
|
67
|
+
request = Mocrata::Request.new('', nil, :paginate => true, :page => 2)
|
68
|
+
|
69
|
+
result = request.send(:soda_params)
|
70
|
+
|
71
|
+
expect(result).to eq(:$limit => 1000, :$offset => 1000)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
describe 'without pagination' do
|
76
|
+
it 'is empty by default' do
|
77
|
+
request = Mocrata::Request.new('', nil)
|
78
|
+
|
79
|
+
expect(request.send(:soda_params)).to eq({})
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'ignores unrecognized parameters' do
|
84
|
+
request = Mocrata::Request.new('', nil, {}, { :foo => 'bar', :top => 0 })
|
85
|
+
|
86
|
+
expect(request.send(:soda_params)).to eq(:$top => 0)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
describe '#pagination_params' do
|
91
|
+
it 'is formed with default pagination options' do
|
92
|
+
request = Mocrata::Request.new('', nil, :paginate => true)
|
93
|
+
|
94
|
+
result = request.send(:pagination_params)
|
95
|
+
|
96
|
+
expect(result).to eq(:$limit => 1000, :$offset => 0)
|
97
|
+
end
|
98
|
+
|
99
|
+
it 'is formed with custom pagination options' do
|
100
|
+
request = Mocrata::Request.new('', nil,
|
101
|
+
:page => 5, :per_page => 100)
|
102
|
+
|
103
|
+
result = request.send(:pagination_params)
|
104
|
+
|
105
|
+
expect(result).to eq(:$limit => 100, :$offset => 400)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
describe '#paginate?' do
|
110
|
+
it 'is false by default' do
|
58
111
|
request = Mocrata::Request.new('', nil)
|
59
112
|
|
60
|
-
expect(request.send(:
|
113
|
+
expect(request.send(:paginate?)).to eq(false)
|
61
114
|
end
|
62
115
|
|
63
|
-
it '
|
64
|
-
request = Mocrata::Request.new('', nil, :
|
116
|
+
it 'allows override' do
|
117
|
+
request = Mocrata::Request.new('', nil, :paginate => true)
|
65
118
|
|
66
|
-
expect(request.send(:
|
119
|
+
expect(request.send(:paginate?)).to eq(true)
|
67
120
|
end
|
68
121
|
|
69
|
-
it '
|
70
|
-
request = Mocrata::Request.new('', nil, :
|
122
|
+
it 'is true if page option is present' do
|
123
|
+
request = Mocrata::Request.new('', nil, :page => 1)
|
71
124
|
|
72
|
-
expect(request.send(:
|
125
|
+
expect(request.send(:paginate?)).to eq(true)
|
126
|
+
end
|
127
|
+
|
128
|
+
it 'is true if per_page option is present' do
|
129
|
+
request = Mocrata::Request.new('', nil, :per_page => 1)
|
130
|
+
|
131
|
+
expect(request.send(:paginate?)).to eq(true)
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
describe '#response_options' do
|
136
|
+
it 'is empty by default' do
|
137
|
+
request = Mocrata::Request.new('', nil)
|
138
|
+
|
139
|
+
expect(request.send(:response_options)).to eq({})
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'filters request options' do
|
143
|
+
request = Mocrata::Request.new('', nil,
|
144
|
+
:page => 1,
|
145
|
+
:preserve_header => true)
|
146
|
+
|
147
|
+
expect(request.send(:response_options)).to eq(:preserve_header => true)
|
73
148
|
end
|
74
149
|
end
|
75
150
|
|
@@ -80,7 +155,7 @@ describe Mocrata::Request do
|
|
80
155
|
|
81
156
|
it 'is formed with default parameters' do
|
82
157
|
request = Mocrata::Request.new(
|
83
|
-
'https://data.sfgov.org/resource/funx-qxxn', nil)
|
158
|
+
'https://data.sfgov.org/resource/funx-qxxn', nil, :paginate => true)
|
84
159
|
|
85
160
|
result = request.send(:uri).to_s
|
86
161
|
|
@@ -92,7 +167,8 @@ describe Mocrata::Request do
|
|
92
167
|
request = Mocrata::Request.new(
|
93
168
|
'https://data.sfgov.org/resource/funx-qxxn', nil,
|
94
169
|
:per_page => 5,
|
95
|
-
:page => 3
|
170
|
+
:page => 3,
|
171
|
+
:paginate => true)
|
96
172
|
|
97
173
|
result = request.send(:uri).to_s
|
98
174
|
|
@@ -9,27 +9,40 @@ describe Mocrata::Response do
|
|
9
9
|
|
10
10
|
describe '#validate!' do
|
11
11
|
it 'returns true without content type' do
|
12
|
+
expect(response).to receive(:code).and_return(200)
|
12
13
|
expect(response).to receive(:content_type).and_return(nil)
|
13
14
|
expect(response.validate!).to be true
|
14
15
|
end
|
15
16
|
|
16
17
|
it 'returns true with csv content' do
|
18
|
+
expect(response).to receive(:code).and_return(200)
|
17
19
|
expect(response).to receive(:content_type).and_return(:csv)
|
18
20
|
expect(response.validate!).to be true
|
19
21
|
end
|
20
22
|
|
21
23
|
it 'returns true with json array' do
|
24
|
+
expect(response).to receive(:code).and_return(200)
|
22
25
|
expect(response).to receive(:content_type).and_return(:json)
|
23
26
|
expect(response).to receive(:body).and_return([])
|
24
27
|
expect(response.validate!).to be true
|
25
28
|
end
|
26
29
|
|
27
30
|
it 'returns true with json array' do
|
31
|
+
expect(response).to receive(:code).and_return(200)
|
28
32
|
expect(response).to receive(:content_type).and_return(:json)
|
29
33
|
expect(response).to receive(:body).at_least(:once).and_return({})
|
30
34
|
expect(response.validate!).to be true
|
31
35
|
end
|
32
36
|
|
37
|
+
it 'raises exception if response code is unexpected' do
|
38
|
+
expect(response).to receive(:code).and_return(201).at_least(:once)
|
39
|
+
expect(response).to receive(:content_type).and_return(:json)
|
40
|
+
expect(response).to receive(:body).at_least(:once).and_return({})
|
41
|
+
|
42
|
+
expect { response.validate! }.to raise_error(
|
43
|
+
Mocrata::Response::ResponseError)
|
44
|
+
end
|
45
|
+
|
33
46
|
it 'raises exception with json error' do
|
34
47
|
expect(response).to receive(:content_type).and_return(:json)
|
35
48
|
expect(response).to receive(:body).at_least(:once).and_return(
|
@@ -79,10 +92,21 @@ describe Mocrata::Response do
|
|
79
92
|
end
|
80
93
|
|
81
94
|
describe '#csv' do
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
95
|
+
describe 'without header' do
|
96
|
+
it 'parses body and excludes header' do
|
97
|
+
csv = "\"header1\"\n\"row1\"\n\"row2\""
|
98
|
+
expect(response.send(:http_response)).to receive(:body).and_return(csv)
|
99
|
+
expect(response.send(:csv)).to eq([['row1'], ['row2']])
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
describe 'with header' do
|
104
|
+
it 'parses body and preserves header' do
|
105
|
+
expect(response).to receive(:preserve_header?).and_return(true)
|
106
|
+
csv = "\"header1\"\n\"row1\"\n\"row2\""
|
107
|
+
expect(response.send(:http_response)).to receive(:body).and_return(csv)
|
108
|
+
expect(response.send(:csv)).to eq([['header1'], ['row1'], ['row2']])
|
109
|
+
end
|
86
110
|
end
|
87
111
|
end
|
88
112
|
|
@@ -129,4 +153,27 @@ describe Mocrata::Response do
|
|
129
153
|
'x-soda2-fields' => { 'name' => 'value' })
|
130
154
|
end
|
131
155
|
end
|
156
|
+
|
157
|
+
describe '#code' do
|
158
|
+
it 'converts http response status to integer' do
|
159
|
+
http_response = double(:http_response, :code => '200')
|
160
|
+
response = Mocrata::Response.new(http_response)
|
161
|
+
|
162
|
+
expect(response.code).to eq(200)
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
describe '#preserve_header?' do
|
167
|
+
it 'is false by default' do
|
168
|
+
response = Mocrata::Response.new(true)
|
169
|
+
|
170
|
+
expect(response.send(:preserve_header?)).to eq(false)
|
171
|
+
end
|
172
|
+
|
173
|
+
it 'allows override' do
|
174
|
+
response = Mocrata::Response.new(true, :preserve_header => true)
|
175
|
+
|
176
|
+
expect(response.send(:preserve_header?)).to eq(true)
|
177
|
+
end
|
178
|
+
end
|
132
179
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mocrata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Heather Rivers
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-07
|
11
|
+
date: 2014-08-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,20 +66,6 @@ dependencies:
|
|
66
66
|
- - '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: rdoc
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - '>='
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :development
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - '>='
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
69
|
description: Mode's SODA client
|
84
70
|
email:
|
85
71
|
- heather@modeanalytics.com
|
@@ -88,6 +74,7 @@ extensions: []
|
|
88
74
|
extra_rdoc_files: []
|
89
75
|
files:
|
90
76
|
- .gitignore
|
77
|
+
- .travis.yml
|
91
78
|
- .yardopts
|
92
79
|
- Gemfile
|
93
80
|
- LICENSE.txt
|