datasift 3.1.3 → 3.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/CHANGELOG.md +7 -0
- data/LICENSE +1 -1
- data/README.md +4 -3
- data/VERSION +1 -1
- data/examples/live_stream_eg.rb +3 -3
- data/lib/api/api_resource.rb +12 -1
- data/lib/datasift.rb +87 -90
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bed62d86c7bd4459a24a217e685f36e4cd35e9a7
|
4
|
+
data.tar.gz: 1316874d66e385306c307d9f58a2907594e0fa54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ce03cf0ff67454e8805cd04598b773c654bb69b03927e0e6b4e6364502120da9590f442660489060e8d40b96615b52550f6ffa391095a07294851d757d178f0a
|
7
|
+
data.tar.gz: 760580e9ca38f757fd4f6c8e3d2e27147c8f32b7061aa45c039ebb68d9b6351fa7c5df8c8fdef9febeaef6f2805ba55894c4226108187e3cf9aca49bea5bef41
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,13 @@
|
|
1
1
|
CHANGELOG
|
2
2
|
================================
|
3
3
|
|
4
|
+
v.3.1.4 (2015-02-10)
|
5
|
+
--------------------
|
6
|
+
####Improvements
|
7
|
+
* Use TLSv1.2 by default, and allow users to specify SSL version in config with the :ssl_version parameter
|
8
|
+
* Minor refactoring according to Rubocop style guides
|
9
|
+
* Test against Ruby 2.2 on Travis
|
10
|
+
|
4
11
|
v.3.1.3 (2014-12-02)
|
5
12
|
--------------------
|
6
13
|
####Fixes
|
data/LICENSE
CHANGED
data/README.md
CHANGED
@@ -8,7 +8,7 @@ DataSift
|
|
8
8
|
[travis]: https://travis-ci.org/datasift/datasift-ruby
|
9
9
|
|
10
10
|
|
11
|
-
The official Ruby library for accessing the DataSift API.
|
11
|
+
The official Ruby library for accessing the DataSift API.
|
12
12
|
|
13
13
|
Getting Started
|
14
14
|
---------------
|
@@ -93,7 +93,8 @@ This version of the client library has been tested, and is known to work against
|
|
93
93
|
### Language Versions
|
94
94
|
* Ruby 1.9.3 (Seems to work, but NOT officially supported/thoroughly tested)
|
95
95
|
* Ruby 2.0.0
|
96
|
-
* Ruby 2.1
|
96
|
+
* Ruby 2.1
|
97
|
+
* Ruby 2.2
|
97
98
|
|
98
99
|
### Operating Systems
|
99
100
|
* Linux
|
@@ -104,7 +105,7 @@ This version of the client library has been tested, and is known to work against
|
|
104
105
|
License
|
105
106
|
-------
|
106
107
|
|
107
|
-
All code contained in this repository is Copyright 2011-
|
108
|
+
All code contained in this repository is Copyright 2011-2015 MediaSift Ltd.
|
108
109
|
|
109
110
|
This code is released under the BSD license. Please see the [LICENSE](https://github.com/datasift/datasift-ruby/blob/master/LICENSE) file for
|
110
111
|
more details.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.1.
|
1
|
+
3.1.4
|
data/examples/live_stream_eg.rb
CHANGED
@@ -9,12 +9,12 @@ class StreamingApi < DataSiftExample
|
|
9
9
|
|
10
10
|
def run
|
11
11
|
begin
|
12
|
-
ruby_received
|
12
|
+
ruby_received = 0
|
13
13
|
python_received = 0
|
14
|
-
ruby
|
14
|
+
ruby = 'interaction.content contains "ruby"'
|
15
15
|
ruby_stream = @datasift.compile ruby
|
16
16
|
|
17
|
-
python
|
17
|
+
python = 'interaction.content contains "python"'
|
18
18
|
python_stream = @datasift.compile python
|
19
19
|
|
20
20
|
on_delete = lambda { |stream, m| puts 'We must delete this to be compliant ==> ' + m }
|
data/lib/api/api_resource.rb
CHANGED
@@ -7,7 +7,18 @@ module DataSift
|
|
7
7
|
config[:api_host] = 'api.datasift.com' unless config.has_key?(:api_host)
|
8
8
|
config[:stream_host] = 'websocket.datasift.com' unless config.has_key?(:stream_host)
|
9
9
|
config[:api_version] = 'v1' unless config.has_key?(:api_version)
|
10
|
-
config[:enable_ssl] =
|
10
|
+
config[:enable_ssl] = true unless config.has_key?(:enable_ssl)
|
11
|
+
# Only SSLv3, TLSv1 and TLSv1.2 currently supported, TLSv1.2 preferred
|
12
|
+
# this is fixed in REST client and is scheduled for the 1.7.0 release
|
13
|
+
# see https://github.com/rest-client/rest-client/pull/123
|
14
|
+
|
15
|
+
ssl_default = "TLSv1_2"
|
16
|
+
if RUBY_VERSION.to_i == 1
|
17
|
+
ssl_default = "TLSv1"
|
18
|
+
end
|
19
|
+
OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version] = config[:ssl_version] ||
|
20
|
+
ssl_default
|
21
|
+
|
11
22
|
# max 320 seconds retry - http://dev.datasift.com/docs/streaming-api/reconnecting
|
12
23
|
config[:max_retry_time] = 320 unless config.has_key?(:max_retry_time)
|
13
24
|
config[:retry_timeout] = 0 unless config.has_key?(:retry_timeout)
|
data/lib/datasift.rb
CHANGED
@@ -33,28 +33,20 @@ module DataSift
|
|
33
33
|
while DETECT_DEAD_SOCKETS
|
34
34
|
now = Time.now.to_i
|
35
35
|
KNOWN_SOCKETS.clone.map { |connection, last_time|
|
36
|
-
if now - last_time > SOCKET_DETECTOR_TIMEOUT
|
37
|
-
connection.stream.reconnect
|
38
|
-
end
|
36
|
+
connection.stream.reconnect if now - last_time > SOCKET_DETECTOR_TIMEOUT
|
39
37
|
}
|
40
38
|
sleep SOCKET_DETECTOR_TIMEOUT * 10
|
41
39
|
end
|
42
40
|
end
|
43
41
|
|
44
42
|
class Client < ApiResource
|
45
|
-
|
46
43
|
#+config+:: A hash containing configuration options for the client for e.g.
|
47
44
|
# {username => 'some_user', api_key => 'ds_api_key', 'enable_ssl' => true, open_timeout => 30, timeout => 30}
|
48
|
-
def initialize
|
49
|
-
|
50
|
-
|
51
|
-
# see https://github.com/rest-client/rest-client/pull/123
|
52
|
-
OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version] = 'TLSv1'
|
53
|
-
|
54
|
-
if config == nil
|
55
|
-
raise InvalidConfigError.new ('Config cannot be nil')
|
45
|
+
def initialize(config)
|
46
|
+
if config.nil?
|
47
|
+
raise InvalidConfigError.new('Config cannot be nil')
|
56
48
|
end
|
57
|
-
if config.key?(:
|
49
|
+
if !config.key?(:username) || !config.key?(:api_key)
|
58
50
|
raise InvalidConfigError.new('A valid username and API key are required. ' +
|
59
51
|
'You can check your API credentials at https://datasift.com/settings')
|
60
52
|
end
|
@@ -70,16 +62,17 @@ module DataSift
|
|
70
62
|
@dynamic_list_replace = DataSift::DynamicListReplace.new(config)
|
71
63
|
end
|
72
64
|
|
73
|
-
attr_reader :historics, :push, :managed_source, :managed_source_resource,
|
74
|
-
:historics_preview, :dynamic_list,
|
65
|
+
attr_reader :historics, :push, :managed_source, :managed_source_resource,
|
66
|
+
:managed_source_auth, :historics_preview, :dynamic_list,
|
67
|
+
:dynamic_list_replace
|
75
68
|
|
76
69
|
##
|
77
70
|
# Checks if the syntax of the given CSDL is valid
|
78
|
-
#+boolResponse+ If true then a boolean is returned indicating whether the
|
79
|
-
# the response object itself is returned
|
71
|
+
#+boolResponse+ If true then a boolean is returned indicating whether the
|
72
|
+
# CSDL is valid, otherwise the response object itself is returned
|
80
73
|
def valid?(csdl, boolResponse = true)
|
81
|
-
requires({:csdl => csdl})
|
82
|
-
res = DataSift.request(:POST, 'validate', @config,
|
74
|
+
requires({ :csdl => csdl })
|
75
|
+
res = DataSift.request(:POST, 'validate', @config, :csdl => csdl )
|
83
76
|
boolResponse ? res[:http][:status] == 200 : res
|
84
77
|
end
|
85
78
|
|
@@ -87,22 +80,22 @@ module DataSift
|
|
87
80
|
# Compile CSDL code.
|
88
81
|
#+csdl+:: The CSDL you wish to compile
|
89
82
|
def compile(csdl)
|
90
|
-
requires({:csdl => csdl})
|
91
|
-
DataSift.request(:POST, 'compile', @config,
|
83
|
+
requires({ :csdl => csdl })
|
84
|
+
DataSift.request(:POST, 'compile', @config, :csdl => csdl )
|
92
85
|
end
|
93
86
|
|
94
87
|
##
|
95
|
-
# Check the number of objects processed and delivered
|
88
|
+
# Check the number of objects processed and delivered in a given period
|
96
89
|
#+period+:: Can be "day", "hour", or "current", defaults to hour
|
97
90
|
def usage(period = :hour)
|
98
|
-
DataSift.request(:POST, 'usage', @config,
|
91
|
+
DataSift.request(:POST, 'usage', @config, :period => period )
|
99
92
|
end
|
100
93
|
|
101
94
|
##
|
102
95
|
# Calculate the DPU cost of consuming a stream.
|
103
96
|
def dpu(hash)
|
104
|
-
requires ({:hash => hash})
|
105
|
-
DataSift.request(:POST, 'dpu', @config,
|
97
|
+
requires ({ :hash => hash })
|
98
|
+
DataSift.request(:POST, 'dpu', @config, :hash => hash )
|
106
99
|
end
|
107
100
|
|
108
101
|
##
|
@@ -113,40 +106,40 @@ module DataSift
|
|
113
106
|
|
114
107
|
##
|
115
108
|
# Collect a batch of interactions from a push queue
|
116
|
-
def pull(id, size =
|
117
|
-
DataSift.request(:POST, 'pull', @config, {:id => id, :size => size, :cursor => cursor})
|
109
|
+
def pull(id, size = 20_971_520, cursor = '')
|
110
|
+
DataSift.request(:POST, 'pull', @config, { :id => id, :size => size, :cursor => cursor })
|
118
111
|
end
|
119
|
-
|
120
112
|
end
|
121
113
|
|
122
|
-
|
123
114
|
# Generates and executes an HTTP request from the params provided
|
124
115
|
# Params:
|
125
116
|
# +method+:: the HTTP method to use e.g. GET,POST
|
126
117
|
# +path+:: the DataSift path relevant to the base URL of the API
|
127
118
|
# +username+:: API username
|
128
119
|
# +api_key+:: DS api key
|
129
|
-
# +params+:: A hash representing the params to use in the request, if
|
130
|
-
#
|
131
|
-
#
|
132
|
-
|
120
|
+
# +params+:: A hash representing the params to use in the request, if its a
|
121
|
+
# GET, HEAD or DELETE request these params are used as query string params,
|
122
|
+
# if not they become form url encoded params
|
123
|
+
# +headers+:: any headers to pass to the API, Authorization header is
|
124
|
+
# automatically included
|
125
|
+
def self.request(method, path, config, params = {}, headers = {}, timeout = 30, open_timeout = 30, new_line_separated = false)
|
133
126
|
validate config
|
134
127
|
options = {}
|
135
|
-
url
|
128
|
+
url = build_url(path, config)
|
136
129
|
|
137
|
-
headers.update
|
130
|
+
headers.update(
|
138
131
|
:user_agent => "DataSift/#{config[:api_version]} Ruby/v#{VERSION}",
|
139
132
|
:authorization => "#{config[:username]}:#{config[:api_key]}",
|
140
133
|
:content_type => 'application/x-www-form-urlencoded'
|
141
|
-
|
134
|
+
)
|
142
135
|
|
143
136
|
case method.to_s.downcase.to_sym
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
137
|
+
when :get, :head, :delete
|
138
|
+
url += "#{URI.parse(url).query ? '&' : '?'}#{encode params}"
|
139
|
+
payload = nil
|
140
|
+
else
|
141
|
+
payload = MultiJson.dump(params)
|
142
|
+
headers.update({ :content_type => 'application/json' })
|
150
143
|
end
|
151
144
|
|
152
145
|
options.update(
|
@@ -156,19 +149,20 @@ module DataSift
|
|
156
149
|
:timeout => timeout,
|
157
150
|
:payload => payload,
|
158
151
|
:url => url,
|
159
|
-
:ssl_version => OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version]
|
152
|
+
:ssl_version => OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:ssl_version],
|
153
|
+
:verify_ssl => OpenSSL::SSL::VERIFY_PEER
|
160
154
|
)
|
161
155
|
|
162
156
|
begin
|
163
157
|
response = RestClient::Request.execute options
|
164
|
-
if response
|
158
|
+
if !response.nil? && response.length > 0
|
165
159
|
if new_line_separated
|
166
160
|
res_arr = response.split("\n")
|
167
161
|
data = []
|
168
162
|
res_arr.each { |e|
|
169
163
|
interaction = MultiJson.load(e, :symbolize_keys => true)
|
170
164
|
data.push(interaction)
|
171
|
-
if params.
|
165
|
+
if params.key? :on_interaction
|
172
166
|
params[:on_interaction].call(interaction)
|
173
167
|
end
|
174
168
|
}
|
@@ -179,7 +173,7 @@ module DataSift
|
|
179
173
|
data = {}
|
180
174
|
end
|
181
175
|
{
|
182
|
-
:data
|
176
|
+
:data => data,
|
183
177
|
:datasift => {
|
184
178
|
:x_ratelimit_limit => response.headers[:x_ratelimit_limit],
|
185
179
|
:x_ratelimit_remaining => response.headers[:x_ratelimit_remaining],
|
@@ -190,7 +184,7 @@ module DataSift
|
|
190
184
|
:headers => response.headers
|
191
185
|
}
|
192
186
|
}
|
193
|
-
rescue MultiJson::DecodeError
|
187
|
+
rescue MultiJson::DecodeError
|
194
188
|
raise DataSiftError.new response
|
195
189
|
rescue SocketError => e
|
196
190
|
process_client_error(e)
|
@@ -208,71 +202,74 @@ module DataSift
|
|
208
202
|
process_client_error(e)
|
209
203
|
end
|
210
204
|
rescue RestClient::Exception, Errno::ECONNREFUSED => e
|
211
|
-
process_client_error
|
205
|
+
process_client_error(e)
|
212
206
|
end
|
213
207
|
end
|
214
208
|
|
215
209
|
def self.build_url(path, config)
|
216
|
-
'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host] +
|
210
|
+
'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host] +
|
211
|
+
'/' + config[:api_version] + '/' + path
|
217
212
|
end
|
218
213
|
|
219
|
-
#
|
220
|
-
def self.is_invalid?
|
214
|
+
# Returns true if username or api key are not set
|
215
|
+
def self.is_invalid?(config)
|
221
216
|
!config.key?(:username) || !config.key?(:api_key)
|
222
217
|
end
|
223
218
|
|
224
|
-
def self.validate
|
225
|
-
if is_invalid?
|
219
|
+
def self.validate(config)
|
220
|
+
if is_invalid? config
|
226
221
|
raise InvalidConfigError.new 'A username and api_key are required'
|
227
222
|
end
|
228
223
|
end
|
229
224
|
|
230
|
-
def self.encode
|
225
|
+
def self.encode(params)
|
231
226
|
params.collect { |param, value| [param, CGI.escape(value.to_s)].join('=') }.join('&')
|
232
227
|
end
|
233
228
|
|
234
229
|
def self.handle_api_error(code, body)
|
235
230
|
case code
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
231
|
+
when 400
|
232
|
+
raise BadRequestError.new(code, body)
|
233
|
+
when 401
|
234
|
+
raise AuthError.new(code, body)
|
235
|
+
when 404
|
236
|
+
raise ApiResourceNotFoundError.new(code, body)
|
237
|
+
else
|
238
|
+
raise DataSiftError.new(code, body)
|
244
239
|
end
|
245
240
|
end
|
246
241
|
|
247
242
|
def self.process_client_error(e)
|
248
243
|
case e
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
244
|
+
when RestClient::ServerBrokeConnection, RestClient::RequestTimeout
|
245
|
+
message = 'Unable to connect to DataSift. Please check your connection and try again'
|
246
|
+
when RestClient::SSLCertificateNotVerified
|
247
|
+
message = 'Failed to complete SSL verification'
|
248
|
+
when SocketError
|
249
|
+
message = 'Communication with DataSift failed. Are you able to resolve the API hostname?'
|
250
|
+
else
|
251
|
+
message = 'Unexpected error.'
|
257
252
|
end
|
258
253
|
raise ConnectionError.new(message + " (Network error: #{e.message})")
|
259
254
|
end
|
260
255
|
|
261
256
|
##
|
262
|
-
#
|
263
|
-
# DataSift and its customers are required to process Twitter's
|
264
|
-
# a
|
265
|
-
#
|
257
|
+
# A Proc/lambda callback to receive delete messages.
|
258
|
+
# DataSift and its customers are required to process Twitter's Tweet delete
|
259
|
+
# requests; a delete handler must be provided.
|
260
|
+
# A Proc/lambda callback to receive errors
|
261
|
+
# Because EventMachine is used errors can be raised from another thread, this
|
262
|
+
# method will receive any such errors
|
266
263
|
def self.new_stream(config, on_delete, on_error, on_open = nil, on_close = nil)
|
267
|
-
if on_delete
|
264
|
+
if on_delete.nil? || on_error.nil?
|
268
265
|
raise NotConfiguredError.new 'on_delete and on_error are required before you can connect'
|
269
266
|
end
|
270
267
|
raise BadParametersError.new('on_delete - 2 parameter required') unless on_delete.arity == 2
|
271
268
|
raise BadParametersError.new('on_error - 2 parameter required') unless on_error.arity == 2
|
272
|
-
|
269
|
+
unless on_open.nil?
|
273
270
|
raise BadParametersError.new('on_open - 1 parameter required') unless on_open.arity == 1
|
274
271
|
end
|
275
|
-
|
272
|
+
unless on_close.nil?
|
276
273
|
raise BadParametersError.new('on_close - 2 parameter required') unless on_close.arity == 2
|
277
274
|
end
|
278
275
|
begin
|
@@ -285,25 +282,25 @@ module DataSift
|
|
285
282
|
stream.on_open = lambda {
|
286
283
|
connection.connected = true
|
287
284
|
connection.retry_timeout = 0
|
288
|
-
on_open.call(connection)
|
285
|
+
on_open.call(connection) unless on_open.nil?
|
289
286
|
}
|
290
287
|
|
291
|
-
stream.on_close
|
288
|
+
stream.on_close = lambda { |message|
|
292
289
|
connection.connected = false
|
293
290
|
retry_connect(config, connection, on_delete, on_error, on_open, on_close, message, true)
|
294
291
|
}
|
295
|
-
stream.on_error
|
292
|
+
stream.on_error = lambda { |message|
|
296
293
|
connection.connected = false
|
297
294
|
retry_connect(config, connection, on_delete, on_error, on_open, on_close, message)
|
298
295
|
}
|
299
296
|
stream.on_message=lambda { |msg|
|
300
|
-
data
|
297
|
+
data = MultiJson.load(msg.data, :symbolize_keys => true)
|
301
298
|
KNOWN_SOCKETS[connection] = Time.new.to_i
|
302
|
-
if data.
|
299
|
+
if data.key?(:deleted)
|
303
300
|
on_delete.call(connection, data)
|
304
|
-
elsif data.
|
301
|
+
elsif data.key?(:status)
|
305
302
|
connection.fire_ds_message(data)
|
306
|
-
elsif data.
|
303
|
+
elsif data.key?(:reconnect)
|
307
304
|
connection.stream.reconnect
|
308
305
|
else
|
309
306
|
connection.fire_on_message(data[:hash], data[:data])
|
@@ -311,21 +308,21 @@ module DataSift
|
|
311
308
|
}
|
312
309
|
rescue Exception => e
|
313
310
|
case e
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
311
|
+
when DataSiftError, ArgumentError
|
312
|
+
raise e
|
313
|
+
else
|
314
|
+
retry_connect(config, connection, on_delete, on_error, on_open, on_close, e.message)
|
318
315
|
end
|
319
316
|
end
|
320
317
|
connection
|
321
318
|
end
|
322
319
|
|
323
320
|
def self.retry_connect(config, connection, on_delete, on_error, on_open, on_close, message = '', use_closed = false)
|
324
|
-
config[:retry_timeout]
|
321
|
+
config[:retry_timeout] = config[:retry_timeout] == 0 || config[:retry_timeout].nil? ? 10 : config[:retry_timeout] * 2
|
325
322
|
connection.retry_timeout = config[:retry_timeout]
|
326
323
|
|
327
324
|
if config[:retry_timeout] > config[:max_retry_time]
|
328
|
-
if use_closed && on_close
|
325
|
+
if use_closed && !on_close.nil?
|
329
326
|
on_close.call(connection, message)
|
330
327
|
else
|
331
328
|
on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has failed, re-connection was attempted but
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datasift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- DataSift
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rest-client
|
@@ -221,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
221
221
|
version: 1.3.6
|
222
222
|
requirements: []
|
223
223
|
rubyforge_project:
|
224
|
-
rubygems_version: 2.4.
|
224
|
+
rubygems_version: 2.4.5
|
225
225
|
signing_key:
|
226
226
|
specification_version: 4
|
227
227
|
summary: DataSift is a simple wrapper for the DataSift API.
|