datasift 3.3.0 → 3.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/.rubocop_todo.yml +300 -0
- data/CHANGELOG.md +10 -0
- data/VERSION +1 -1
- data/datasift.gemspec +1 -1
- data/examples/ingestion/twitter_gnip_batch_eg.rb +61 -0
- data/lib/api/api_resource.rb +3 -5
- data/lib/datasift.rb +45 -22
- data/lib/errors.rb +11 -3
- data/lib/odp.rb +14 -0
- data/test/datasift/odp/batch_upload_test.rb +52 -0
- data/test/fixtures/cassettes/odp/batch/after_upload.json +1 -0
- data/test/fixtures/cassettes/odp/batch/before_upload.json +1 -0
- data/test/fixtures/cassettes/odp/batch/upload_failure_no_source.json +1 -0
- data/test/fixtures/cassettes/odp/batch/upload_success.json +1 -0
- data/test/fixtures/data/fake_gnip_tweets.json +10 -0
- metadata +15 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 231dfbf3fbb478090969896e459179651d9aab7d
|
4
|
+
data.tar.gz: dc955abee233f6cb82a5765065796563a89ef268
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5adc6aa0777edfef9835b08e1e133a63805cf10e926ab67fda130cca85ca79509e466d91b1c7eb21affa44f6829dd7fdc1c2807e0ae66666581c44aa532b3161
|
7
|
+
data.tar.gz: 7074591477540122765b3c91bd3dec1dbd1f2d65371277fb59b24a523455e791f44c21d2fa2572fa4abe04ce21f20a52203485bab10957bc349b8d1ee39ac8b3
|
data/.rubocop.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,300 @@
|
|
1
|
+
# This configuration was generated by `rubocop --auto-gen-config`
|
2
|
+
# on 2015-08-14 10:29:23 +0100 using RuboCop version 0.32.0.
|
3
|
+
# The point is for the user to remove these configuration records
|
4
|
+
# one by one as the offenses are removed from the code base.
|
5
|
+
# Note that changes in the inspected code, or installation of new
|
6
|
+
# versions of RuboCop, may require this file to be generated again.
|
7
|
+
|
8
|
+
# Offense count: 1
|
9
|
+
Lint/ParenthesesAsGroupedExpression:
|
10
|
+
Enabled: false
|
11
|
+
|
12
|
+
# Offense count: 1
|
13
|
+
Lint/RescueException:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
# Offense count: 2
|
17
|
+
# Cop supports --auto-correct.
|
18
|
+
Lint/SpaceBeforeFirstArg:
|
19
|
+
Enabled: false
|
20
|
+
|
21
|
+
# Offense count: 5
|
22
|
+
# Cop supports --auto-correct.
|
23
|
+
Lint/UnusedBlockArgument:
|
24
|
+
Enabled: false
|
25
|
+
|
26
|
+
# Offense count: 2
|
27
|
+
Lint/UselessAssignment:
|
28
|
+
Enabled: false
|
29
|
+
|
30
|
+
# Offense count: 27
|
31
|
+
Metrics/AbcSize:
|
32
|
+
Max: 106
|
33
|
+
|
34
|
+
# Offense count: 1
|
35
|
+
# Configuration parameters: CountComments.
|
36
|
+
Metrics/ClassLength:
|
37
|
+
Max: 108
|
38
|
+
|
39
|
+
# Offense count: 13
|
40
|
+
Metrics/CyclomaticComplexity:
|
41
|
+
Max: 15
|
42
|
+
|
43
|
+
# Offense count: 118
|
44
|
+
# Configuration parameters: AllowURI, URISchemes.
|
45
|
+
LineLength:
|
46
|
+
Max: 90
|
47
|
+
|
48
|
+
# Offense count: 33
|
49
|
+
# Configuration parameters: CountComments.
|
50
|
+
Metrics/MethodLength:
|
51
|
+
Max: 91
|
52
|
+
|
53
|
+
# Offense count: 1
|
54
|
+
# Configuration parameters: CountComments.
|
55
|
+
Metrics/ModuleLength:
|
56
|
+
Max: 232
|
57
|
+
|
58
|
+
# Offense count: 8
|
59
|
+
# Configuration parameters: CountKeywordArgs.
|
60
|
+
Metrics/ParameterLists:
|
61
|
+
Max: 8
|
62
|
+
|
63
|
+
# Offense count: 8
|
64
|
+
Metrics/PerceivedComplexity:
|
65
|
+
Max: 16
|
66
|
+
|
67
|
+
# Offense count: 2
|
68
|
+
# Cop supports --auto-correct.
|
69
|
+
Style/AlignArray:
|
70
|
+
Enabled: false
|
71
|
+
|
72
|
+
# Offense count: 1
|
73
|
+
# Cop supports --auto-correct.
|
74
|
+
# Configuration parameters: EnforcedHashRocketStyle, EnforcedColonStyle, EnforcedLastArgumentHashStyle, SupportedLastArgumentHashStyles.
|
75
|
+
Style/AlignHash:
|
76
|
+
Enabled: false
|
77
|
+
|
78
|
+
# Offense count: 6
|
79
|
+
# Cop supports --auto-correct.
|
80
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
81
|
+
Style/AlignParameters:
|
82
|
+
Enabled: false
|
83
|
+
|
84
|
+
# Offense count: 3
|
85
|
+
# Cop supports --auto-correct.
|
86
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles, ProceduralMethods, FunctionalMethods, IgnoredMethods.
|
87
|
+
Style/BlockDelimiters:
|
88
|
+
Enabled: false
|
89
|
+
|
90
|
+
# Offense count: 20
|
91
|
+
# Cop supports --auto-correct.
|
92
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
93
|
+
Style/BracesAroundHashParameters:
|
94
|
+
Enabled: false
|
95
|
+
|
96
|
+
# Offense count: 20
|
97
|
+
# Cop supports --auto-correct.
|
98
|
+
# Configuration parameters: IndentWhenRelativeTo, SupportedStyles, IndentOneStep.
|
99
|
+
Style/CaseIndentation:
|
100
|
+
Enabled: false
|
101
|
+
|
102
|
+
# Offense count: 1
|
103
|
+
# Cop supports --auto-correct.
|
104
|
+
Style/ColonMethodCall:
|
105
|
+
Enabled: false
|
106
|
+
|
107
|
+
# Offense count: 7
|
108
|
+
# Cop supports --auto-correct.
|
109
|
+
Style/CommentIndentation:
|
110
|
+
Enabled: false
|
111
|
+
|
112
|
+
# Offense count: 6
|
113
|
+
# Cop supports --auto-correct.
|
114
|
+
Style/DeprecatedHashMethods:
|
115
|
+
Enabled: false
|
116
|
+
|
117
|
+
# Offense count: 14
|
118
|
+
Style/Documentation:
|
119
|
+
Enabled: false
|
120
|
+
|
121
|
+
# Offense count: 5
|
122
|
+
# Cop supports --auto-correct.
|
123
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
124
|
+
Style/EmptyElse:
|
125
|
+
Enabled: false
|
126
|
+
|
127
|
+
# Offense count: 2
|
128
|
+
# Cop supports --auto-correct.
|
129
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
130
|
+
Style/EmptyLinesAroundBlockBody:
|
131
|
+
Enabled: false
|
132
|
+
|
133
|
+
# Offense count: 2
|
134
|
+
# Cop supports --auto-correct.
|
135
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
136
|
+
Style/EmptyLinesAroundClassBody:
|
137
|
+
Enabled: false
|
138
|
+
|
139
|
+
# Offense count: 1
|
140
|
+
# Configuration parameters: MinBodyLength.
|
141
|
+
Style/GuardClause:
|
142
|
+
Enabled: false
|
143
|
+
|
144
|
+
# Offense count: 181
|
145
|
+
# Cop supports --auto-correct.
|
146
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues.
|
147
|
+
Style/HashSyntax:
|
148
|
+
Enabled: false
|
149
|
+
|
150
|
+
# Offense count: 2
|
151
|
+
# Cop supports --auto-correct.
|
152
|
+
# Configuration parameters: MaxLineLength.
|
153
|
+
Style/IfUnlessModifier:
|
154
|
+
Enabled: false
|
155
|
+
|
156
|
+
# Offense count: 2
|
157
|
+
# Cop supports --auto-correct.
|
158
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
159
|
+
Style/IndentHash:
|
160
|
+
Enabled: false
|
161
|
+
|
162
|
+
# Offense count: 3
|
163
|
+
# Cop supports --auto-correct.
|
164
|
+
Style/Lambda:
|
165
|
+
Enabled: false
|
166
|
+
|
167
|
+
# Offense count: 12
|
168
|
+
# Cop supports --auto-correct.
|
169
|
+
Style/LeadingCommentSpace:
|
170
|
+
Enabled: false
|
171
|
+
|
172
|
+
# Offense count: 1
|
173
|
+
# Cop supports --auto-correct.
|
174
|
+
Style/LineEndConcatenation:
|
175
|
+
Enabled: false
|
176
|
+
|
177
|
+
# Offense count: 2
|
178
|
+
# Cop supports --auto-correct.
|
179
|
+
Style/MethodCallParentheses:
|
180
|
+
Enabled: false
|
181
|
+
|
182
|
+
# Offense count: 1
|
183
|
+
# Cop supports --auto-correct.
|
184
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
185
|
+
Style/MethodDefParentheses:
|
186
|
+
Enabled: false
|
187
|
+
|
188
|
+
# Offense count: 2
|
189
|
+
Style/MultilineTernaryOperator:
|
190
|
+
Enabled: false
|
191
|
+
|
192
|
+
# Offense count: 4
|
193
|
+
# Cop supports --auto-correct.
|
194
|
+
Style/NegatedIf:
|
195
|
+
Enabled: false
|
196
|
+
|
197
|
+
# Offense count: 2
|
198
|
+
# Cop supports --auto-correct.
|
199
|
+
Style/NilComparison:
|
200
|
+
Enabled: false
|
201
|
+
|
202
|
+
# Offense count: 7
|
203
|
+
# Cop supports --auto-correct.
|
204
|
+
Style/NumericLiterals:
|
205
|
+
MinDigits: 10
|
206
|
+
|
207
|
+
# Offense count: 2
|
208
|
+
# Cop supports --auto-correct.
|
209
|
+
# Configuration parameters: PreferredDelimiters.
|
210
|
+
Style/PercentLiteralDelimiters:
|
211
|
+
Enabled: false
|
212
|
+
|
213
|
+
# Offense count: 1
|
214
|
+
# Configuration parameters: NamePrefix, NamePrefixBlacklist.
|
215
|
+
Style/PredicateName:
|
216
|
+
Enabled: false
|
217
|
+
|
218
|
+
# Offense count: 14
|
219
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
220
|
+
Style/RaiseArgs:
|
221
|
+
Enabled: false
|
222
|
+
|
223
|
+
# Offense count: 11
|
224
|
+
# Cop supports --auto-correct.
|
225
|
+
Style/RedundantBegin:
|
226
|
+
Enabled: false
|
227
|
+
|
228
|
+
# Offense count: 19
|
229
|
+
# Cop supports --auto-correct.
|
230
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
231
|
+
Style/SignalException:
|
232
|
+
Enabled: false
|
233
|
+
|
234
|
+
# Offense count: 2
|
235
|
+
# Cop supports --auto-correct.
|
236
|
+
Style/SpaceAfterComma:
|
237
|
+
Enabled: false
|
238
|
+
|
239
|
+
# Offense count: 1
|
240
|
+
# Cop supports --auto-correct.
|
241
|
+
Style/SpaceAfterMethodName:
|
242
|
+
Enabled: false
|
243
|
+
|
244
|
+
# Offense count: 1
|
245
|
+
# Cop supports --auto-correct.
|
246
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
247
|
+
Style/SpaceAroundEqualsInParameterDefault:
|
248
|
+
Enabled: false
|
249
|
+
|
250
|
+
# Offense count: 7
|
251
|
+
# Cop supports --auto-correct.
|
252
|
+
# Configuration parameters: MultiSpaceAllowedForOperators.
|
253
|
+
Style/SpaceAroundOperators:
|
254
|
+
Enabled: false
|
255
|
+
|
256
|
+
# Offense count: 1
|
257
|
+
# Cop supports --auto-correct.
|
258
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
259
|
+
Style/SpaceBeforeBlockBraces:
|
260
|
+
Enabled: false
|
261
|
+
|
262
|
+
# Offense count: 15
|
263
|
+
# Cop supports --auto-correct.
|
264
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
|
265
|
+
Style/SpaceInsideHashLiteralBraces:
|
266
|
+
Enabled: false
|
267
|
+
|
268
|
+
# Offense count: 3
|
269
|
+
# Cop supports --auto-correct.
|
270
|
+
Style/SpaceInsideParens:
|
271
|
+
Enabled: false
|
272
|
+
|
273
|
+
# Offense count: 23
|
274
|
+
# Cop supports --auto-correct.
|
275
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
276
|
+
Style/StringLiterals:
|
277
|
+
Enabled: false
|
278
|
+
|
279
|
+
# Offense count: 1
|
280
|
+
# Cop supports --auto-correct.
|
281
|
+
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
282
|
+
Style/TrailingBlankLines:
|
283
|
+
Enabled: false
|
284
|
+
|
285
|
+
# Offense count: 1
|
286
|
+
# Cop supports --auto-correct.
|
287
|
+
# Configuration parameters: EnforcedStyleForMultiline, SupportedStyles.
|
288
|
+
Style/TrailingComma:
|
289
|
+
Enabled: false
|
290
|
+
|
291
|
+
# Offense count: 2
|
292
|
+
# Cop supports --auto-correct.
|
293
|
+
Style/UnneededPercentQ:
|
294
|
+
Enabled: false
|
295
|
+
|
296
|
+
# Offense count: 3
|
297
|
+
# Cop supports --auto-correct.
|
298
|
+
# Configuration parameters: WordRegex.
|
299
|
+
Style/WordArray:
|
300
|
+
MinSize: 4
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,15 @@
|
|
1
1
|
CHANGELOG
|
2
2
|
================================
|
3
|
+
## v.3.4.0 (2015-08-20)
|
4
|
+
### Added
|
5
|
+
* Support for [Open Data Processing](https://datasift.com/products/open-data-processing-for-twitter/) batch uploads (Thanks @giovannelli)
|
6
|
+
* Explicit supprot for 413 and 422 errors from API
|
7
|
+
* Ability to get at API response headers using the ```object.response``` accessor. (Thanks again @giovannelli)
|
8
|
+
|
9
|
+
### Changed
|
10
|
+
* Bumped [rest-client](https://github.com/rest-client/rest-client) dependency to ~> 1.8
|
11
|
+
* The OpenSSL version you select within the DataSift gem no longer changes the default OpenSSL version in your wider application; it only affects API calls made by the DataSift gem to DataSift's API
|
12
|
+
|
3
13
|
## v.3.3.0 (2015-08-05)
|
4
14
|
### Added
|
5
15
|
* Explicit support for 429 errors from the API
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.
|
1
|
+
3.4.0
|
data/datasift.gemspec
CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.test_files = `git ls-files -- {test}/*`.split("\n")
|
16
16
|
s.require_paths = ["lib"]
|
17
17
|
|
18
|
-
s.add_runtime_dependency('rest-client', '~> 1.
|
18
|
+
s.add_runtime_dependency('rest-client', '~> 1.8')
|
19
19
|
s.add_runtime_dependency('multi_json', '~> 1.7')
|
20
20
|
s.add_runtime_dependency('websocket-td', '~> 0.0.5')
|
21
21
|
s.add_development_dependency('bundler', '~> 1.0')
|
@@ -0,0 +1,61 @@
|
|
1
|
+
require './../auth'
|
2
|
+
class TwitterGnipBatchEg < DataSiftExample
|
3
|
+
def initialize
|
4
|
+
super
|
5
|
+
@datasift = DataSift::Client.new(@config)
|
6
|
+
run
|
7
|
+
end
|
8
|
+
|
9
|
+
def run
|
10
|
+
begin
|
11
|
+
# "gnip_1" is the default mapping used to map Tweets ingested from Gnip into a
|
12
|
+
# format DataSift can use. Using this mapping will altomatically define the
|
13
|
+
# IDML you need to use for this source
|
14
|
+
resource = [{
|
15
|
+
parameters: {
|
16
|
+
mapping: "gnip_1"
|
17
|
+
}
|
18
|
+
}]
|
19
|
+
puts "Creating Managed Source for ODP Ingestion\n"
|
20
|
+
source = @datasift.managed_source.create('twitter_gnip', 'Ruby ODP API', {}, resource)
|
21
|
+
puts "Manage Source with ID #{source[:data][:id]} created"
|
22
|
+
|
23
|
+
lines = 0
|
24
|
+
payload = ''
|
25
|
+
# Read interactions out of the fake Tweets fixture file
|
26
|
+
File.readlines('./../../test/fixtures/data/fake_gnip_tweets.json').each do |line|
|
27
|
+
lines += 1
|
28
|
+
payload += line
|
29
|
+
|
30
|
+
# Upload interactions as batches of five
|
31
|
+
if lines % 5 == 0
|
32
|
+
puts "\nUploading a batch of five interactions\n"
|
33
|
+
puts @datasift.odp.ingest(source[:data][:id], payload)[:data].to_json
|
34
|
+
payload = ''
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
puts "\nCleanup after this test and delete the Managed Source"
|
39
|
+
@datasift.managed_source.delete(source[:data][:id])
|
40
|
+
|
41
|
+
rescue DataSiftError => dse
|
42
|
+
puts dse.message
|
43
|
+
# Then match specific one to take action - All errors thrown by the client extend
|
44
|
+
# DataSiftError
|
45
|
+
case dse
|
46
|
+
when ConnectionError
|
47
|
+
# some connection error
|
48
|
+
when AuthError
|
49
|
+
when BadRequestError
|
50
|
+
when PayloadTooLargeError
|
51
|
+
# You've probably hit your x_ingestion_data_ratelimit_limit rate limit; back off
|
52
|
+
# and try again in a bit. Check your x_ingestion_request_ratelimit_reset header
|
53
|
+
# to see when you can retry this request successfully
|
54
|
+
else
|
55
|
+
# do something else...
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
TwitterGnipBatchEg.new
|
data/lib/api/api_resource.rb
CHANGED
@@ -11,18 +11,16 @@ module DataSift
|
|
11
11
|
@config = config
|
12
12
|
config[:api_host] = 'api.datasift.com' unless config.has_key?(:api_host)
|
13
13
|
config[:stream_host] = 'websocket.datasift.com' unless config.has_key?(:stream_host)
|
14
|
+
config[:ingestion_host] = 'in.datasift.com' unless config.has_key?(:ingestion_host)
|
14
15
|
config[:api_version] = 'v1.2' unless config.has_key?(:api_version)
|
15
16
|
config[:enable_ssl] = true unless config.has_key?(:enable_ssl)
|
16
|
-
# Only SSLv3, TLSv1 and TLSv1.2 currently supported, TLSv1.2 preferred
|
17
|
-
# this is fixed in REST client and is scheduled for the 1.7.0 release
|
18
|
-
# see https://github.com/rest-client/rest-client/pull/123
|
19
17
|
|
20
18
|
ssl_default = "TLSv1_2"
|
21
19
|
if RUBY_VERSION.to_i == 1
|
20
|
+
# Ruby 1.x does not seem to support > TLSv1
|
22
21
|
ssl_default = "TLSv1"
|
23
22
|
end
|
24
|
-
|
25
|
-
ssl_default
|
23
|
+
config[:ssl_version] = config[:ssl_version] || ssl_default
|
26
24
|
|
27
25
|
# max 320 seconds retry - http://dev.datasift.com/docs/streaming-api/reconnecting
|
28
26
|
config[:max_retry_time] = 320 unless config.key?(:max_retry_time)
|
data/lib/datasift.rb
CHANGED
@@ -21,6 +21,7 @@ require dir + '/account'
|
|
21
21
|
require dir + '/account_identity'
|
22
22
|
require dir + '/account_identity_token'
|
23
23
|
require dir + '/account_identity_limit'
|
24
|
+
require dir + '/odp'
|
24
25
|
#
|
25
26
|
require 'rbconfig'
|
26
27
|
|
@@ -51,7 +52,7 @@ module DataSift
|
|
51
52
|
raise InvalidConfigError.new('Config cannot be nil') if config.nil?
|
52
53
|
if !config.key?(:username) || !config.key?(:api_key)
|
53
54
|
raise InvalidConfigError.new('A valid username and API key are required. ' +
|
54
|
-
'You can check your API credentials at https://datasift.com/settings')
|
55
|
+
'You can check your API credentials at https://app.datasift.com/settings')
|
55
56
|
end
|
56
57
|
|
57
58
|
@config = config
|
@@ -66,11 +67,12 @@ module DataSift
|
|
66
67
|
@account_identity = DataSift::AccountIdentity.new(config)
|
67
68
|
@account_identity_token = DataSift::AccountIdentityToken.new(config)
|
68
69
|
@account_identity_limit = DataSift::AccountIdentityLimit.new(config)
|
70
|
+
@odp = DataSift::Odp.new(config)
|
69
71
|
end
|
70
72
|
|
71
73
|
attr_reader :historics, :push, :managed_source, :managed_source_resource,
|
72
74
|
:managed_source_auth, :historics_preview, :pylon, :account,
|
73
|
-
:account_identity, :account_identity_token, :account_identity_limit
|
75
|
+
:account_identity, :account_identity_token, :account_identity_limit, :odp
|
74
76
|
|
75
77
|
# Checks if the syntax of the given CSDL is valid
|
76
78
|
#
|
@@ -167,7 +169,7 @@ module DataSift
|
|
167
169
|
url += "#{URI.parse(url).query ? '&' : '?'}#{encode params}"
|
168
170
|
payload = nil
|
169
171
|
else
|
170
|
-
payload = MultiJson.dump(params)
|
172
|
+
payload = params.is_a?(String) ? params : MultiJson.dump(params)
|
171
173
|
headers.update({ :content_type => 'application/json' })
|
172
174
|
end
|
173
175
|
|
@@ -178,10 +180,11 @@ module DataSift
|
|
178
180
|
:timeout => timeout,
|
179
181
|
:payload => payload,
|
180
182
|
:url => url,
|
181
|
-
:ssl_version =>
|
183
|
+
:ssl_version => config[:ssl_version],
|
182
184
|
:verify_ssl => OpenSSL::SSL::VERIFY_PEER
|
183
185
|
)
|
184
186
|
|
187
|
+
response = nil
|
185
188
|
begin
|
186
189
|
response = RestClient::Request.execute options
|
187
190
|
if !response.nil? && response.length > 0
|
@@ -223,7 +226,19 @@ module DataSift
|
|
223
226
|
body = e.http_body
|
224
227
|
if code && body
|
225
228
|
error = MultiJson.load(body)
|
226
|
-
|
229
|
+
response_on_error = {
|
230
|
+
:data => nil,
|
231
|
+
:datasift => {
|
232
|
+
:x_ratelimit_limit => e.response.headers[:x_ratelimit_limit],
|
233
|
+
:x_ratelimit_remaining => e.response.headers[:x_ratelimit_remaining],
|
234
|
+
:x_ratelimit_cost => e.response.headers[:x_ratelimit_cost]
|
235
|
+
},
|
236
|
+
:http => {
|
237
|
+
:status => e.response.code,
|
238
|
+
:headers => e.response.headers
|
239
|
+
}
|
240
|
+
}
|
241
|
+
handle_api_error(e.http_code, (error['error'] ? error['error'] : '') + " for URL #{url}", response_on_error)
|
227
242
|
else
|
228
243
|
process_client_error(e)
|
229
244
|
end
|
@@ -238,8 +253,12 @@ module DataSift
|
|
238
253
|
private
|
239
254
|
|
240
255
|
def self.build_url(path, config)
|
241
|
-
'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host]
|
242
|
-
|
256
|
+
url = 'http' + (config[:enable_ssl] ? 's' : '') + '://' + config[:api_host]
|
257
|
+
if !config[:api_version].nil?
|
258
|
+
url += '/' + config[:api_version] + '/' + path
|
259
|
+
else
|
260
|
+
url += '/' + path
|
261
|
+
end
|
243
262
|
end
|
244
263
|
|
245
264
|
# Returns true if username or api key are not set
|
@@ -257,22 +276,26 @@ module DataSift
|
|
257
276
|
params.collect { |param, value| [param, CGI.escape(value.to_s)].join('=') }.join('&')
|
258
277
|
end
|
259
278
|
|
260
|
-
def self.handle_api_error(code, body)
|
279
|
+
def self.handle_api_error(code, body, response)
|
261
280
|
case code
|
262
281
|
when 400
|
263
|
-
raise BadRequestError.new(code, body)
|
282
|
+
raise BadRequestError.new(code, body, response)
|
264
283
|
when 401
|
265
|
-
raise AuthError.new(code, body)
|
284
|
+
raise AuthError.new(code, body, response)
|
266
285
|
when 404
|
267
|
-
raise ApiResourceNotFoundError.new(code, body)
|
286
|
+
raise ApiResourceNotFoundError.new(code, body, response)
|
268
287
|
when 409
|
269
|
-
raise ConflictError.new(code, body)
|
288
|
+
raise ConflictError.new(code, body, response)
|
270
289
|
when 410
|
271
|
-
raise GoneError.new(code, body)
|
290
|
+
raise GoneError.new(code, body, response)
|
291
|
+
when 413
|
292
|
+
raise PayloadTooLargeError.new(code, body, response)
|
293
|
+
when 422
|
294
|
+
raise UnprocessableEntityError.new(code, body, response)
|
272
295
|
when 429
|
273
|
-
raise TooManyRequestsError.new(code, body)
|
296
|
+
raise TooManyRequestsError.new(code, body, response)
|
274
297
|
else
|
275
|
-
raise DataSiftError.new(code, body)
|
298
|
+
raise DataSiftError.new(code, body, response)
|
276
299
|
end
|
277
300
|
end
|
278
301
|
|
@@ -310,7 +333,7 @@ module DataSift
|
|
310
333
|
raise BadParametersError.new('on_close - 2 parameter required') unless on_close.arity == 2
|
311
334
|
end
|
312
335
|
begin
|
313
|
-
stream = WebsocketTD::Websocket.new(
|
336
|
+
stream = WebsocketTD::Websocket.new(config[:stream_host], '/multi', "username=#{config[:username]}&api_key=#{config[:api_key]}")
|
314
337
|
connection = LiveStream.new(config, stream)
|
315
338
|
KNOWN_SOCKETS[connection] = Time.new.to_i
|
316
339
|
stream.on_ping = lambda { |data|
|
@@ -362,12 +385,12 @@ module DataSift
|
|
362
385
|
if use_closed && !on_close.nil?
|
363
386
|
on_close.call(connection, message)
|
364
387
|
else
|
365
|
-
on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
388
|
+
on_error.call(connection, ReconnectTimeoutError.new("Connecting to DataSift has " \
|
389
|
+
"failed, re-connection was attempted but multiple consecutive failures where " \
|
390
|
+
"encountered. As a result no further re-connection will be automatically " \
|
391
|
+
"attempted. Manually invoke connect() after investigating the cause of the " \
|
392
|
+
"failure, be sure to observe DataSift's re-connect policies available at " \
|
393
|
+
"http://dev.datasift.com/docs/streaming-api/reconnecting - Error {#{message}}"))
|
371
394
|
end
|
372
395
|
else
|
373
396
|
sleep config[:retry_timeout]
|
data/lib/errors.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# Custom error class for rescuing DataSift errors
|
2
2
|
class DataSiftError < StandardError
|
3
|
-
attr_reader :status
|
4
|
-
attr_reader :body
|
3
|
+
attr_reader :status, :body, :response
|
5
4
|
|
6
|
-
def initialize(http_status = nil, http_body = nil)
|
5
|
+
def initialize(http_status = nil, http_body = nil, response_on_error = nil)
|
7
6
|
@status = http_status
|
8
7
|
@body = http_body
|
8
|
+
@response = response_on_error
|
9
9
|
end
|
10
10
|
|
11
11
|
def message
|
@@ -48,6 +48,14 @@ end
|
|
48
48
|
class GoneError < DataSiftError
|
49
49
|
end
|
50
50
|
|
51
|
+
# Standard error returned when receiving a 413 response from the API
|
52
|
+
class PayloadTooLargeError < DataSiftError
|
53
|
+
end
|
54
|
+
|
55
|
+
# Standard error returned when receiving a 422 response from the API
|
56
|
+
class UnprocessableEntityError < DataSiftError
|
57
|
+
end
|
58
|
+
|
51
59
|
# Standard error returned when receiving a 429 response from the API
|
52
60
|
class TooManyRequestsError < DataSiftError
|
53
61
|
end
|
data/lib/odp.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
module DataSift
|
2
|
+
##
|
3
|
+
# Use DataSift's Open Data Processing (ODP) to upload your own data to
|
4
|
+
# DataSift for processing
|
5
|
+
class Odp < DataSift::ApiResource
|
6
|
+
def ingest(source_id, data)
|
7
|
+
fail ArgumentError, 'source_id is required' if source_id.nil?
|
8
|
+
fail ArgumentError, 'data payload is required' if data.nil?
|
9
|
+
|
10
|
+
config = @config.merge(api_host: @config[:ingestion_host], api_version: nil)
|
11
|
+
DataSift.request(:POST, source_id, config, data)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require File.expand_path('../../../test_helper', __FILE__)
|
2
|
+
|
3
|
+
describe 'DataSift::Odp' do
|
4
|
+
before do
|
5
|
+
auth = DataSiftExample.new
|
6
|
+
@datasift = auth.datasift
|
7
|
+
@data = OpenStruct.new
|
8
|
+
end
|
9
|
+
|
10
|
+
##
|
11
|
+
# Batch Upload
|
12
|
+
#
|
13
|
+
describe '#ingest (success)' do
|
14
|
+
before do
|
15
|
+
VCR.use_cassette('odp/batch/before_upload') do
|
16
|
+
resource = [{ parameters: { mapping: "gnip_1" } }]
|
17
|
+
@source = @datasift.managed_source.create('twitter_gnip', 'Ruby ODP API', {}, resource)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
after do
|
22
|
+
VCR.use_cassette('odp/batch/after_upload') do
|
23
|
+
@datasift.managed_source.delete @source[:data][:id]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'can batch upload gnip twitter data' do
|
28
|
+
VCR.use_cassette('odp/batch/upload_success') do
|
29
|
+
payload = File.open(File.expand_path('../../fixtures/data/fake_gnip_tweets.json', File.dirname(__FILE__))).read
|
30
|
+
response = @datasift.odp.ingest(@source[:data][:id], payload)
|
31
|
+
assert_equal STATUS.valid, response[:http][:status]
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe '#ingest (failure)' do
|
37
|
+
it 'handles 404 when Managed Source can not be found' do
|
38
|
+
VCR.use_cassette('odp/batch/upload_failure_no_source') do
|
39
|
+
payload = File.open(File.expand_path('../../fixtures/data/fake_gnip_tweets.json', File.dirname(__FILE__))).read
|
40
|
+
assert_raises ApiResourceNotFoundError do
|
41
|
+
response = @datasift.odp.ingest('invalid_source_id', payload)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'raises BadParametersError when payload is missing' do
|
47
|
+
assert_raises ArgumentError do
|
48
|
+
@datasift.odp.ingest('invalid_source_id')
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
{"http_interactions":[{"request":{"method":"delete","uri":"https://api.datasift.com/v1.2/source/delete?id=3446d92fd7274adeaf8b27f4e99f3b60","body":{"encoding":"US-ASCII","base64_string":""},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/v1.2 Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/x-www-form-urlencoded"]}},"response":{"status":{"code":204,"message":"No Content"},"headers":{"Server":["nginx"],"Date":["Fri, 14 Aug 2015 19:27:56 GMT"],"Content-Type":["text/html"],"Connection":["close"],"X-Api-Version":["1.2"],"P3p":["CP=\"CAO PSA\""],"X-Ratelimit-Limit":["10000"],"X-Ratelimit-Remaining":["9816"],"X-Ratelimit-Cost":["25"],"X-Ratelimit-Reset":["1439582455"],"X-Ratelimit-Reset-Ttl":["1979"],"Access-Control-Allow-Origin":["*"],"Access-Control-Allow-Headers":["Authorization,Content-Type"],"X-Served-By":["ded2584"],"X-Frame-Options":["SAMEORIGIN"]},"body":{"encoding":"UTF-8","base64_string":""},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:56 GMT"}],"recorded_with":"VCR 2.9.3"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"http_interactions":[{"request":{"method":"post","uri":"https://api.datasift.com/v1.2/source/create","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/v1.2 Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["110"]}},"response":{"status":{"code":201,"message":"Created"},"headers":{"Server":["nginx"],"Date":["Fri, 14 Aug 2015 19:27:36 GMT"],"Content-Type":["application/json"],"Transfer-Encoding":["chunked"],"Connection":["close"],"X-Api-Version":["1.2"],"P3p":["CP=\"CAO PSA\""],"X-Ratelimit-Limit":["10000"],"X-Ratelimit-Remaining":["9842"],"X-Ratelimit-Cost":["25"],"X-Ratelimit-Reset":["1439582455"],"X-Ratelimit-Reset-Ttl":["2000"],"Access-Control-Allow-Origin":["*"],"Access-Control-Allow-Headers":["Authorization,Content-Type"],"X-Served-By":["ded2587"],"X-Frame-Options":["SAMEORIGIN"]},"body":{"encoding":"UTF-8","base64_string":"eyJpZCI6IjM0NDZkOTJmZDcyNzRhZGVhZjhiMjdmNGU5OWYzYjYwIiwic3Rh\ndHVzIjoic3RvcHBlZCIsInNvdXJjZV90eXBlIjoidHdpdHRlcl9nbmlwIiwi\nbmFtZSI6IlJ1YnkgT0RQIEFQSSIsImF1dGgiOltdLCJyZXNvdXJjZXMiOlt7\nInJlc291cmNlX2lkIjoiMjUyM2FhMDUzNWIzNGYzNmFmMzlhYmEzZTJkY2Rj\nYmIiLCJwYXJhbWV0ZXJzIjp7InVuaXF1ZV9pZF9sb2NhdGlvbiI6ImlkIiwi\naW50ZXJhY3Rpb25fdHlwZSI6InR3aXR0ZXJfZ25pcCIsImF1Z21lbnRhdGlv\nbnMiOlsiZ2VuZGVyIiwia2xvdXQiLCJsYW5ndWFnZWRldGVjdGlvbiIsInNh\nbGllbmNlX2NvbnRlbnQiLCJzYWxpZW5jZV90aXRsZSIsInRyZW5kcyIsInpl\nbnlhX2NvbnRlbnQiLCJ6ZW55YV90aXRsZSJdLCJpZG1sIjoiIyBNYXBzIHRo\nZSBHbmlwIEFjdGl2aXR5U3RyZWFtIGZvcm1hdCB0byBhIFR3aXR0ZXIgb2Jq\nZWN0XG5bbWFpbl1cbnR3aXR0ZXIgPSAoYXBwbHkoXCJjcmVhdGVfcG9zdFwi\nKSBbIHJvb3QudmVyYiA9PSBcInBvc3RcIiBdIHxcbiAgICAgICAgICAgYXBw\nbHkoXCJjcmVhdGVfc2hhcmVcIikgWyByb290LnZlcmIgPT0gXCJzaGFyZVwi\nXSB8XG4gICAgICAgICAgIGFwcGx5KFwiY3JlYXRlX2RlbGV0ZVwiKSBbIHJv\nb3QudmVyYiA9PSBcImRlbGV0ZVwiIF0gfFxuICAgICAgICAgICBhcHBseShc\nImNyZWF0ZV9zdGF0dXNfd2l0aGhlbGRcIikgWyByb290LnZlcmIgPT0gXCJz\ndGF0dXNfd2l0aGhlbGRcIiBdIHxcbiAgICAgICAgICAgYXBwbHkoXCJjcmVh\ndGVfc2NydWJfZ2VvXCIpIFsgcm9vdC52ZXJiID09IFwic2NydWJfZ2VvXCIg\nXSB8XG4gICAgICAgICAgIGFwcGx5KFwiY3JlYXRlX290aGVyX2NvbXBsaWFu\nY2VcIilcbiAgICAgICAgICApXG5cbiMgZGVsZXRlcyBoYXZlIGFuIGV4dHJh\nIHRvcC1sZXZlbCBmaWVsZFxuZGVsZXRlZCA9IHRydWUgW3Jvb3QudmVyYiA9\nPSBcImRlbGV0ZVwiXVxuXG5bY3JlYXRlX2RlbGV0ZV1cbmlkICAgICAgICAg\nID0gb2JqZWN0LmlkLnNwbGl0KFwiOlwiKVsyXVxuY3JlYXRlZF9hdCAgPSB0\naW1lc3RhbXBNcy5kYXRlKClcbnVzZXIgICAgICAgID0gYWN0b3IuYXBwbHko\nXCJjcmVhdGVfYWN0b3JcIilcblxuW2NyZWF0ZV9zdGF0dXNfd2l0aGhlbGRd\nXG5jcmVhdGVkX2F0ICAgICAgICAgPSB0aW1lc3RhbXBNcy5kYXRlKClcbnN0\nYXR1cyAgICAgICAgICAgICA9IHZlcmJcbndpdGhoZWxkLmNvdW50cmllcyA9\nIG9iamVjdC53aXRoaGVsZF9pbl9jb3VudHJpZXNcblxuIyB0aGUgb2JqZWN0\nIHJlZmVycyB0byB0aGUgc3RhdHVzIG1lc3NhZ2UgYW5kIHRoZXJlJ3MgYSB1\nc2VyX2lkIGZpZWxkIGluc3RlYWQgb2YgYW4gYWN0b3JcbmlkICAgICAgICAg\nID0gb2JqZWN0LmlkLm1hdGNoKFwiLio/KFxcXFxkKylcIilbMF1cbnVzZXIu\naWQgICAgID0gb2JqZWN0LnVzZXJfaWQubWF0Y2goXCIuKj8oXFxcXGQrKVwi\nKVswXS5pbnQoKVxudXNlci5pZF9zdHIgPSBvYmplY3QudXNlcl9pZC5tYXRj\naChcIi4qPyhcXFxcZCspXCIpWzBdXG5cbltjcmVhdGVfc2NydWJfZ2VvXVxu\nY3JlYXRlZF9hdCAgPSB0aW1lc3RhbXBNcy5kYXRlKClcbnN0YXR1cyAgICAg\nID0gdmVyYlxudXNlciAgICAgICAgPSBhY3Rvci5hcHBseShcImNyZWF0ZV9h\nY3RvclwiKVxuc2NydWJfZ2VvLnVwX3RvX3N0YXR1c19pZCA9IHRhcmdldC51\ncF90b19pZC5tYXRjaChcIi4qPyhcXFxcZCspXCIpWzBdXG5cbltjcmVhdGVf\nb3RoZXJfY29tcGxpYW5jZV1cbmNyZWF0ZWRfYXQgID0gdGltZXN0YW1wTXMu\nZGF0ZSgpXG5zdGF0dXMgICAgICA9IHZlcmJcbnVzZXIgICAgICAgID0gb2Jq\nZWN0LmFwcGx5KFwiY3JlYXRlX2FjdG9yXCIpXG53aXRoaGVsZC5jb3VudHJp\nZXMgPSB3aXRoaGVsZF9pbl9jb3VudHJpZXNcblxuW2NyZWF0ZV9nZW9fZnJv\nbV9wb2x5XVxubGF0aXR1ZGUgICAgPSBnZW8uY29vcmRpbmF0ZXNbMF0uZXh0\ncmFjdCh0aGlzWzFdKS5hdmVyYWdlKClcbmxvbmdpdHVkZSAgID0gZ2VvLmNv\nb3JkaW5hdGVzWzBdLmV4dHJhY3QodGhpc1swXSkuYXZlcmFnZSgpXG5cbltj\ncmVhdGVfZ2VvX2Zyb21fcG9pbnRdXG5sYXRpdHVkZSAgICA9IGdlby5jb29y\nZGluYXRlc1swXVxubG9uZ2l0dWRlICAgPSBnZW8uY29vcmRpbmF0ZXNbMV1c\nblxuW2NyZWF0ZV9sb2NfZ2VvX2Zyb21fcG9seV1cbmxhdGl0dWRlICAgID0g\nbG9jYXRpb24uZ2VvLmNvb3JkaW5hdGVzWzBdLmV4dHJhY3QodGhpc1sxXSku\nYXZlcmFnZSgpXG5sb25naXR1ZGUgICA9IGxvY2F0aW9uLmdlby5jb29yZGlu\nYXRlc1swXS5leHRyYWN0KHRoaXNbMF0pLmF2ZXJhZ2UoKVxuXG5bY3JlYXRl\nX2xvY19nZW9fZnJvbV9wb2ludF1cbmxhdGl0dWRlICAgID0gbG9jYXRpb24u\nZ2VvLmNvb3JkaW5hdGVzWzBdXG5sb25naXR1ZGUgICA9IGxvY2F0aW9uLmdl\nby5jb29yZGluYXRlc1sxXVxuXG5bY3JlYXRlX3Bvc3RdXG5pZCAgICAgICAg\nICAgICAgPSBpZC5zcGxpdChcIjpcIilbMl1cbmxpbmsgICAgICAgICAgICA9\nIGxpbmtcbnVzZXIgICAgICAgICAgICA9IGFjdG9yLmFwcGx5KFwiY3JlYXRl\nX2FjdG9yXCIpXG5sYW5nICAgICAgICAgICAgPSB0d2l0dGVyX2xhbmdcbnRl\neHQgICAgICAgICAgICA9IGJvZHkuc2l6ZSgxMDAwKVxuY3JlYXRlZF9hdCAg\nICAgID0gcG9zdGVkVGltZS5kYXRlKClcbmZpbHRlcl9sZXZlbCAgICA9IHR3\naXR0ZXJfZmlsdGVyX2xldmVsXG5zb3VyY2UgICAgICAgICAgPSBnZW5lcmF0\nb3IuZGlzcGxheU5hbWVcbmhhc2h0YWdzICAgICAgICA9IHR3aXR0ZXJfZW50\naXRpZXMuaGFzaHRhZ3MgWyB0cnVuY2F0ZWQgIT0gdHJ1ZSBdIC5leHRyYWN0\nKHRleHQpXG5tZW50aW9ucyAgICAgICAgPSB0d2l0dGVyX2VudGl0aWVzLnVz\nZXJfbWVudGlvbnMuZXh0cmFjdChzY3JlZW5fbmFtZSlcbm1lbnRpb25faWRz\nICAgICA9IHR3aXR0ZXJfZW50aXRpZXMudXNlcl9tZW50aW9ucy5leHRyYWN0\nKGlkKVxubWVkaWEgICAgICAgICAgID0gdHdpdHRlcl9lbnRpdGllcy5tZWRp\nYS5leHRyYWN0KGJsYWNrbGlzdChcImluZGljZXNcIikpXG5wbGFjZSAgICAg\nICAgICAgPSBsb2NhdGlvbi5hcHBseShcImNyZWF0ZV9wbGFjZVwiKVxuc3lt\nYm9scyAgICAgICAgID0gc3ltYm9scy5leHRyYWN0KHRleHQpXG5nZW8gICAg\nICAgICAgICAgPSAoYXBwbHkoXCJjcmVhdGVfZ2VvX2Zyb21fcG9pbnRcIikg\nICAgIFsgcm9vdC5nZW8udHlwZSA9PSBcIlBvaW50XCIgXSAgICAgICAgICAg\nfCAjIHRyeSB0aGUgYWNjdXJhdGUgcG9pbnRcbiAgICAgICAgICAgICAgICAg\nICAgICAgYXBwbHkoXCJjcmVhdGVfZ2VvX2Zyb21fcG9seVwiKSAgICAgIFsg\ncm9vdC5nZW8udHlwZSA9PSBcIlBvbHlnb25cIiBdICAgICAgICAgfCAjIHRy\neSBhbiBhY2N1cmF0ZSBwb2x5XG4gICAgICAgICAgICAgICAgICAgICAgICAg\nICAgICAgICAgICAgICBhcHBseShcImNyZWF0ZV9sb2NfZ2VvX2Zyb21fcG9p\nbnRcIikgWyByb290LmxvY2F0aW9uLmdlby50eXBlID09IFwiUG9pbnRcIiBd\nICB8ICMgdHJ5IHRoZSBsb2NhdGlvbiBwb2ludFxuICAgICAgICAgICAgICAg\nICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFwcGx5\nKFwiY3JlYXRlX2xvY19nZW9fZnJvbV9wb2x5XCIpICBbIHJvb3QubG9jYXRp\nb24uZ2VvLnR5cGUgPT0gXCJQb2x5Z29uXCIgXSkgIyB0cnkgdGhlIGxvY2F0\naW9uIHBvbHlcblxuIyBzcGxpdCBsaW5rcyBsaWtlIGh0dHA6XC9cL3R3aXR0\nZXIuY29tXC9zY3JlZW5fbmFtZVwvc3RhdHVzZXNcL3N0YXR1c19pZFxuaW5f\ncmVwbHlfdG9fc2NyZWVuX25hbWUgICA9IGluUmVwbHlUby5saW5rLnNwbGl0\nKFwiXFxcL1wiKSBbM11cbmluX3JlcGx5X3RvX3N0YXR1c19pZCAgICAgPSBp\nblJlcGx5VG8ubGluay5zcGxpdChcIlxcXC9cIikgWzVdXG5cbiMgZXh0cmFj\ndCBsaW5rcyBmcm9tIHR3aXR0ZXIgZW50aXRpZXNcbmxpbmtzICAgICAgICAg\nICA9IHR3aXR0ZXJfZW50aXRpZXMudXJscy5leHRyYWN0KChleHBhbmRlZF91\ncmwgfCB1cmwpKVswOjVdXG5kb21haW5zICAgICAgICAgPSB0d2l0dGVyX2Vu\ndGl0aWVzLnVybHMuZXh0cmFjdCgoZXhwYW5kZWRfdXJsLnVybCgpLmhvc3Qg\nfCB1cmwudXJsKCkuaG9zdCkpWzA6NV1cbmRpc3BsYXlfdXJscyAgICA9IHR3\naXR0ZXJfZW50aXRpZXMudXJscy5leHRyYWN0KGRpc3BsYXlfdXJsKVswOjVd\nXG5cbltjcmVhdGVfc2hhcmVdXG5yZXR3ZWV0ICAgICA9IGFwcGx5KFwiY3Jl\nYXRlX3Bvc3RcIilcbnJldHdlZXRlZCAgID0gb2JqZWN0LmFwcGx5KFwiY3Jl\nYXRlX3Bvc3RcIilcbmlkICAgICAgICAgID0gaWQuc3BsaXQoXCI6XCIpWzJd\nXG5cbiMgd2UgY2FuIGdldCB0aGUgb3JkZXIgdGhlIHJldHdlZXQgd2FzIG1h\nZGUgaW4gYnkgZ2V0dGluZyB0aGUgb3JpZ2luYWwgdHdlZXRzIHJldHdlZXQg\nY291bnRcbnJldHdlZXQuY291bnQgPSByZXR3ZWV0Q291bnQuaW50KCkuZGVm\nYXVsdCgwKVxuXG4jIHRoZSByZXR3ZWV0IGlzIG9mdGVuIHRydW5jYXRlZCBi\neSB0d2l0dGVyLiBsZXQncyBmb3JtYXQgaXQgb3Vyc2VsZlxucmV0d2VldC50\nZXh0ID0gXCJSVCBAJXM6ICVzXCIuZm9ybWF0KG9iamVjdC5hY3Rvci5wcmVm\nZXJyZWRVc2VybmFtZSwgb2JqZWN0LmJvZHkpXG5cbiMgd2UgZG9uJ3QgbWFw\nIHRoZSBmYXZvcml0ZXMgY291bnQgb2YgYSBuZXcgdHdlZXQgc2luY2UgaXQn\nbGwgbGlrZWx5IGJlIHplcm9cbnJldHdlZXRlZC5mYXZvdXJpdGVzX2NvdW50\nID0gb2JqZWN0LmZhdm9yaXRlc0NvdW50XG5cbiMgcHJlcGVuZCB0aGUgcmV0\nd2VldGVkIHVzZXIgc2NyZWVuIG5hbWUgYW5kIGlkIHRvIHRoZSBtZW50aW9u\ncyBsaXN0XG5yZXR3ZWV0Lm1lbnRpb25zID0gQHJldHdlZXQubWVudGlvbnMu\ncHJlcGVuZChAcmV0d2VldGVkLnVzZXIuc2NyZWVuX25hbWUpXG5yZXR3ZWV0\nLm1lbnRpb25faWRzID0gQHJldHdlZXQubWVudGlvbl9pZHMucHJlcGVuZChA\ncmV0d2VldGVkLnVzZXIuaWQpXG5cbltjcmVhdGVfcGxhY2VdXG5jb3VudHJ5\nICAgICAgICAgPSBjb3VudHJ5X2NvZGVcbmNvdW50cnlfY29kZSAgICA9IHR3\naXR0ZXJfY291bnRyeV9jb2RlXG5mdWxsX25hbWUgICAgICAgPSBkaXNwbGF5\nTmFtZVxubmFtZSAgICAgICAgICAgID0gbmFtZVxucGxhY2VfdHlwZSAgICAg\nID0gdHdpdHRlcl9wbGFjZV90eXBlXG51cmwgICAgICAgICAgICAgPSBsaW5r\nXG5cbltjcmVhdGVfYWN0b3JdXG5pZCAgICAgICAgICAgICAgICAgICAgICA9\nIGlkLm1hdGNoKFwiLio/KFxcXFxkKylcIilbMF0uaW50KClcbmlkX3N0ciAg\nICAgICAgICAgICAgICAgID0gaWQubWF0Y2goXCIuKj8oXFxcXGQrKVwiKVsw\nXVxudXJsICAgICAgICAgICAgICAgICAgICAgPSBsaW5rXG5uYW1lICAgICAg\nICAgICAgICAgICAgICA9IGRpc3BsYXlOYW1lXG5wcm9maWxlX2ltYWdlX3Vy\nbCAgICAgICA9IGltYWdlXG5wcm9maWxlX2ltYWdlX3VybF9odHRwcyA9IGlt\nYWdlXG5kZXNjcmlwdGlvbiAgICAgICAgICAgICA9IHN1bW1hcnlcbmNyZWF0\nZWRfYXQgICAgICAgICAgICAgID0gcG9zdGVkVGltZS5kYXRlKClcbmxvY2F0\naW9uICAgICAgICAgICAgICAgID0gbG9jYXRpb24uZGlzcGxheU5hbWVcbnV0\nY19vZmZzZXQgICAgICAgICAgICAgID0gdXRjT2Zmc2V0LmludCgpXG5zY3Jl\nZW5fbmFtZSAgICAgICAgICAgICA9IHByZWZlcnJlZFVzZXJuYW1lXG5sYW5n\nICAgICAgICAgICAgICAgICAgICA9IGxhbmd1YWdlc1swXVxudGltZV96b25l\nICAgICAgICAgICAgICAgPSB0d2l0dGVyVGltZVpvbmVcbmZyaWVuZHNfY291\nbnQgICAgICAgICAgID0gZnJpZW5kc0NvdW50XG5mb2xsb3dlcnNfY291bnQg\nICAgICAgICA9IGZvbGxvd2Vyc0NvdW50XG5saXN0ZWRfY291bnQgICAgICAg\nICAgICA9IGxpc3RlZENvdW50XG5zdGF0dXNlc19jb3VudCAgICAgICAgICA9\nIHN0YXR1c2VzQ291bnRcbnZlcmlmaWVkICAgICAgICAgICAgICAgID0gdmVy\naWZpZWRcbmZhdm91cml0ZXNfY291bnQgICAgICAgID0gZmF2b3JpdGVzQ291\nbnQiLCJjYXRlZ29yeSI6InR3aXR0ZXJjYXRlZ29yeV8xIn19XSwicGFyYW1l\ndGVycyI6W10sImNyZWF0ZWRfYXQiOjE0Mzk1ODA0NTZ9\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:36 GMT"}],"recorded_with":"VCR 2.9.3"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"http_interactions":[{"request":{"method":"post","uri":"https://in.datasift.com/invalid_source_id","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/ Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["27255"]}},"response":{"status":{"code":404,"message":"Not Found"},"headers":{"Cache-Control":["private"],"Content-Type":["application/json"],"Server":["DataSift Ingestion/1.0"],"X-Ingestion-Data-Ratelimit-Reset":["0"],"X-Ingestion-Request-Ratelimit-Limit":["10000"],"X-Ingestion-Request-Ratelimit-Reset":["60"],"X-Ingestion-Data-Ratelimit-Remaining":["36864000"],"X-Ingestion-Request-Ratelimit-Reset-Ttl":["1439580312"],"X-Ingestion-Data-Ratelimit-Reset-Ttl":["1439580252"],"X-Ingestion-Data-Ratelimit-Limit":["36864000"],"X-Ingestion-Request-Ratelimit-Remaining":["9999"],"Transfer-Encoding":["chunked"],"Strict-Transport-Security":["max-age=31536000"]},"body":{"encoding":"ASCII-8BIT","base64_string":"eyJlcnJvciI6IlVua25vd24gdXNlciBzb3VyY2UifQ==\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:24:12 GMT"}],"recorded_with":"VCR 2.9.3"}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"http_interactions":[{"request":{"method":"post","uri":"https://in.datasift.com/3446d92fd7274adeaf8b27f4e99f3b60","body":{"encoding":"UTF-8","base64_string":"PEJBU0U2NF9TVFJJTkc+\n"},"headers":{"Accept":["*/*; q=0.5, application/xml"],"Accept-Encoding":["gzip, deflate"],"User-Agent":["DataSift/ Ruby/v3.4.0"],"Authorization":["<USERNAME:API_KEY>"],"Content-Type":["application/json"],"Content-Length":["27255"]}},"response":{"status":{"code":200,"message":"OK"},"headers":{"Cache-Control":["private"],"Content-Type":["application/json"],"Server":["DataSift Ingestion/1.0"],"X-Ingestion-Data-Ratelimit-Reset":["60"],"X-Ingestion-Request-Ratelimit-Limit":["10000"],"X-Ingestion-Request-Ratelimit-Reset":["60"],"X-Ingestion-Data-Ratelimit-Remaining":["36836754"],"X-Ingestion-Request-Ratelimit-Reset-Ttl":["1439580536"],"X-Ingestion-Data-Ratelimit-Reset-Ttl":["1439580536"],"X-Ingestion-Data-Ratelimit-Limit":["36864000"],"X-Ingestion-Request-Ratelimit-Remaining":["9999"],"Transfer-Encoding":["chunked"],"Strict-Transport-Security":["max-age=31536000"]},"body":{"encoding":"ASCII-8BIT","base64_string":"eyJhY2NlcHRlZCI6MTAsInRvdGFsX21lc3NhZ2VfYnl0ZXMiOjI3MjQ2fQ==\n"},"http_version":null},"recorded_at":"Fri, 14 Aug 2015 19:27:56 GMT"}],"recorded_with":"VCR 2.9.3"}
|
@@ -0,0 +1,10 @@
|
|
1
|
+
{"id":"tag:search.twitter.com,2005:587648815764746240","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:36136637","link":"http://www.twitter.com/davidmarkwales","displayName":"Dave Thomas","postedTime":"2009-04-28T18:14:12.000Z","image":"https://pbs.twimg.com/profile_images/1173129755/image_normal.jpg","summary":"Instagram: davidmarkwales","links":[{"href":null,"rel":"me"}],"friendsCount":340,"followersCount":122,"listedCount":3,"statusesCount":499,"twitterTimeZone":"London","verified":false,"utcOffset":"3600","preferredUsername":"davidmarkwales","languages":["en"],"location":{"objectType":"place","displayName":"Liverpool"},"favoritesCount":33},"verb":"post","postedTime":"2015-04-13T16:09:30.000Z","generator":{"displayName":"Twitter for Android","link":"http://twitter.com/download/android"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/davidmarkwales/statuses/587648815764746240","body":"@asda has cancelled my home delivery today, no reason given, not even a courtesy call or email to let me know. Any idea why they would?","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648815764746240","summary":"@asda has cancelled my home delivery today, no reason given, not even a courtesy call or email to let me know. Any idea why they would?","link":"http://twitter.com/davidmarkwales/statuses/587648815764746240","postedTime":"2015-04-13T16:09:30.000Z"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Garston, England","name":"Garston","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/12f69ad404352073.json","geo":{"type":"Polygon","coordinates":[[[-2.940659,53.331287],[-2.940659,53.38523],[-2.818002,53.38523],[-2.818002,53.331287]]]}},"geo":{"type":"Point","coordinates":[53.345841,-2.87278]},"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"asda","name":"Asda","id":20123366,"id_str":"20123366","indices":[0,5]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
|
2
|
+
{"id":"tag:search.twitter.com,2005:587648834689445889","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:3157665856","link":"http://www.twitter.com/emilywlittle","displayName":"Emily ","postedTime":"2015-04-11T20:20:40.000Z","image":"https://pbs.twimg.com/profile_images/586987964225445912/ubKhVysy_normal.jpg","summary":null,"links":[{"href":null,"rel":"me"}],"friendsCount":61,"followersCount":20767,"listedCount":0,"statusesCount":128,"twitterTimeZone":"Amsterdam","verified":false,"utcOffset":"7200","preferredUsername":"emilywlittle","languages":["en"],"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:09:34.000Z","generator":{"displayName":"Yappem","link":"http://www.yappem.com/"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/emilywlittle/statuses/587648834689445889","body":"@amazon online bookstore http://t.co/Cz3XUWDvva http://t.co/6lNnsbABii","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648834689445889","summary":"@amazon online bookstore http://t.co/Cz3XUWDvva http://t.co/6lNnsbABii","link":"http://twitter.com/emilywlittle/statuses/587648834689445889","postedTime":"2015-04-13T16:09:34.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[{"url":"http://t.co/Cz3XUWDvva","expanded_url":"http://goo.gl/iXPwB4","display_url":"goo.gl/iXPwB4","indices":[25,47]}],"user_mentions":[{"screen_name":"amazon","name":"Amazon","id":20793816,"id_str":"20793816","indices":[0,7]}],"symbols":[],"media":[{"id":587648834580455424,"id_str":"587648834580455424","indices":[48,70],"media_url":"http://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","media_url_https":"https://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","url":"http://t.co/6lNnsbABii","display_url":"pic.twitter.com/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","type":"photo","sizes":{"small":{"w":340,"h":340,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":600,"h":600,"resize":"fit"},"large":{"w":600,"h":600,"resize":"fit"}}}]},"twitter_extended_entities":{"media":[{"id":587648834580455424,"id_str":"587648834580455424","indices":[48,70],"media_url":"http://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","media_url_https":"https://pbs.twimg.com/media/CCe_gGXXIAAWYU6.jpg","url":"http://t.co/6lNnsbABii","display_url":"pic.twitter.com/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","type":"photo","sizes":{"small":{"w":340,"h":340,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":600,"h":600,"resize":"fit"},"large":{"w":600,"h":600,"resize":"fit"}}}]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"urls":[{"url":"http://t.co/Cz3XUWDvva","expanded_url":"https://www.yappem.com/redirect/?pid=782008","expanded_status":403},{"url":"http://t.co/6lNnsbABii","expanded_url":"http://twitter.com/emilywlittle/status/587648834689445889/photo/1","expanded_status":200}],"language":{"value":"en"}}}
|
3
|
+
{"id":"tag:search.twitter.com,2005:587648869485379585","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:265357386","link":"http://www.twitter.com/dr_acula365","displayName":"Andy","postedTime":"2011-03-13T12:16:17.000Z","image":"https://pbs.twimg.com/profile_images/2438808933/to47tp1pbrww1gqwbe9k_normal.jpeg","summary":"Just a guy who likes sport, music and food. You'll find me in the LGBT thread on f365 forum. Please don't follow me if you are under 18. You will be blocked.","links":[{"href":null,"rel":"me"}],"friendsCount":185,"followersCount":128,"listedCount":3,"statusesCount":916,"twitterTimeZone":"Casablanca","verified":false,"utcOffset":"3600","preferredUsername":"dr_acula365","languages":["en"],"location":{"objectType":"place","displayName":"Newcastle"},"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:09:43.000Z","generator":{"displayName":"Tweetbot for iΟS","link":"http://tapbots.com/tweetbot"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dr_acula365/statuses/587648869485379585","body":"@Tesco when your website says next day click and collect and the email says pick up from 3pm, when do you actually mean I can pick it up?","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648869485379585","summary":"@Tesco when your website says next day click and collect and the email says pick up from 3pm, when do you actually mean I can pick it up?","link":"http://twitter.com/dr_acula365/statuses/587648869485379585","postedTime":"2015-04-13T16:09:43.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Tesco","name":"Tesco","id":271986064,"id_str":"271986064","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
|
4
|
+
{"id":"tag:search.twitter.com,2005:587648932337098752","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:2642049675","link":"http://www.twitter.com/michaeljohnmcr","displayName":"Michael Delaney","postedTime":"2014-06-25T22:59:21.000Z","image":"https://pbs.twimg.com/profile_images/544998090609291264/W-4vCwC7_normal.jpeg","summary":"邁克爾 ❌❌","links":[{"href":null,"rel":"me"}],"friendsCount":48,"followersCount":19,"listedCount":0,"statusesCount":302,"twitterTimeZone":"Azores","verified":false,"utcOffset":"0","preferredUsername":"michaeljohnmcr","languages":["en"],"favoritesCount":10},"verb":"post","postedTime":"2015-04-13T16:09:58.000Z","generator":{"displayName":"Twitter for iPhone","link":"http://twitter.com/download/iphone"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/michaeljohnmcr/statuses/587648932337098752","body":"@asda I need a number to make a complaint with in the uk can't find it online","object":{"objectType":"note","id":"object:search.twitter.com,2005:587648932337098752","summary":"@asda I need a number to make a complaint with in the uk can't find it online","link":"http://twitter.com/michaeljohnmcr/statuses/587648932337098752","postedTime":"2015-04-13T16:09:58.000Z"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Manchester, England","name":"Manchester","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/315b740b108481f6.json","geo":{"type":"Polygon","coordinates":[[[-2.319934,53.343623],[-2.319934,53.5702824],[-2.147026,53.5702824],[-2.147026,53.343623]]]}},"geo":{"type":"Point","coordinates":[53.38604,-2.251666]},"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"asda","name":"Asda","id":20123366,"id_str":"20123366","indices":[0,5]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
|
5
|
+
{"id":"tag:search.twitter.com,2005:587649014415278080","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:14577165","link":"http://www.twitter.com/marinmaven","displayName":"marinmaven","postedTime":"2008-04-28T23:11:01.000Z","image":"https://pbs.twimg.com/profile_images/489833602222661632/dKsJXZaX_normal.jpeg","summary":"I am a San Francisco native who grew up on the Peninsula. I muse about politics, economics, culture, and living in Marin and beyond.","links":[{"href":"http://www.marinmaven.com/","rel":"me"}],"friendsCount":2316,"followersCount":2360,"listedCount":123,"statusesCount":19661,"twitterTimeZone":"Pacific Time (US & Canada)","verified":false,"utcOffset":"-25200","preferredUsername":"marinmaven","languages":["en"],"location":{"objectType":"place","displayName":"ÜT: 38.006898,-122.531684"},"favoritesCount":355},"verb":"post","postedTime":"2015-04-13T16:10:17.000Z","generator":{"displayName":"Twitter for iPhone","link":"http://twitter.com/download/iphone"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/marinmaven/statuses/587649014415278080","body":"@Macys I have sent you an email detailing what has happened.","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649014415278080","summary":"@Macys I have sent you an email detailing what has happened.","link":"http://twitter.com/marinmaven/statuses/587649014415278080","postedTime":"2015-04-13T16:10:17.000Z"},"inReplyTo":{"link":"http://twitter.com/Macys/statuses/587641764682403841"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Macys","name":"Macy's","id":50687788,"id_str":"50687788","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
|
6
|
+
{"id":"tag:search.twitter.com,2005:587649047697215488","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:265357386","link":"http://www.twitter.com/dr_acula365","displayName":"Andy","postedTime":"2011-03-13T12:16:17.000Z","image":"https://pbs.twimg.com/profile_images/2438808933/to47tp1pbrww1gqwbe9k_normal.jpeg","summary":"Just a guy who likes sport, music and food. You'll find me in the LGBT thread on f365 forum. Please don't follow me if you are under 18. You will be blocked.","links":[{"href":null,"rel":"me"}],"friendsCount":185,"followersCount":128,"listedCount":3,"statusesCount":917,"twitterTimeZone":"Casablanca","verified":false,"utcOffset":"3600","preferredUsername":"dr_acula365","languages":["en"],"location":{"objectType":"place","displayName":"Newcastle"},"favoritesCount":0},"verb":"post","postedTime":"2015-04-13T16:10:25.000Z","generator":{"displayName":"Tweetbot for iΟS","link":"http://tapbots.com/tweetbot"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dr_acula365/statuses/587649047697215488","body":"@Tesco the store has no idea where my order will be and suggests I go back tomorrow as it \"might\" be with them by then.","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649047697215488","summary":"@Tesco the store has no idea where my order will be and suggests I go back tomorrow as it \"might\" be with them by then.","link":"http://twitter.com/dr_acula365/statuses/587649047697215488","postedTime":"2015-04-13T16:10:25.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Tesco","name":"Tesco","id":271986064,"id_str":"271986064","indices":[0,6]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
|
7
|
+
{"id":"tag:search.twitter.com,2005:587649057914490880","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:509474447","link":"http://www.twitter.com/DebbieMeldrum","displayName":"Debstweeted","postedTime":"2012-03-01T00:24:28.000Z","image":"https://pbs.twimg.com/profile_images/483673950602682368/dsKmu4AK_normal.jpeg","summary":"Professional. Linguist. Fashion follower. Juice addict. Day dreamer. Number 11.","links":[{"href":null,"rel":"me"}],"friendsCount":677,"followersCount":132,"listedCount":3,"statusesCount":323,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"DebbieMeldrum","languages":["en"],"location":{"objectType":"place","displayName":"Aberdeen"},"favoritesCount":337},"verb":"post","postedTime":"2015-04-13T16:10:28.000Z","generator":{"displayName":"Twitter for iPad","link":"http://twitter.com/#!/download/ipad"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/DebbieMeldrum/statuses/587649057914490880","body":"@Topshop really want to but unfortunately my local Topshop in #Aberdeen doesn't stock it 😟","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649057914490880","summary":"@Topshop really want to but unfortunately my local Topshop in #Aberdeen doesn't stock it 😟","link":"http://twitter.com/DebbieMeldrum/statuses/587649057914490880","postedTime":"2015-04-13T16:10:28.000Z"},"inReplyTo":{"link":"http://twitter.com/Topshop/statuses/587648638467366912"},"favoritesCount":0,"location":{"objectType":"place","displayName":"Aberdeen, Scotland","name":"Aberdeen","country_code":"United Kingdom","twitter_country_code":"GB","link":"https://api.twitter.com/1.1/geo/id/73cc26d418860ddd.json","geo":{"type":"Polygon","coordinates":[[[-2.1899763,57.107695],[-2.1899763,57.2134174],[-2.0641046,57.2134174],[-2.0641046,57.107695]]]}},"geo":{"type":"Point","coordinates":[57.170311,-2.094876]},"twitter_entities":{"hashtags":[{"text":"Aberdeen","indices":[62,71]}],"trends":[],"urls":[],"user_mentions":[{"screen_name":"Topshop","name":"Topshop","id":20908109,"id_str":"20908109","indices":[0,8]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"language":{"value":"en"}}}
|
8
|
+
{"id":"tag:search.twitter.com,2005:587649092089733120","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:274108611","link":"http://www.twitter.com/IAmKatieDevs","displayName":"Katie Devlin","postedTime":"2011-03-29T19:02:59.000Z","image":"https://pbs.twimg.com/profile_images/454566612080033793/du2a4BhY_normal.jpeg","summary":"Boston bred, fashion fanatic, music enthusiast, IBMer. {{MY tweets are MY own}}","links":[{"href":null,"rel":"me"}],"friendsCount":382,"followersCount":234,"listedCount":10,"statusesCount":1673,"twitterTimeZone":"Quito","verified":false,"utcOffset":"-18000","preferredUsername":"IAmKatieDevs","languages":["en"],"location":{"objectType":"place","displayName":"Boston, MA"},"favoritesCount":263},"verb":"post","postedTime":"2015-04-13T16:10:36.000Z","generator":{"displayName":"Hootsuite","link":"http://www.hootsuite.com"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/IAmKatieDevs/statuses/587649092089733120","body":"@BedBathBeyond marries wedding registry with augmented reality http://t.co/0bojF4HY12 #mobile #mcommerce","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649092089733120","summary":"@BedBathBeyond marries wedding registry with augmented reality http://t.co/0bojF4HY12 #mobile #mcommerce","link":"http://twitter.com/IAmKatieDevs/statuses/587649092089733120","postedTime":"2015-04-13T16:10:36.000Z"},"favoritesCount":0,"twitter_entities":{"hashtags":[{"text":"mobile","indices":[86,93]},{"text":"mcommerce","indices":[94,104]}],"trends":[],"urls":[{"url":"http://t.co/0bojF4HY12","expanded_url":"http://www.mobilecommercedaily.com/bed-bath-and-beyond-marries-wedding-registry-with-augmented-reality","display_url":"mobilecommercedaily.com/bed-bath-and-b…","indices":[63,85]}],"user_mentions":[{"screen_name":"BedBathBeyond","name":"Bed Bath & Beyond","id":372817561,"id_str":"372817561","indices":[0,14]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"}],"urls":[{"url":"http://t.co/0bojF4HY12","expanded_url":"http://www.mobilecommercedaily.com/bed-bath-and-beyond-marries-wedding-registry-with-augmented-reality","expanded_status":200}],"language":{"value":"en"}}}
|
9
|
+
{"id":"tag:search.twitter.com,2005:587649207844081664","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:944015658","link":"http://www.twitter.com/dianekendall716","displayName":"diane kendall","postedTime":"2012-11-12T16:59:18.000Z","image":"https://pbs.twimg.com/profile_images/534077775309975553/b0Z9SWCF_normal.jpeg","summary":null,"links":[{"href":null,"rel":"me"}],"friendsCount":24,"followersCount":10,"listedCount":0,"statusesCount":34,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"dianekendall716","languages":["en"],"favoritesCount":63},"verb":"post","postedTime":"2015-04-13T16:11:03.000Z","generator":{"displayName":"Mobile Web (M5)","link":"https://mobile.twitter.com"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/dianekendall716/statuses/587649207844081664","body":"@marksandspencer black please","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649207844081664","summary":"@marksandspencer black please","link":"http://twitter.com/dianekendall716/statuses/587649207844081664","postedTime":"2015-04-13T16:11:03.000Z"},"inReplyTo":{"link":"http://twitter.com/marksandspencer/statuses/587541076187090944"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"marksandspencer","name":"M&S","id":37916167,"id_str":"37916167","indices":[0,16]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"BedBathBeyond\" OR to:\"WilliamsSonoma\" OR to:\"GAP\" OR to:\"Abercrombie\" OR to:\"hm\" OR to:\"VictoriasSecret\" OR to:\"LLBean\" OR to:\"LandsEnd\" OR to:\"eBay\" OR to:\"Overstock_help\" OR to:\"Overstock\" OR to:\"QVC\" OR to:\"BandQ\" OR to:\"asda\" OR to:\"waitrose\" OR to:\"LidlUK\" OR to:\"Ocado\" OR to:\"Burberry\" OR to:\"marksandspencer\" OR to:\"johnlewisretail\" OR to:\"nextofficial\" OR to:\"ASOS\" OR to:\"Topshop\" OR to:\"ZARA\" OR to:\"NewLookFashion\")","tag":"retailers"},{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
|
10
|
+
{"id":"tag:search.twitter.com,2005:587649264156811264","objectType":"activity","actor":{"objectType":"person","id":"id:twitter.com:920532930","link":"http://www.twitter.com/karengodfrey06","displayName":"Karen Godfrey-Young","postedTime":"2012-11-02T07:26:39.000Z","image":"https://pbs.twimg.com/profile_images/579378574119051264/NutpKf5S_normal.jpg","summary":"Wife/ Mother / Grandma","links":[{"href":null,"rel":"me"}],"friendsCount":89,"followersCount":61,"listedCount":0,"statusesCount":1771,"twitterTimeZone":null,"verified":false,"utcOffset":null,"preferredUsername":"karengodfrey06","languages":["en"],"location":{"objectType":"place","displayName":" Oxfordshire"},"favoritesCount":1383},"verb":"post","postedTime":"2015-04-13T16:11:17.000Z","generator":{"displayName":"Twitter for Android","link":"http://twitter.com/download/android"},"provider":{"objectType":"service","displayName":"Twitter","link":"http://www.twitter.com"},"link":"http://twitter.com/karengodfrey06/statuses/587649264156811264","body":"@AldiUK nothing is happening. Opens to blank white page? \nI shop with you every week so would love to enter comp xx","object":{"objectType":"note","id":"object:search.twitter.com,2005:587649264156811264","summary":"@AldiUK nothing is happening. Opens to blank white page? \nI shop with you every week so would love to enter comp xx","link":"http://twitter.com/karengodfrey06/statuses/587649264156811264","postedTime":"2015-04-13T16:11:17.000Z"},"inReplyTo":{"link":"http://twitter.com/AldiUK/statuses/587623827787493376"},"favoritesCount":0,"twitter_entities":{"hashtags":[],"trends":[],"urls":[],"user_mentions":[{"screen_name":"AldiUK","name":"Aldi Stores UK","id":28596803,"id_str":"28596803","indices":[0,7]}],"symbols":[]},"twitter_filter_level":"low","twitter_lang":"en","retweetCount":0,"gnip":{"matching_rules":[{"value":"lang:en (to:\"amazon\" OR to:\"amazonhelp\" OR to:\"BestBuy\" OR to:\"BestBuySupport\" OR to:\"jcpenney\" OR to:\"Kohls\" OR to:\"kroger\" OR to:\"Macys\" OR to:\"Sears\" OR to:\"searscares\" OR to:\"ToysRUs\" OR to:\"Walmart\" OR to:\"Target\" OR to:\"costco\" OR to:\"publix\" OR to:\"publixhelp\" OR to:\"Nordstrom\" OR to:\"neimanmarcus\" OR to:\"marksandspencer\" OR to:\"tesco\" OR to:\"AldiUK\" OR to:\"AldiUSA\" OR to:\"LoblawsON\" OR to:\"safeway\" OR to:\"WholeFoods\" OR to:\"sainsburys\" OR to:\"woolworths\" OR to:\"TrueValue\" OR to:\"IKEAUSA\")","tag":"retailers"}],"language":{"value":"en"}}}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datasift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- DataSift
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2015-08-
|
13
|
+
date: 2015-08-20 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rest-client
|
@@ -18,14 +18,14 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - "~>"
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: '1.
|
21
|
+
version: '1.8'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
26
|
- - "~>"
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version: '1.
|
28
|
+
version: '1.8'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: multi_json
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
@@ -77,6 +77,8 @@ extensions: []
|
|
77
77
|
extra_rdoc_files: []
|
78
78
|
files:
|
79
79
|
- ".gitignore"
|
80
|
+
- ".rubocop.yml"
|
81
|
+
- ".rubocop_todo.yml"
|
80
82
|
- ".travis.yml"
|
81
83
|
- ".yardopts"
|
82
84
|
- CHANGELOG.md
|
@@ -95,6 +97,7 @@ files:
|
|
95
97
|
- examples/core_api_eg.rb
|
96
98
|
- examples/historics_eg.rb
|
97
99
|
- examples/historics_preview_eg.rb
|
100
|
+
- examples/ingestion/twitter_gnip_batch_eg.rb
|
98
101
|
- examples/live_stream_eg.rb
|
99
102
|
- examples/managed_source_fb_eg.rb
|
100
103
|
- examples/managed_source_ig_eg.rb
|
@@ -115,10 +118,12 @@ files:
|
|
115
118
|
- lib/managed_source.rb
|
116
119
|
- lib/managed_source_auth.rb
|
117
120
|
- lib/managed_source_resource.rb
|
121
|
+
- lib/odp.rb
|
118
122
|
- lib/push.rb
|
119
123
|
- lib/pylon.rb
|
120
124
|
- test/datasift/core_api_test.rb
|
121
125
|
- test/datasift/historics_preview_api_test.rb
|
126
|
+
- test/datasift/odp/batch_upload_test.rb
|
122
127
|
- test/datasift/push_api_test.rb
|
123
128
|
- test/fixtures/cassettes/core/after_historic_dpu.json
|
124
129
|
- test/fixtures/cassettes/core/balance_get.json
|
@@ -132,6 +137,10 @@ files:
|
|
132
137
|
- test/fixtures/cassettes/core/validate_invalid_hash.json
|
133
138
|
- test/fixtures/cassettes/core/validate_success_bool.json
|
134
139
|
- test/fixtures/cassettes/core/validate_success_hash.json
|
140
|
+
- test/fixtures/cassettes/odp/batch/after_upload.json
|
141
|
+
- test/fixtures/cassettes/odp/batch/before_upload.json
|
142
|
+
- test/fixtures/cassettes/odp/batch/upload_failure_no_source.json
|
143
|
+
- test/fixtures/cassettes/odp/batch/upload_success.json
|
135
144
|
- test/fixtures/cassettes/preview/before_preview_create.json
|
136
145
|
- test/fixtures/cassettes/preview/before_preview_get.json
|
137
146
|
- test/fixtures/cassettes/preview/preview_create_success.json
|
@@ -160,6 +169,7 @@ files:
|
|
160
169
|
- test/fixtures/cassettes/push/push_stop.json
|
161
170
|
- test/fixtures/cassettes/push/push_update.json
|
162
171
|
- test/fixtures/cassettes/push/push_validate.json
|
172
|
+
- test/fixtures/data/fake_gnip_tweets.json
|
163
173
|
- test/test_helper.rb
|
164
174
|
homepage: https://github.com/datasift/datasift-ruby
|
165
175
|
licenses:
|
@@ -181,7 +191,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
181
191
|
version: 1.3.5
|
182
192
|
requirements: []
|
183
193
|
rubyforge_project:
|
184
|
-
rubygems_version: 2.4.
|
194
|
+
rubygems_version: 2.4.8
|
185
195
|
signing_key:
|
186
196
|
specification_version: 4
|
187
197
|
summary: DataSift is a simple wrapper for the DataSift API.
|