gnip_api 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +14 -0
- data/lib/gnip_api/gnip/activity.rb +9 -1
- data/lib/gnip_api/version.rb +1 -1
- data/spec/fixtures/activities/real_activity_long_object.json +306 -0
- data/spec/gnip_api/gnip/activity_spec.rb +9 -0
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a4b23aa44097600cdcd2a60ed769511a39999a6a
|
|
4
|
+
data.tar.gz: b94409aca9eb2a64c653ba006a45db6c3fb3d806
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e66a5a77b304723a7a8416f6d4674426c19ae70526ea3eeeb94473ec8beaa09b5942918b02b92c933cbc948b50db10b57c6080df08b527168c88c7a500fb0dbd
|
|
7
|
+
data.tar.gz: 8c886b7b16b51c2461885a33e11f1813b5fc847acccc43e0d6cad47d1956bfb490a4a4d5ab9c964d8f026b26bac0d4dc27673c661485b1547889abf65fcbae2d
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -63,6 +63,20 @@ Note that you'll need a source and a label. Source is the data source within Gni
|
|
|
63
63
|
|
|
64
64
|
While using the Full Archived Histocial keep in mind Gnip seems to have some issues with their pagination on certain situations. You may face 503 errors which are unrecoverable from client side. If you come across this error please report the situationt to Gnip.
|
|
65
65
|
|
|
66
|
+
## Some notes
|
|
67
|
+
|
|
68
|
+
While using the Full Archive Search or FAS as we call it we faced some issues that you may encounter as
|
|
69
|
+
well if you use it. The most notorious one is the 503 "You encountered a problem in our software" which
|
|
70
|
+
is mentioned avobe. Upon troubleshooting this error, the client side "solution" or workaround to better
|
|
71
|
+
put it, is to iterate from the client the period. Instead of letting GNIP paginate the data build smaller
|
|
72
|
+
periods of time. For example, instead of requesting from year 2016 to 2017, do 12 requests of 1 month each.
|
|
73
|
+
We found that making this period size smaller and smaller ends up making it work. A higher process built
|
|
74
|
+
using this gem splits any given period in smaller ones and iterates over the data, re running missing periods
|
|
75
|
+
split further to fill in missing data. The smalles period that seems to have 100% chances of success is 1 hour.
|
|
76
|
+
If you wonder who came up with this ugly solution, the answer is GNIP itself, upon talking to their support
|
|
77
|
+
area about this. It doesn't seem proper to include this on the gem since this errors is not supposed to happen,
|
|
78
|
+
but it may eventually be included as an alternative querying to mitigate the problem.
|
|
79
|
+
|
|
66
80
|
## Overview
|
|
67
81
|
|
|
68
82
|
The Search API allows you to get counts or activities in a time period, with a maximum period size of 30 days per request. PowerTrack rules are used as query parameter, but be careful **PowerTrack operators may not be supported on Search API or could behave differently**. Read the Gnip docs to make sure.
|
|
@@ -2,7 +2,7 @@ module Gnip
|
|
|
2
2
|
class Activity < Gnip::Message
|
|
3
3
|
attr_reader :id, :object_type, :actor, :verb, :posted_time, :generator, :provider, :link,
|
|
4
4
|
:body, :object, :favorites_count, :twitter_entities, :twitter_filter_level, :twitter_lang,
|
|
5
|
-
:retweet_count, :gnip, :raw
|
|
5
|
+
:retweet_count, :gnip, :raw, :long_object, :display_text_range
|
|
6
6
|
|
|
7
7
|
def initialize params = {}
|
|
8
8
|
@raw = params
|
|
@@ -21,6 +21,8 @@ module Gnip
|
|
|
21
21
|
@twitter_filter_level = params['twitter_filter_level']
|
|
22
22
|
@twitter_lang = params['twitter_lang']
|
|
23
23
|
@retweet_count = params['retweetCount']
|
|
24
|
+
@long_object = params['long_object']
|
|
25
|
+
@display_text_range = params['display_text_range']
|
|
24
26
|
@gnip = Gnip::GnipData.new(params['gnip']) if params['gnip']
|
|
25
27
|
end
|
|
26
28
|
|
|
@@ -41,6 +43,8 @@ module Gnip
|
|
|
41
43
|
:twitter_filter_level => @twitter_filter_level,
|
|
42
44
|
:twitter_lang => @twitter_lang,
|
|
43
45
|
:retweetCount => @retweet_count,
|
|
46
|
+
:longObject => @long_object,
|
|
47
|
+
:display_text_range => @display_text_range,
|
|
44
48
|
:gnip => @gnip ? @gnip.to_h : nil
|
|
45
49
|
}
|
|
46
50
|
end
|
|
@@ -68,6 +72,10 @@ module Gnip
|
|
|
68
72
|
def retweet?
|
|
69
73
|
verb == 'share'
|
|
70
74
|
end
|
|
75
|
+
|
|
76
|
+
def hidden_data?
|
|
77
|
+
!@display_text_range.nil? && !@long_object.nil?
|
|
78
|
+
end
|
|
71
79
|
end
|
|
72
80
|
end
|
|
73
81
|
|
data/lib/gnip_api/version.rb
CHANGED
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "tag:search.twitter.com,2005:111111111111111111",
|
|
3
|
+
"objectType": "activity",
|
|
4
|
+
"verb": "post",
|
|
5
|
+
"postedTime": "2017-01-01T00:09:10.000Z",
|
|
6
|
+
"generator": {
|
|
7
|
+
"displayName": "TweetDeck",
|
|
8
|
+
"link": "https://someurl.somwhere.com/something"
|
|
9
|
+
},
|
|
10
|
+
"provider": {
|
|
11
|
+
"objectType": "service",
|
|
12
|
+
"displayName": "Twitter",
|
|
13
|
+
"link": "http://www.twitter.com"
|
|
14
|
+
},
|
|
15
|
+
"link": "http://twitter.com/someone/statuses/111111111111111111",
|
|
16
|
+
"body": "This is an example of a long text that gnip returns. Notice the object long_object that contains more info about this tweet that… https://bla.com/asdasdasda",
|
|
17
|
+
"long_object": {
|
|
18
|
+
"body": "This is an example of a long text that gnip returns. Notice the object long_object that contains more info about this tweet which is truncated on the other body object so careful https://bla.com/asdasdasda",
|
|
19
|
+
"display_text_range": [0, 143],
|
|
20
|
+
"twitter_entities": {
|
|
21
|
+
"hashtags": [],
|
|
22
|
+
"urls": [],
|
|
23
|
+
"user_mentions": [],
|
|
24
|
+
"symbols": [{
|
|
25
|
+
"text": "BLA",
|
|
26
|
+
"indices": [1, 1]
|
|
27
|
+
}, {
|
|
28
|
+
"text": "BLI",
|
|
29
|
+
"indices": [1, 1]
|
|
30
|
+
}, {
|
|
31
|
+
"text": "BLO",
|
|
32
|
+
"indices": [1, 1]
|
|
33
|
+
}, {
|
|
34
|
+
"text": "BLU",
|
|
35
|
+
"indices": [1, 1]
|
|
36
|
+
}, {
|
|
37
|
+
"text": "BLE",
|
|
38
|
+
"indices": [1, 1]
|
|
39
|
+
}],
|
|
40
|
+
"media": [{
|
|
41
|
+
"id": 111111111111111111,
|
|
42
|
+
"id_str": "111111111111111111",
|
|
43
|
+
"indices": [1, 1],
|
|
44
|
+
"media_url": "http://pbs.twimg.com/media/somemedialink.jpg",
|
|
45
|
+
"media_url_https": "https://pbs.twimg.com/media/somemedialink.jpg",
|
|
46
|
+
"url": "https://a.be/someidshort",
|
|
47
|
+
"display_url": "pic.twitter.com/somepiclol",
|
|
48
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
49
|
+
"type": "photo",
|
|
50
|
+
"sizes": {
|
|
51
|
+
"medium": {
|
|
52
|
+
"w": 987,
|
|
53
|
+
"h": 599,
|
|
54
|
+
"resize": "fit"
|
|
55
|
+
},
|
|
56
|
+
"large": {
|
|
57
|
+
"w": 987,
|
|
58
|
+
"h": 599,
|
|
59
|
+
"resize": "fit"
|
|
60
|
+
},
|
|
61
|
+
"thumb": {
|
|
62
|
+
"w": 150,
|
|
63
|
+
"h": 150,
|
|
64
|
+
"resize": "crop"
|
|
65
|
+
},
|
|
66
|
+
"small": {
|
|
67
|
+
"w": 680,
|
|
68
|
+
"h": 413,
|
|
69
|
+
"resize": "fit"
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}, {
|
|
73
|
+
"id": 111111111111111111,
|
|
74
|
+
"id_str": "111111111111111111",
|
|
75
|
+
"indices": [1, 1],
|
|
76
|
+
"media_url": "http://pbs.twimg.com/media/someothermedia.jpg",
|
|
77
|
+
"media_url_https": "https://pbs.twimg.com/media/someothermedia.jpg",
|
|
78
|
+
"url": "https://t.co/someother",
|
|
79
|
+
"display_url": "pic.twitter.com/someother",
|
|
80
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
81
|
+
"type": "photo",
|
|
82
|
+
"sizes": {
|
|
83
|
+
"large": {
|
|
84
|
+
"w": 1571,
|
|
85
|
+
"h": 725,
|
|
86
|
+
"resize": "fit"
|
|
87
|
+
},
|
|
88
|
+
"small": {
|
|
89
|
+
"w": 680,
|
|
90
|
+
"h": 314,
|
|
91
|
+
"resize": "fit"
|
|
92
|
+
},
|
|
93
|
+
"thumb": {
|
|
94
|
+
"w": 150,
|
|
95
|
+
"h": 150,
|
|
96
|
+
"resize": "crop"
|
|
97
|
+
},
|
|
98
|
+
"medium": {
|
|
99
|
+
"w": 1200,
|
|
100
|
+
"h": 554,
|
|
101
|
+
"resize": "fit"
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}, {
|
|
105
|
+
"id": 111111111111111111,
|
|
106
|
+
"id_str": "111111111111111111",
|
|
107
|
+
"indices": [1, 1],
|
|
108
|
+
"media_url": "http://pbs.twimg.com/media/moremedia.jpg",
|
|
109
|
+
"media_url_https": "https://pbs.twimg.com/media/moremedia.jpg",
|
|
110
|
+
"url": "https://t.co/moreids",
|
|
111
|
+
"display_url": "pic.twitter.com/moreids",
|
|
112
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
113
|
+
"type": "photo",
|
|
114
|
+
"sizes": {
|
|
115
|
+
"medium": {
|
|
116
|
+
"w": 1200,
|
|
117
|
+
"h": 557,
|
|
118
|
+
"resize": "fit"
|
|
119
|
+
},
|
|
120
|
+
"thumb": {
|
|
121
|
+
"w": 150,
|
|
122
|
+
"h": 150,
|
|
123
|
+
"resize": "crop"
|
|
124
|
+
},
|
|
125
|
+
"large": {
|
|
126
|
+
"w": 1566,
|
|
127
|
+
"h": 727,
|
|
128
|
+
"resize": "fit"
|
|
129
|
+
},
|
|
130
|
+
"small": {
|
|
131
|
+
"w": 680,
|
|
132
|
+
"h": 316,
|
|
133
|
+
"resize": "fit"
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}]
|
|
137
|
+
},
|
|
138
|
+
"twitter_extended_entities": {
|
|
139
|
+
"media": [{
|
|
140
|
+
"id": 111111111111111111,
|
|
141
|
+
"id_str": "111111111111111111",
|
|
142
|
+
"indices": [1, 1],
|
|
143
|
+
"media_url": "http://pbs.twimg.com/media/tiredofchanging.jpg",
|
|
144
|
+
"media_url_https": "https://pbs.twimg.com/media/tiredofchanging.jpg",
|
|
145
|
+
"url": "https://t.co/tiredtired",
|
|
146
|
+
"display_url": "pic.twitter.com/tiredtired",
|
|
147
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
148
|
+
"type": "photo",
|
|
149
|
+
"sizes": {
|
|
150
|
+
"medium": {
|
|
151
|
+
"w": 987,
|
|
152
|
+
"h": 599,
|
|
153
|
+
"resize": "fit"
|
|
154
|
+
},
|
|
155
|
+
"large": {
|
|
156
|
+
"w": 987,
|
|
157
|
+
"h": 599,
|
|
158
|
+
"resize": "fit"
|
|
159
|
+
},
|
|
160
|
+
"thumb": {
|
|
161
|
+
"w": 150,
|
|
162
|
+
"h": 150,
|
|
163
|
+
"resize": "crop"
|
|
164
|
+
},
|
|
165
|
+
"small": {
|
|
166
|
+
"w": 680,
|
|
167
|
+
"h": 413,
|
|
168
|
+
"resize": "fit"
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}, {
|
|
172
|
+
"id": 111111111111111111,
|
|
173
|
+
"id_str": "111111111111111111",
|
|
174
|
+
"indices": [1, 1],
|
|
175
|
+
"media_url": "http://pbs.twimg.com/media/muchmedialol.jpg",
|
|
176
|
+
"media_url_https": "https://pbs.twimg.com/media/muchmedialol.jpg",
|
|
177
|
+
"url": "https://t.co/muchid",
|
|
178
|
+
"display_url": "pic.twitter.com/muchid",
|
|
179
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
180
|
+
"type": "photo",
|
|
181
|
+
"sizes": {
|
|
182
|
+
"large": {
|
|
183
|
+
"w": 1571,
|
|
184
|
+
"h": 725,
|
|
185
|
+
"resize": "fit"
|
|
186
|
+
},
|
|
187
|
+
"small": {
|
|
188
|
+
"w": 680,
|
|
189
|
+
"h": 314,
|
|
190
|
+
"resize": "fit"
|
|
191
|
+
},
|
|
192
|
+
"thumb": {
|
|
193
|
+
"w": 150,
|
|
194
|
+
"h": 150,
|
|
195
|
+
"resize": "crop"
|
|
196
|
+
},
|
|
197
|
+
"medium": {
|
|
198
|
+
"w": 1200,
|
|
199
|
+
"h": 554,
|
|
200
|
+
"resize": "fit"
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}, {
|
|
204
|
+
"id": 111111111111111111,
|
|
205
|
+
"id_str": "111111111111111111",
|
|
206
|
+
"indices": [1, 1],
|
|
207
|
+
"media_url": "http://pbs.twimg.com/media/thisneverends.jpg",
|
|
208
|
+
"media_url_https": "https://pbs.twimg.com/media/thisneverends.jpg",
|
|
209
|
+
"url": "https://t.co/yetanotherid",
|
|
210
|
+
"display_url": "pic.twitter.com/yetanotherid",
|
|
211
|
+
"expanded_url": "https://twitter.com/someone/status/111111111111111111/photo/1",
|
|
212
|
+
"type": "photo",
|
|
213
|
+
"sizes": {
|
|
214
|
+
"medium": {
|
|
215
|
+
"w": 1200,
|
|
216
|
+
"h": 557,
|
|
217
|
+
"resize": "fit"
|
|
218
|
+
},
|
|
219
|
+
"thumb": {
|
|
220
|
+
"w": 150,
|
|
221
|
+
"h": 150,
|
|
222
|
+
"resize": "crop"
|
|
223
|
+
},
|
|
224
|
+
"large": {
|
|
225
|
+
"w": 1566,
|
|
226
|
+
"h": 727,
|
|
227
|
+
"resize": "fit"
|
|
228
|
+
},
|
|
229
|
+
"small": {
|
|
230
|
+
"w": 680,
|
|
231
|
+
"h": 316,
|
|
232
|
+
"resize": "fit"
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}]
|
|
236
|
+
}
|
|
237
|
+
},
|
|
238
|
+
"display_text_range": [0, 140],
|
|
239
|
+
"actor": {
|
|
240
|
+
"objectType": "person",
|
|
241
|
+
"id": "id:twitter.com:111111111",
|
|
242
|
+
"link": "http://www.twitter.com/someone",
|
|
243
|
+
"displayName": "Some One",
|
|
244
|
+
"postedTime": "2010-10-30T18:07:33.000Z",
|
|
245
|
+
"image": "https://pbs.twimg.com/profile_images/111111111111111111/mycustomawesomepic.jpeg",
|
|
246
|
+
"summary": "I'm someone replacing this info because it's not cool to make this public. This is quite a process to do.",
|
|
247
|
+
"friendsCount": 1234,
|
|
248
|
+
"followersCount": 1234,
|
|
249
|
+
"listedCount": 12,
|
|
250
|
+
"statusesCount": 1234,
|
|
251
|
+
"twitterTimeZone": "Mountain Time (US \u0026 Canada)",
|
|
252
|
+
"verified": false,
|
|
253
|
+
"utcOffset": "-21600",
|
|
254
|
+
"preferredUsername": "Some_One",
|
|
255
|
+
"languages": ["en"],
|
|
256
|
+
"links": [{
|
|
257
|
+
"href": "https://www.youtube.com/user/ohlookatmychannel",
|
|
258
|
+
"rel": "me"
|
|
259
|
+
}],
|
|
260
|
+
"location": {
|
|
261
|
+
"objectType": "place",
|
|
262
|
+
"displayName": "USA"
|
|
263
|
+
},
|
|
264
|
+
"favoritesCount": 1234
|
|
265
|
+
},
|
|
266
|
+
"object": {
|
|
267
|
+
"objectType": "note",
|
|
268
|
+
"id": "object:search.twitter.com,2005:111111111111111111",
|
|
269
|
+
"summary": "This is an example of a long text that gnip returns. Notice the object long_object that contains more info about this tweet that… https://bla.com/asdasdasda",
|
|
270
|
+
"link": "http://twitter.com/someone/statuses/111111111111111111",
|
|
271
|
+
"postedTime": "2017-07-14T00:09:10.000Z"
|
|
272
|
+
},
|
|
273
|
+
"favoritesCount": 0,
|
|
274
|
+
"twitter_entities": {
|
|
275
|
+
"hashtags": [],
|
|
276
|
+
"urls": [{
|
|
277
|
+
"url": "https://a.be/asdasdasda",
|
|
278
|
+
"expanded_url": "https://twitter.com/i/web/status/111111111111111111",
|
|
279
|
+
"display_url": "twitter.com/i/web/status/1…",
|
|
280
|
+
"indices": [1, 1]
|
|
281
|
+
}],
|
|
282
|
+
"user_mentions": [],
|
|
283
|
+
"symbols": []
|
|
284
|
+
},
|
|
285
|
+
"twitter_lang": "en",
|
|
286
|
+
"retweetCount": 0,
|
|
287
|
+
"gnip": {
|
|
288
|
+
"matching_rules": [{
|
|
289
|
+
"tag": "1",
|
|
290
|
+
"id": 1
|
|
291
|
+
}, {
|
|
292
|
+
"tag": "2",
|
|
293
|
+
"id": 2
|
|
294
|
+
}, {
|
|
295
|
+
"tag": "3",
|
|
296
|
+
"id": 3
|
|
297
|
+
}, {
|
|
298
|
+
"tag": "4",
|
|
299
|
+
"id": 4
|
|
300
|
+
}, {
|
|
301
|
+
"tag": "5",
|
|
302
|
+
"id": 5
|
|
303
|
+
}]
|
|
304
|
+
},
|
|
305
|
+
"twitter_filter_level": "low"
|
|
306
|
+
}
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
describe Gnip::Message do
|
|
4
|
+
context 'long tweet' do
|
|
5
|
+
let(:long_tweet){ JSON.parse(File.read(fixture_path.join('activities', 'real_activity_long_object.json'))) }
|
|
6
|
+
let(:parsed_tweet){ Gnip::Activity.new(long_tweet) }
|
|
7
|
+
it('parses without problem'){ expect(Proc.new{Gnip::Activity.new(long_tweet)}).not_to raise_error }
|
|
8
|
+
it('has hidden data'){ expect(parsed_tweet.hidden_data?).to eq(true) }
|
|
9
|
+
it('has display_text_range data'){ expect(parsed_tweet.display_text_range).not_to eq(nil) }
|
|
10
|
+
it('has long_object data'){ expect(parsed_tweet.long_object).not_to eq(nil) }
|
|
11
|
+
end
|
|
12
|
+
|
|
4
13
|
context 'twitter' do
|
|
5
14
|
before do
|
|
6
15
|
@retweet_json = File.read(fixture_path.join('twitter_messages', 'retweet.json'))
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: gnip_api
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Rayko
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-07-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -195,6 +195,7 @@ files:
|
|
|
195
195
|
- spec/fixtures/activities/nil_urls.json
|
|
196
196
|
- spec/fixtures/activities/nil_urls2.json
|
|
197
197
|
- spec/fixtures/activities/real_activity.json
|
|
198
|
+
- spec/fixtures/activities/real_activity_long_object.json
|
|
198
199
|
- spec/fixtures/activities/real_activity_long_rules.json
|
|
199
200
|
- spec/fixtures/search_api/search_activities_response.json
|
|
200
201
|
- spec/fixtures/search_api/search_counts_response.json
|
|
@@ -266,6 +267,7 @@ test_files:
|
|
|
266
267
|
- spec/fixtures/activities/nil_urls.json
|
|
267
268
|
- spec/fixtures/activities/nil_urls2.json
|
|
268
269
|
- spec/fixtures/activities/real_activity.json
|
|
270
|
+
- spec/fixtures/activities/real_activity_long_object.json
|
|
269
271
|
- spec/fixtures/activities/real_activity_long_rules.json
|
|
270
272
|
- spec/fixtures/search_api/search_activities_response.json
|
|
271
273
|
- spec/fixtures/search_api/search_counts_response.json
|