gnip-client 0.2.6 → 0.2.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +6 -0
- data/.ruby-version +1 -1
- data/.travis.yml +5 -3
- data/Gemfile +1 -1
- data/README.md +12 -3
- data/Rakefile +1 -1
- data/bin/console +3 -3
- data/gnip.gemspec +10 -10
- data/lib/gnip.rb +3 -4
- data/lib/gnip/gnip-full-archive/full_archive.rb +59 -43
- data/lib/gnip/gnip-full-archive/thirty_day.rb +13 -0
- data/lib/gnip/gnip-rules/rules.rb +70 -81
- data/lib/gnip/gnip-stream/error_reconnect.rb +4 -5
- data/lib/gnip/gnip-stream/json_data_bufffer.rb +6 -8
- data/lib/gnip/gnip-stream/replay.rb +16 -18
- data/lib/gnip/gnip-stream/stream.rb +15 -17
- data/lib/gnip/power_track_client.rb +11 -10
- data/lib/gnip/version.rb +4 -2
- metadata +24 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 00cc97da9051a8751dfd2858070e4a3efa2a4c9ffa79006a74abc34ea8e572cc
|
4
|
+
data.tar.gz: d261740d92796c40c3afe31fc19c644314c8cf95a838774258bf5c2757152ca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbe3c18c8238b98f3a2d97a7573bd0cb3d32587a49a14f663d262c042ccfe501df5adfe7b69f3f3a5095a16cf56fac687d22cd627dbd60a25ccb9d295079e434
|
7
|
+
data.tar.gz: 6d798c3988a6bae2b9af93c9eaaaa4fe5e4b268d04380a0df5cb8c266b150c6f2492fafc360e3102e966daba339a2f03e35025216bc6298fad800faa9079b8fa
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.7.2
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -48,12 +48,21 @@ client.replay_rules.list
|
|
48
48
|
client.replay_rules.delete_all!
|
49
49
|
|
50
50
|
```
|
51
|
+
|
51
52
|
**Full Archive search**
|
52
53
|
|
53
54
|
```ruby
|
54
|
-
client.full_archive.search(query: "hello", date_from:
|
55
|
-
client.full_archive.total_by_time_period(query: "hello", date_from:
|
56
|
-
client.full_archive.total(query: "hello", date_from:
|
55
|
+
client.full_archive.search(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
56
|
+
client.full_archive.total_by_time_period(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
57
|
+
client.full_archive.total(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
58
|
+
```
|
59
|
+
|
60
|
+
**30day search**
|
61
|
+
|
62
|
+
```ruby
|
63
|
+
client.thirty_day.search(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
64
|
+
client.thirty_day.total_by_time_period(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
65
|
+
client.thirty_day.total(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
57
66
|
```
|
58
67
|
|
59
68
|
**Stream**
|
data/Rakefile
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/console
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'gnip'
|
5
5
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +10,5 @@ require "gnip"
|
|
10
10
|
# require "pry"
|
11
11
|
# Pry.start
|
12
12
|
|
13
|
-
require
|
13
|
+
require 'irb'
|
14
14
|
IRB.start
|
data/gnip.gemspec
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'gnip/version'
|
5
6
|
|
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ['Duccio Giovannelli']
|
10
11
|
spec.email = ['giovannelli@extendi.it']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
+
spec.summary = 'A Ruby library for accessing the Gnip API. See https://gnip.com/ for full details and to sign up for an account.'
|
13
14
|
spec.homepage = 'https://github.com/giovannelli/gnip-client'
|
14
15
|
spec.license = 'MIT'
|
15
16
|
|
@@ -18,12 +19,11 @@ Gem::Specification.new do |spec|
|
|
18
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
20
|
spec.require_paths = ['lib']
|
20
21
|
|
21
|
-
spec.
|
22
|
-
spec.add_development_dependency 'rake', '~> 10.0'
|
23
|
-
spec.add_development_dependency 'pry', '~> 0'
|
24
|
-
spec.add_development_dependency 'rspec', '~> 0'
|
25
|
-
|
26
|
-
spec.add_dependency 'httparty', '~> 0'
|
22
|
+
spec.add_dependency 'activesupport', '>= 4.2'
|
27
23
|
spec.add_dependency 'em-http-request', '~> 1'
|
28
|
-
spec.add_dependency '
|
24
|
+
spec.add_dependency 'httparty', '~> 0.16'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3'
|
28
|
+
|
29
29
|
end
|
data/lib/gnip.rb
CHANGED
@@ -4,21 +4,20 @@ require 'gnip/version'
|
|
4
4
|
require 'gnip/power_track_client'
|
5
5
|
require 'gnip/gnip-rules/rules'
|
6
6
|
require 'gnip/gnip-full-archive/full_archive'
|
7
|
+
require 'gnip/gnip-full-archive/thirty_day'
|
7
8
|
require 'gnip/gnip-stream/error_reconnect'
|
8
9
|
require 'gnip/gnip-stream/json_data_bufffer'
|
9
10
|
require 'gnip/gnip-stream/stream'
|
10
11
|
require 'gnip/gnip-stream/replay'
|
11
12
|
|
12
13
|
begin
|
13
|
-
require
|
14
|
+
require 'pry'
|
14
15
|
rescue LoadError => e
|
15
|
-
|
16
|
+
|
16
17
|
end
|
17
18
|
|
18
19
|
module Gnip
|
19
|
-
|
20
20
|
def self.format_date(datetime)
|
21
21
|
datetime.to_datetime.utc.strftime('%Y%m%d%H%M')
|
22
22
|
end
|
23
|
-
|
24
23
|
end
|
@@ -1,21 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipFullArchive
|
3
5
|
class FullArchive
|
4
|
-
|
5
|
-
|
6
|
-
|
6
|
+
class InvalidRequestException < StandardError
|
7
|
+
end
|
8
|
+
|
7
9
|
include HTTParty
|
8
|
-
|
10
|
+
|
9
11
|
attr_reader :search_url, :counts_url
|
10
|
-
|
11
|
-
#alias :total :total_entries
|
12
|
-
|
12
|
+
|
13
|
+
# alias :total :total_entries
|
14
|
+
|
13
15
|
def initialize(client)
|
14
16
|
@search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json"
|
15
17
|
@counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json"
|
16
18
|
@auth = { username: client.username, password: client.password }
|
17
19
|
end
|
18
|
-
|
20
|
+
|
19
21
|
# Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page
|
20
22
|
# options[:query] query to twitter
|
21
23
|
# options[:per_page] default is 500
|
@@ -24,63 +26,79 @@ module Gnip
|
|
24
26
|
# options[:cursor_next] cursor to the next page
|
25
27
|
def search(options = {})
|
26
28
|
search_options = {}
|
27
|
-
search_options[:query] = options[:query]||''
|
28
|
-
search_options[:maxResults]
|
29
|
-
search_options[:fromDate]
|
30
|
-
search_options[:toDate]
|
31
|
-
search_options[:next]
|
32
|
-
url = [
|
29
|
+
search_options[:query] = options[:query] || ''
|
30
|
+
search_options[:maxResults] = options[:per_page] || 500
|
31
|
+
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
32
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
33
|
+
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
34
|
+
url = [search_url, search_options.to_query].join('?')
|
33
35
|
begin
|
34
36
|
gnip_call = self.class.get(url, basic_auth: @auth)
|
35
37
|
response = gnip_call.response
|
36
38
|
parsed_response = gnip_call.parsed_response
|
37
|
-
parsed_response = (parsed_response||{}).with_indifferent_access
|
38
|
-
raise response.message
|
39
|
+
parsed_response = (parsed_response || {}).with_indifferent_access
|
40
|
+
raise response.message unless parsed_response.present?
|
41
|
+
|
39
42
|
if parsed_response[:error].present?
|
40
|
-
response = { results: [],
|
43
|
+
response = { results: [],
|
44
|
+
next: nil,
|
45
|
+
url: url,
|
46
|
+
error: parsed_response[:error][:message],
|
47
|
+
code: response.code.to_i }
|
41
48
|
else
|
42
|
-
response = { results: parsed_response[:results],
|
49
|
+
response = { results: parsed_response[:results],
|
50
|
+
url: url,
|
51
|
+
next: parsed_response[:next],
|
52
|
+
code: response.code.to_i }
|
43
53
|
end
|
44
|
-
rescue
|
45
|
-
response = { results: [],
|
54
|
+
rescue StandardError => e
|
55
|
+
response = { results: [],
|
56
|
+
url: url,
|
57
|
+
next: nil,
|
58
|
+
error: e.message,
|
59
|
+
code: 500 }
|
46
60
|
end
|
47
|
-
|
61
|
+
response
|
48
62
|
end
|
49
|
-
|
63
|
+
|
50
64
|
# full aarchive search endpoints return total contents by day, minute, hour paginated
|
51
65
|
# so to get totals across time period passed may need to run more than one call, the stop condition is cursor nil
|
52
66
|
# bucket: must be one of [minute, hour, day]
|
53
|
-
def total_by_time_period(options={})
|
54
|
-
response = options[:response]||{}
|
67
|
+
def total_by_time_period(options = {})
|
68
|
+
response = options[:response] || {}
|
55
69
|
search_options = {}
|
56
|
-
search_options[:query]
|
57
|
-
search_options[:bucket]
|
70
|
+
search_options[:query] = options[:query] || ''
|
71
|
+
search_options[:bucket] = options[:bucket] || 'day'
|
58
72
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
59
|
-
search_options[:toDate]
|
60
|
-
search_options[:next]
|
73
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
74
|
+
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
75
|
+
|
76
|
+
url = [counts_url, search_options.to_query].join('?')
|
77
|
+
call_done = 0
|
61
78
|
|
62
|
-
url = [self.counts_url, search_options.to_query].join('?')
|
63
|
-
|
64
79
|
begin
|
65
80
|
gnip_call = self.class.get(url, basic_auth: @auth)
|
66
81
|
|
67
82
|
parsed_response = gnip_call.parsed_response
|
68
|
-
parsed_response = (parsed_response||{}).with_indifferent_access
|
83
|
+
parsed_response = (parsed_response || {}).with_indifferent_access
|
69
84
|
|
70
|
-
raise gnip_call.response.message
|
85
|
+
raise gnip_call.response.message unless parsed_response.present?
|
71
86
|
|
72
87
|
if parsed_response[:error].present?
|
73
|
-
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls]||0) + 1 }
|
88
|
+
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
74
89
|
else
|
75
|
-
|
90
|
+
call_done = 1 # we have received a valid response
|
91
|
+
parsed_response[:results].each_with_index do |item, i|
|
76
92
|
parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
|
77
93
|
end
|
78
|
-
response = { results: (response[:results]||[]) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls]||0) + 1 }
|
94
|
+
response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
79
95
|
end
|
80
|
-
rescue
|
81
|
-
response = { results: [], next: nil, error: e.message, code: 500 }
|
96
|
+
rescue StandardError => e
|
97
|
+
response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
|
82
98
|
end
|
83
|
-
|
99
|
+
# If the next cursor is not present we fetched all the data
|
100
|
+
# It happens that twitter returns the same cursor, in that case we stop
|
101
|
+
return response if !parsed_response[:next].to_s.present? || (parsed_response[:next].to_s.present? && parsed_response[:next] == search_options[:next])
|
84
102
|
|
85
103
|
total_by_time_period(query: search_options[:query],
|
86
104
|
date_from: search_options[:fromDate],
|
@@ -89,16 +107,14 @@ module Gnip
|
|
89
107
|
next_cursor: parsed_response[:next],
|
90
108
|
response: response)
|
91
109
|
end
|
92
|
-
|
110
|
+
|
93
111
|
# return total contents in a specific date interval with a passed query
|
94
|
-
def total(options={})
|
112
|
+
def total(options = {})
|
95
113
|
extra = {}
|
96
114
|
response = total_by_time_period(options)
|
97
115
|
extra = { error: response[:error] } if response[:error].present?
|
98
|
-
|
116
|
+
{ query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
|
99
117
|
end
|
100
|
-
|
101
118
|
end
|
102
|
-
|
103
119
|
end
|
104
120
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gnip
|
4
|
+
module GnipFullArchive
|
5
|
+
class ThirtyDay < FullArchive
|
6
|
+
def initialize(client)
|
7
|
+
@search_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}.json"
|
8
|
+
@counts_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}/counts.json"
|
9
|
+
@auth = { username: client.username, password: client.password }
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -1,16 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipRules
|
3
5
|
class Rules
|
4
|
-
|
5
6
|
include HTTParty
|
6
|
-
|
7
|
+
|
7
8
|
attr_reader :rules_url, :version
|
8
|
-
|
9
|
-
def initialize(client, replay=false)
|
9
|
+
|
10
|
+
def initialize(client, replay = false)
|
10
11
|
@version = client.power_track_version
|
11
|
-
case
|
12
|
+
case version
|
12
13
|
when '1.0'
|
13
|
-
@rules_url = "https://api.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/#{replay ?
|
14
|
+
@rules_url = "https://api.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/#{replay ? 'replay' : 'streams'}/track/#{client.label}/rules.json"
|
14
15
|
when '2.0'
|
15
16
|
if replay
|
16
17
|
@rules_url = "https://gnip-api.twitter.com/rules/powertrack-replay/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
@@ -18,111 +19,99 @@ module Gnip
|
|
18
19
|
@rules_url = "https://gnip-api.twitter.com/rules/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
20
|
end
|
20
21
|
else
|
21
|
-
raise Exception
|
22
|
+
raise Exception, "version #{version} is not supported from this gem."
|
22
23
|
end
|
23
24
|
@auth = { username: client.username, password: client.password }
|
24
25
|
end
|
25
|
-
|
26
|
-
#Add rules to PowerTrack rules
|
27
|
-
#rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
26
|
+
|
27
|
+
# Add rules to PowerTrack rules
|
28
|
+
# rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
28
29
|
def add(rules)
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
{ status: :success, code: 200, response: parsed_response }
|
36
|
-
end
|
37
|
-
rescue Exception => e
|
38
|
-
{ status: :error, code: 500, error: e.message }
|
30
|
+
response = self.class.post(rules_url, basic_auth: @auth, body: rules.to_json, headers: { 'Content-Type' => 'application/json', 'Accept' => 'application/json' }, type: :json)
|
31
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
32
|
+
if parsed_response.present? && parsed_response['error'].present?
|
33
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
34
|
+
else
|
35
|
+
{ status: :success, code: 200, response: parsed_response }
|
39
36
|
end
|
40
|
-
|
37
|
+
rescue Exception => e
|
38
|
+
{ status: :error, code: 500, error: e.message }
|
41
39
|
end
|
42
|
-
|
43
|
-
#Remove rules from PowerTrack rules
|
44
|
-
#rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
40
|
+
|
41
|
+
# Remove rules from PowerTrack rules
|
42
|
+
# rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
45
43
|
def remove(rules)
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
{ status: :success, code: 200, response: parsed_response }
|
53
|
-
end
|
54
|
-
rescue Exception => e
|
55
|
-
{ status: :error, code: 500, error: e.message }
|
44
|
+
response = self.class.post(rules_url, query: { _method: 'delete' }, basic_auth: @auth, body: rules.to_json)
|
45
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
46
|
+
if parsed_response.present? && parsed_response['error'].present?
|
47
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
48
|
+
else
|
49
|
+
{ status: :success, code: 200, response: parsed_response }
|
56
50
|
end
|
57
|
-
|
51
|
+
rescue Exception => e
|
52
|
+
{ status: :error, code: 500, error: e.message }
|
58
53
|
end
|
59
|
-
|
60
|
-
#Get the full list of rules
|
54
|
+
|
55
|
+
# Get the full list of rules
|
61
56
|
def list
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
{ status: :success, code: 200, rules: parsed_response["rules"] }
|
69
|
-
end
|
70
|
-
rescue Exception => e
|
71
|
-
{ status: :error, code: 500, error: e.message }
|
57
|
+
response = self.class.get(rules_url, basic_auth: @auth, headers: { 'Content-Type' => 'application/json', 'Accept' => 'application/json' }, type: :json)
|
58
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
59
|
+
if parsed_response.present? && parsed_response['error'].present?
|
60
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
61
|
+
else
|
62
|
+
{ status: :success, code: 200, rules: parsed_response['rules'] }
|
72
63
|
end
|
64
|
+
rescue Exception => e
|
65
|
+
{ status: :error, code: 500, error: e.message }
|
73
66
|
end
|
74
|
-
|
75
|
-
#Get the full list of rules by tag
|
67
|
+
|
68
|
+
# Get the full list of rules by tag
|
76
69
|
def list_by_tag(tag)
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
{ status: :success, code: 200, rules: rules.select{ |rule| rule["tag"] == tag } }
|
85
|
-
end
|
86
|
-
rescue Exception => e
|
87
|
-
{ status: :error, code: 500, error: e.message }
|
70
|
+
response = self.class.get(rules_url, basic_auth: @auth)
|
71
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
72
|
+
if parsed_response.present? && parsed_response['error'].present?
|
73
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
74
|
+
else
|
75
|
+
rules = parsed_response['rules']
|
76
|
+
{ status: :success, code: 200, rules: rules.select { |rule| rule['tag'] == tag } }
|
88
77
|
end
|
89
|
-
|
78
|
+
rescue Exception => e
|
79
|
+
{ status: :error, code: 500, error: e.message }
|
90
80
|
end
|
91
|
-
|
92
|
-
#delete all rules from PowerTrack
|
93
|
-
#http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
|
94
|
-
#Request Body Size Limit 1 MB (~5000 rules)
|
81
|
+
|
82
|
+
# delete all rules from PowerTrack
|
83
|
+
# http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
|
84
|
+
# Request Body Size Limit 1 MB (~5000 rules)
|
95
85
|
def delete_all!
|
96
86
|
retry_times = 0
|
97
87
|
begin
|
98
|
-
rules_list =
|
99
|
-
(rules_list[:rules]||[]).in_groups_of(2, false).each do |group_of_rules|
|
100
|
-
|
88
|
+
rules_list = list
|
89
|
+
(rules_list[:rules] || []).in_groups_of(2, false).each do |group_of_rules|
|
90
|
+
remove("rules": group_of_rules)
|
101
91
|
end
|
102
92
|
sleep 0.05
|
103
|
-
rules_list =
|
104
|
-
if !(rules_list[:rules]||[]).size.zero?
|
105
|
-
|
93
|
+
rules_list = list
|
94
|
+
if !(rules_list[:rules] || []).size.zero?
|
95
|
+
delete_all!
|
106
96
|
else
|
107
|
-
|
97
|
+
{ status: :success, code: 200, rules: [] }
|
108
98
|
end
|
109
99
|
rescue Exception => e
|
110
100
|
retry_times += 1
|
111
101
|
if retry_times <= 3
|
112
102
|
retry
|
113
103
|
else
|
114
|
-
|
115
|
-
end
|
104
|
+
{ status: :error, code: 500, error: e.message }
|
105
|
+
end
|
116
106
|
end
|
117
|
-
|
118
107
|
end
|
119
|
-
|
108
|
+
|
120
109
|
private
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
110
|
+
|
111
|
+
def safe_parsed_response(parsed_response)
|
112
|
+
ret = parsed_response.present? ? (parsed_response.is_a?(String) ? JSON.parse(parsed_response).with_indifferent_access : parsed_response) : nil
|
113
|
+
ret
|
114
|
+
end
|
126
115
|
end
|
127
116
|
end
|
128
117
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipStream
|
3
|
-
|
4
5
|
class ErrorReconnect
|
5
|
-
|
6
6
|
def initialize(source_class, method_name)
|
7
7
|
@source_class = source_class
|
8
8
|
@method_name = method_name
|
@@ -12,7 +12,7 @@ module Gnip
|
|
12
12
|
def attempt_to_reconnect(error_message)
|
13
13
|
@error_message = error_message
|
14
14
|
if @reconnect_attempts < 5
|
15
|
-
@reconnect_attempts +=1
|
15
|
+
@reconnect_attempts += 1
|
16
16
|
sleep(2)
|
17
17
|
@source_class.send(@method_name)
|
18
18
|
else
|
@@ -23,7 +23,6 @@ module Gnip
|
|
23
23
|
def reconnect_failed_raise_error
|
24
24
|
raise @error_message
|
25
25
|
end
|
26
|
-
|
27
26
|
end
|
28
27
|
end
|
29
|
-
end
|
28
|
+
end
|
@@ -1,13 +1,12 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
|
-
class JsonDataBuffer
|
4
|
-
|
3
|
+
class JsonDataBuffer
|
5
4
|
attr_accessor :split_pattern, :check_pattern
|
6
|
-
|
5
|
+
|
7
6
|
def initialize(split_pattern, check_pattern)
|
8
7
|
@split_pattern = split_pattern
|
9
8
|
@check_pattern = check_pattern
|
10
|
-
@buffer =
|
9
|
+
@buffer = ''
|
11
10
|
end
|
12
11
|
|
13
12
|
def process(chunk)
|
@@ -21,11 +20,10 @@ module Gnip
|
|
21
20
|
activities = @buffer.split(split_pattern)
|
22
21
|
entries << activities.shift
|
23
22
|
@buffer = activities.join(split_pattern)
|
24
|
-
@buffer
|
23
|
+
@buffer += "\r\n" if !@buffer.empty? && new_line
|
25
24
|
end
|
26
|
-
entries.
|
25
|
+
entries.reject(&:empty?)
|
27
26
|
end
|
28
27
|
end
|
29
|
-
|
30
28
|
end
|
31
|
-
end
|
29
|
+
end
|
@@ -1,49 +1,47 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
3
|
class Replay < Stream
|
4
|
-
|
5
4
|
def initialize(client)
|
6
|
-
super #version is setted in the super
|
7
|
-
case
|
5
|
+
super # version is setted in the super
|
6
|
+
case version
|
8
7
|
when '1.0'
|
9
8
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/replay/track/#{client.replay_label}.json"
|
10
9
|
when '2.0'
|
11
10
|
@url = "https://gnip-stream.gnip.com/replay/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
12
11
|
else
|
13
|
-
raise Exception.new("version #{
|
12
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
14
13
|
end
|
15
14
|
end
|
16
|
-
|
15
|
+
|
17
16
|
def configure_handlers
|
18
|
-
|
19
|
-
|
17
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
18
|
+
on_connection_close { puts 'Gnip::GnipStream::Replay -> Connection closed' }
|
20
19
|
end
|
21
|
-
|
22
|
-
def consume(options={}, &block)
|
20
|
+
|
21
|
+
def consume(options = {}, &block)
|
23
22
|
@client_callback = block if block
|
24
|
-
|
25
|
-
|
23
|
+
on_message(&@client_callback)
|
24
|
+
connect(options)
|
26
25
|
end
|
27
|
-
|
26
|
+
|
28
27
|
def connect(options)
|
29
28
|
search_options = {}
|
30
29
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
31
30
|
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
32
|
-
stream_url = [
|
31
|
+
stream_url = [url, search_options.to_query].join('?')
|
33
32
|
EM.run do
|
34
33
|
http = EM::HttpRequest.new(stream_url, inactivity_timeout: 45, connection_timeout: 75).get(head: @headers)
|
35
34
|
http.stream { |chunk| process_chunk(chunk) }
|
36
|
-
http.callback {
|
37
|
-
handle_connection_close(http)
|
35
|
+
http.callback {
|
36
|
+
handle_connection_close(http)
|
38
37
|
EM.stop
|
39
38
|
}
|
40
|
-
http.errback {
|
39
|
+
http.errback {
|
41
40
|
handle_error(http)
|
42
41
|
EM.stop
|
43
42
|
}
|
44
43
|
end
|
45
44
|
end
|
46
|
-
|
47
45
|
end
|
48
46
|
end
|
49
|
-
end
|
47
|
+
end
|
@@ -4,40 +4,39 @@ require 'em-http-request'
|
|
4
4
|
module Gnip
|
5
5
|
module GnipStream
|
6
6
|
class Stream
|
7
|
-
|
8
7
|
EventMachine.threadpool_size = 5
|
9
8
|
|
10
9
|
attr_accessor :url, :backfill_client, :version
|
11
10
|
|
12
11
|
def initialize(client)
|
13
12
|
self.version = client.power_track_version
|
14
|
-
case
|
13
|
+
case version
|
15
14
|
when '1.0'
|
16
15
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/streams/track/#{client.label}.json"
|
17
16
|
when '2.0'
|
18
17
|
@url = "https://gnip-stream.twitter.com/stream/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
18
|
else
|
20
|
-
raise Exception.new("version #{
|
19
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
21
20
|
end
|
22
21
|
@backfill_client = client.backfill_client
|
23
22
|
@processor = JsonDataBuffer.new("\r\n", Regexp.new(/^\{.*\}\r\n/))
|
24
|
-
@headers = {'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed'}
|
23
|
+
@headers = { 'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed' }
|
25
24
|
@error_handler = ErrorReconnect.new(self, :consume)
|
26
25
|
@connection_close_handler = ErrorReconnect.new(self, :consume)
|
27
26
|
configure_handlers
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def configure_handlers
|
31
|
-
|
32
|
-
|
30
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
31
|
+
on_connection_close { @connection_close_handler.attempt_to_reconnect('Gnip Connection Closed') }
|
33
32
|
end
|
34
33
|
|
35
34
|
def consume(&block)
|
36
35
|
@client_callback = block if block
|
37
|
-
|
38
|
-
|
36
|
+
on_message(&@client_callback)
|
37
|
+
connect
|
39
38
|
end
|
40
|
-
|
39
|
+
|
41
40
|
def on_message(&block)
|
42
41
|
@on_message = block
|
43
42
|
end
|
@@ -53,14 +52,14 @@ module Gnip
|
|
53
52
|
def connect
|
54
53
|
EM.run do
|
55
54
|
options = {}
|
56
|
-
options = { query: {
|
57
|
-
http = EM::HttpRequest.new(
|
55
|
+
options = { query: { 'client' => backfill_client } } if backfill_client.present?
|
56
|
+
http = EM::HttpRequest.new(url, inactivity_timeout: 45, connection_timeout: 75).get({ head: @headers }.merge!(options))
|
58
57
|
http.stream { |chunk| process_chunk(chunk) }
|
59
|
-
http.callback {
|
60
|
-
handle_connection_close(http)
|
58
|
+
http.callback {
|
59
|
+
handle_connection_close(http)
|
61
60
|
EM.stop
|
62
61
|
}
|
63
|
-
http.errback {
|
62
|
+
http.errback {
|
64
63
|
handle_error(http)
|
65
64
|
EM.stop
|
66
65
|
}
|
@@ -81,7 +80,6 @@ module Gnip
|
|
81
80
|
def handle_connection_close(http_connection)
|
82
81
|
@on_connection_close.call(http_connection)
|
83
82
|
end
|
84
|
-
|
85
83
|
end
|
86
84
|
end
|
87
|
-
end
|
85
|
+
end
|
@@ -1,27 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
class PowerTrackClient
|
3
|
-
|
4
5
|
attr_accessor :publisher, :label, :account,
|
5
6
|
:username, :password,
|
6
7
|
:backfill_client, :replay_label
|
7
|
-
|
8
|
-
attr_reader :rules, :replay_rules, :full_archive, :stream, :replay, :power_track_version
|
9
|
-
|
8
|
+
|
9
|
+
attr_reader :rules, :replay_rules, :full_archive, :thirty_day, :stream, :replay, :power_track_version
|
10
|
+
|
10
11
|
def initialize(options = {})
|
11
12
|
@account = options[:account]
|
12
|
-
@publisher = options[:publisher]||
|
13
|
-
@label = options[:label]||
|
14
|
-
@replay_label = options[:replay_label]
|
13
|
+
@publisher = options[:publisher] || 'twitter'
|
14
|
+
@label = options[:label] || 'dev'
|
15
|
+
@replay_label = options[:replay_label] || @label
|
15
16
|
@username = options[:username]
|
16
17
|
@password = options[:password]
|
17
|
-
@backfill_client = options[:backfill_client]||nil
|
18
|
-
@power_track_version = options[:power_track_version]||'2.0'
|
18
|
+
@backfill_client = options[:backfill_client] || nil
|
19
|
+
@power_track_version = options[:power_track_version] || '2.0'
|
19
20
|
@rules = Gnip::GnipRules::Rules.new(self)
|
20
21
|
@replay_rules = Gnip::GnipRules::Rules.new(self, true)
|
21
22
|
@full_archive = Gnip::GnipFullArchive::FullArchive.new(self)
|
23
|
+
@thirty_day = Gnip::GnipFullArchive::ThirtyDay.new(self)
|
22
24
|
@stream = Gnip::GnipStream::Stream.new(self)
|
23
25
|
@replay = Gnip::GnipStream::Replay.new(self)
|
24
26
|
end
|
25
|
-
|
26
27
|
end
|
27
28
|
end
|
data/lib/gnip/version.rb
CHANGED
metadata
CHANGED
@@ -1,113 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gnip-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Duccio Giovannelli
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-11-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.9'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.9'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '10.0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '10.0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: pry
|
14
|
+
name: activesupport
|
43
15
|
requirement: !ruby/object:Gem::Requirement
|
44
16
|
requirements:
|
45
|
-
- - "
|
17
|
+
- - ">="
|
46
18
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
49
21
|
prerelease: false
|
50
22
|
version_requirements: !ruby/object:Gem::Requirement
|
51
23
|
requirements:
|
52
|
-
- - "
|
24
|
+
- - ">="
|
53
25
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
26
|
+
version: '4.2'
|
55
27
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
28
|
+
name: em-http-request
|
57
29
|
requirement: !ruby/object:Gem::Requirement
|
58
30
|
requirements:
|
59
31
|
- - "~>"
|
60
32
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
62
|
-
type: :
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
63
35
|
prerelease: false
|
64
36
|
version_requirements: !ruby/object:Gem::Requirement
|
65
37
|
requirements:
|
66
38
|
- - "~>"
|
67
39
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
40
|
+
version: '1'
|
69
41
|
- !ruby/object:Gem::Dependency
|
70
42
|
name: httparty
|
71
43
|
requirement: !ruby/object:Gem::Requirement
|
72
44
|
requirements:
|
73
45
|
- - "~>"
|
74
46
|
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
47
|
+
version: '0.16'
|
76
48
|
type: :runtime
|
77
49
|
prerelease: false
|
78
50
|
version_requirements: !ruby/object:Gem::Requirement
|
79
51
|
requirements:
|
80
52
|
- - "~>"
|
81
53
|
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
54
|
+
version: '0.16'
|
83
55
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
56
|
+
name: rake
|
85
57
|
requirement: !ruby/object:Gem::Requirement
|
86
58
|
requirements:
|
87
59
|
- - "~>"
|
88
60
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
90
|
-
type: :
|
61
|
+
version: '13.0'
|
62
|
+
type: :development
|
91
63
|
prerelease: false
|
92
64
|
version_requirements: !ruby/object:Gem::Requirement
|
93
65
|
requirements:
|
94
66
|
- - "~>"
|
95
67
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
68
|
+
version: '13.0'
|
97
69
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
70
|
+
name: rspec
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
100
72
|
requirements:
|
101
73
|
- - "~>"
|
102
74
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
104
|
-
type: :
|
75
|
+
version: '3'
|
76
|
+
type: :development
|
105
77
|
prerelease: false
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
107
79
|
requirements:
|
108
80
|
- - "~>"
|
109
81
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
82
|
+
version: '3'
|
111
83
|
description:
|
112
84
|
email:
|
113
85
|
- giovannelli@extendi.it
|
@@ -129,6 +101,7 @@ files:
|
|
129
101
|
- gnip.gemspec
|
130
102
|
- lib/gnip.rb
|
131
103
|
- lib/gnip/gnip-full-archive/full_archive.rb
|
104
|
+
- lib/gnip/gnip-full-archive/thirty_day.rb
|
132
105
|
- lib/gnip/gnip-rules/rules.rb
|
133
106
|
- lib/gnip/gnip-stream/error_reconnect.rb
|
134
107
|
- lib/gnip/gnip-stream/json_data_bufffer.rb
|
@@ -155,8 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
128
|
- !ruby/object:Gem::Version
|
156
129
|
version: '0'
|
157
130
|
requirements: []
|
158
|
-
|
159
|
-
rubygems_version: 2.5.2
|
131
|
+
rubygems_version: 3.1.4
|
160
132
|
signing_key:
|
161
133
|
specification_version: 4
|
162
134
|
summary: A Ruby library for accessing the Gnip API. See https://gnip.com/ for full
|