gnip-client 0.2.6 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +6 -0
- data/.ruby-version +1 -1
- data/.travis.yml +5 -3
- data/Gemfile +1 -1
- data/README.md +12 -3
- data/Rakefile +1 -1
- data/bin/console +3 -3
- data/gnip.gemspec +10 -10
- data/lib/gnip.rb +3 -4
- data/lib/gnip/gnip-full-archive/full_archive.rb +59 -43
- data/lib/gnip/gnip-full-archive/thirty_day.rb +13 -0
- data/lib/gnip/gnip-rules/rules.rb +70 -81
- data/lib/gnip/gnip-stream/error_reconnect.rb +4 -5
- data/lib/gnip/gnip-stream/json_data_bufffer.rb +6 -8
- data/lib/gnip/gnip-stream/replay.rb +16 -18
- data/lib/gnip/gnip-stream/stream.rb +15 -17
- data/lib/gnip/power_track_client.rb +11 -10
- data/lib/gnip/version.rb +4 -2
- metadata +24 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 00cc97da9051a8751dfd2858070e4a3efa2a4c9ffa79006a74abc34ea8e572cc
|
4
|
+
data.tar.gz: d261740d92796c40c3afe31fc19c644314c8cf95a838774258bf5c2757152ca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbe3c18c8238b98f3a2d97a7573bd0cb3d32587a49a14f663d262c042ccfe501df5adfe7b69f3f3a5095a16cf56fac687d22cd627dbd60a25ccb9d295079e434
|
7
|
+
data.tar.gz: 6d798c3988a6bae2b9af93c9eaaaa4fe5e4b268d04380a0df5cb8c266b150c6f2492fafc360e3102e966daba339a2f03e35025216bc6298fad800faa9079b8fa
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.7.2
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -48,12 +48,21 @@ client.replay_rules.list
|
|
48
48
|
client.replay_rules.delete_all!
|
49
49
|
|
50
50
|
```
|
51
|
+
|
51
52
|
**Full Archive search**
|
52
53
|
|
53
54
|
```ruby
|
54
|
-
client.full_archive.search(query: "hello", date_from:
|
55
|
-
client.full_archive.total_by_time_period(query: "hello", date_from:
|
56
|
-
client.full_archive.total(query: "hello", date_from:
|
55
|
+
client.full_archive.search(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
56
|
+
client.full_archive.total_by_time_period(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
57
|
+
client.full_archive.total(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
58
|
+
```
|
59
|
+
|
60
|
+
**30day search**
|
61
|
+
|
62
|
+
```ruby
|
63
|
+
client.thirty_day.search(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
64
|
+
client.thirty_day.total_by_time_period(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
65
|
+
client.thirty_day.total(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
57
66
|
```
|
58
67
|
|
59
68
|
**Stream**
|
data/Rakefile
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/console
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'gnip'
|
5
5
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +10,5 @@ require "gnip"
|
|
10
10
|
# require "pry"
|
11
11
|
# Pry.start
|
12
12
|
|
13
|
-
require
|
13
|
+
require 'irb'
|
14
14
|
IRB.start
|
data/gnip.gemspec
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'gnip/version'
|
5
6
|
|
@@ -9,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ['Duccio Giovannelli']
|
10
11
|
spec.email = ['giovannelli@extendi.it']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
+
spec.summary = 'A Ruby library for accessing the Gnip API. See https://gnip.com/ for full details and to sign up for an account.'
|
13
14
|
spec.homepage = 'https://github.com/giovannelli/gnip-client'
|
14
15
|
spec.license = 'MIT'
|
15
16
|
|
@@ -18,12 +19,11 @@ Gem::Specification.new do |spec|
|
|
18
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
19
20
|
spec.require_paths = ['lib']
|
20
21
|
|
21
|
-
spec.
|
22
|
-
spec.add_development_dependency 'rake', '~> 10.0'
|
23
|
-
spec.add_development_dependency 'pry', '~> 0'
|
24
|
-
spec.add_development_dependency 'rspec', '~> 0'
|
25
|
-
|
26
|
-
spec.add_dependency 'httparty', '~> 0'
|
22
|
+
spec.add_dependency 'activesupport', '>= 4.2'
|
27
23
|
spec.add_dependency 'em-http-request', '~> 1'
|
28
|
-
spec.add_dependency '
|
24
|
+
spec.add_dependency 'httparty', '~> 0.16'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3'
|
28
|
+
|
29
29
|
end
|
data/lib/gnip.rb
CHANGED
@@ -4,21 +4,20 @@ require 'gnip/version'
|
|
4
4
|
require 'gnip/power_track_client'
|
5
5
|
require 'gnip/gnip-rules/rules'
|
6
6
|
require 'gnip/gnip-full-archive/full_archive'
|
7
|
+
require 'gnip/gnip-full-archive/thirty_day'
|
7
8
|
require 'gnip/gnip-stream/error_reconnect'
|
8
9
|
require 'gnip/gnip-stream/json_data_bufffer'
|
9
10
|
require 'gnip/gnip-stream/stream'
|
10
11
|
require 'gnip/gnip-stream/replay'
|
11
12
|
|
12
13
|
begin
|
13
|
-
require
|
14
|
+
require 'pry'
|
14
15
|
rescue LoadError => e
|
15
|
-
|
16
|
+
|
16
17
|
end
|
17
18
|
|
18
19
|
module Gnip
|
19
|
-
|
20
20
|
def self.format_date(datetime)
|
21
21
|
datetime.to_datetime.utc.strftime('%Y%m%d%H%M')
|
22
22
|
end
|
23
|
-
|
24
23
|
end
|
@@ -1,21 +1,23 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipFullArchive
|
3
5
|
class FullArchive
|
4
|
-
|
5
|
-
|
6
|
-
|
6
|
+
class InvalidRequestException < StandardError
|
7
|
+
end
|
8
|
+
|
7
9
|
include HTTParty
|
8
|
-
|
10
|
+
|
9
11
|
attr_reader :search_url, :counts_url
|
10
|
-
|
11
|
-
#alias :total :total_entries
|
12
|
-
|
12
|
+
|
13
|
+
# alias :total :total_entries
|
14
|
+
|
13
15
|
def initialize(client)
|
14
16
|
@search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json"
|
15
17
|
@counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json"
|
16
18
|
@auth = { username: client.username, password: client.password }
|
17
19
|
end
|
18
|
-
|
20
|
+
|
19
21
|
# Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page
|
20
22
|
# options[:query] query to twitter
|
21
23
|
# options[:per_page] default is 500
|
@@ -24,63 +26,79 @@ module Gnip
|
|
24
26
|
# options[:cursor_next] cursor to the next page
|
25
27
|
def search(options = {})
|
26
28
|
search_options = {}
|
27
|
-
search_options[:query] = options[:query]||''
|
28
|
-
search_options[:maxResults]
|
29
|
-
search_options[:fromDate]
|
30
|
-
search_options[:toDate]
|
31
|
-
search_options[:next]
|
32
|
-
url = [
|
29
|
+
search_options[:query] = options[:query] || ''
|
30
|
+
search_options[:maxResults] = options[:per_page] || 500
|
31
|
+
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
32
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
33
|
+
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
34
|
+
url = [search_url, search_options.to_query].join('?')
|
33
35
|
begin
|
34
36
|
gnip_call = self.class.get(url, basic_auth: @auth)
|
35
37
|
response = gnip_call.response
|
36
38
|
parsed_response = gnip_call.parsed_response
|
37
|
-
parsed_response = (parsed_response||{}).with_indifferent_access
|
38
|
-
raise response.message
|
39
|
+
parsed_response = (parsed_response || {}).with_indifferent_access
|
40
|
+
raise response.message unless parsed_response.present?
|
41
|
+
|
39
42
|
if parsed_response[:error].present?
|
40
|
-
response = { results: [],
|
43
|
+
response = { results: [],
|
44
|
+
next: nil,
|
45
|
+
url: url,
|
46
|
+
error: parsed_response[:error][:message],
|
47
|
+
code: response.code.to_i }
|
41
48
|
else
|
42
|
-
response = { results: parsed_response[:results],
|
49
|
+
response = { results: parsed_response[:results],
|
50
|
+
url: url,
|
51
|
+
next: parsed_response[:next],
|
52
|
+
code: response.code.to_i }
|
43
53
|
end
|
44
|
-
rescue
|
45
|
-
response = { results: [],
|
54
|
+
rescue StandardError => e
|
55
|
+
response = { results: [],
|
56
|
+
url: url,
|
57
|
+
next: nil,
|
58
|
+
error: e.message,
|
59
|
+
code: 500 }
|
46
60
|
end
|
47
|
-
|
61
|
+
response
|
48
62
|
end
|
49
|
-
|
63
|
+
|
50
64
|
# full aarchive search endpoints return total contents by day, minute, hour paginated
|
51
65
|
# so to get totals across time period passed may need to run more than one call, the stop condition is cursor nil
|
52
66
|
# bucket: must be one of [minute, hour, day]
|
53
|
-
def total_by_time_period(options={})
|
54
|
-
response = options[:response]||{}
|
67
|
+
def total_by_time_period(options = {})
|
68
|
+
response = options[:response] || {}
|
55
69
|
search_options = {}
|
56
|
-
search_options[:query]
|
57
|
-
search_options[:bucket]
|
70
|
+
search_options[:query] = options[:query] || ''
|
71
|
+
search_options[:bucket] = options[:bucket] || 'day'
|
58
72
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
59
|
-
search_options[:toDate]
|
60
|
-
search_options[:next]
|
73
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
74
|
+
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
75
|
+
|
76
|
+
url = [counts_url, search_options.to_query].join('?')
|
77
|
+
call_done = 0
|
61
78
|
|
62
|
-
url = [self.counts_url, search_options.to_query].join('?')
|
63
|
-
|
64
79
|
begin
|
65
80
|
gnip_call = self.class.get(url, basic_auth: @auth)
|
66
81
|
|
67
82
|
parsed_response = gnip_call.parsed_response
|
68
|
-
parsed_response = (parsed_response||{}).with_indifferent_access
|
83
|
+
parsed_response = (parsed_response || {}).with_indifferent_access
|
69
84
|
|
70
|
-
raise gnip_call.response.message
|
85
|
+
raise gnip_call.response.message unless parsed_response.present?
|
71
86
|
|
72
87
|
if parsed_response[:error].present?
|
73
|
-
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls]||0) + 1 }
|
88
|
+
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
74
89
|
else
|
75
|
-
|
90
|
+
call_done = 1 # we have received a valid response
|
91
|
+
parsed_response[:results].each_with_index do |item, i|
|
76
92
|
parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
|
77
93
|
end
|
78
|
-
response = { results: (response[:results]||[]) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls]||0) + 1 }
|
94
|
+
response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
79
95
|
end
|
80
|
-
rescue
|
81
|
-
response = { results: [], next: nil, error: e.message, code: 500 }
|
96
|
+
rescue StandardError => e
|
97
|
+
response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
|
82
98
|
end
|
83
|
-
|
99
|
+
# If the next cursor is not present we fetched all the data
|
100
|
+
# It happens that twitter returns the same cursor, in that case we stop
|
101
|
+
return response if !parsed_response[:next].to_s.present? || (parsed_response[:next].to_s.present? && parsed_response[:next] == search_options[:next])
|
84
102
|
|
85
103
|
total_by_time_period(query: search_options[:query],
|
86
104
|
date_from: search_options[:fromDate],
|
@@ -89,16 +107,14 @@ module Gnip
|
|
89
107
|
next_cursor: parsed_response[:next],
|
90
108
|
response: response)
|
91
109
|
end
|
92
|
-
|
110
|
+
|
93
111
|
# return total contents in a specific date interval with a passed query
|
94
|
-
def total(options={})
|
112
|
+
def total(options = {})
|
95
113
|
extra = {}
|
96
114
|
response = total_by_time_period(options)
|
97
115
|
extra = { error: response[:error] } if response[:error].present?
|
98
|
-
|
116
|
+
{ query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
|
99
117
|
end
|
100
|
-
|
101
118
|
end
|
102
|
-
|
103
119
|
end
|
104
120
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gnip
|
4
|
+
module GnipFullArchive
|
5
|
+
class ThirtyDay < FullArchive
|
6
|
+
def initialize(client)
|
7
|
+
@search_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}.json"
|
8
|
+
@counts_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}/counts.json"
|
9
|
+
@auth = { username: client.username, password: client.password }
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -1,16 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipRules
|
3
5
|
class Rules
|
4
|
-
|
5
6
|
include HTTParty
|
6
|
-
|
7
|
+
|
7
8
|
attr_reader :rules_url, :version
|
8
|
-
|
9
|
-
def initialize(client, replay=false)
|
9
|
+
|
10
|
+
def initialize(client, replay = false)
|
10
11
|
@version = client.power_track_version
|
11
|
-
case
|
12
|
+
case version
|
12
13
|
when '1.0'
|
13
|
-
@rules_url = "https://api.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/#{replay ?
|
14
|
+
@rules_url = "https://api.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/#{replay ? 'replay' : 'streams'}/track/#{client.label}/rules.json"
|
14
15
|
when '2.0'
|
15
16
|
if replay
|
16
17
|
@rules_url = "https://gnip-api.twitter.com/rules/powertrack-replay/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
@@ -18,111 +19,99 @@ module Gnip
|
|
18
19
|
@rules_url = "https://gnip-api.twitter.com/rules/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
20
|
end
|
20
21
|
else
|
21
|
-
raise Exception
|
22
|
+
raise Exception, "version #{version} is not supported from this gem."
|
22
23
|
end
|
23
24
|
@auth = { username: client.username, password: client.password }
|
24
25
|
end
|
25
|
-
|
26
|
-
#Add rules to PowerTrack rules
|
27
|
-
#rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
26
|
+
|
27
|
+
# Add rules to PowerTrack rules
|
28
|
+
# rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
28
29
|
def add(rules)
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
{ status: :success, code: 200, response: parsed_response }
|
36
|
-
end
|
37
|
-
rescue Exception => e
|
38
|
-
{ status: :error, code: 500, error: e.message }
|
30
|
+
response = self.class.post(rules_url, basic_auth: @auth, body: rules.to_json, headers: { 'Content-Type' => 'application/json', 'Accept' => 'application/json' }, type: :json)
|
31
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
32
|
+
if parsed_response.present? && parsed_response['error'].present?
|
33
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
34
|
+
else
|
35
|
+
{ status: :success, code: 200, response: parsed_response }
|
39
36
|
end
|
40
|
-
|
37
|
+
rescue Exception => e
|
38
|
+
{ status: :error, code: 500, error: e.message }
|
41
39
|
end
|
42
|
-
|
43
|
-
#Remove rules from PowerTrack rules
|
44
|
-
#rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
40
|
+
|
41
|
+
# Remove rules from PowerTrack rules
|
42
|
+
# rules should be an hash in the format {"rules": [{"value": "rule1", "tag": "tag1"}, {"value":"rule2"}]}"
|
45
43
|
def remove(rules)
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
{ status: :success, code: 200, response: parsed_response }
|
53
|
-
end
|
54
|
-
rescue Exception => e
|
55
|
-
{ status: :error, code: 500, error: e.message }
|
44
|
+
response = self.class.post(rules_url, query: { _method: 'delete' }, basic_auth: @auth, body: rules.to_json)
|
45
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
46
|
+
if parsed_response.present? && parsed_response['error'].present?
|
47
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
48
|
+
else
|
49
|
+
{ status: :success, code: 200, response: parsed_response }
|
56
50
|
end
|
57
|
-
|
51
|
+
rescue Exception => e
|
52
|
+
{ status: :error, code: 500, error: e.message }
|
58
53
|
end
|
59
|
-
|
60
|
-
#Get the full list of rules
|
54
|
+
|
55
|
+
# Get the full list of rules
|
61
56
|
def list
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
{ status: :success, code: 200, rules: parsed_response["rules"] }
|
69
|
-
end
|
70
|
-
rescue Exception => e
|
71
|
-
{ status: :error, code: 500, error: e.message }
|
57
|
+
response = self.class.get(rules_url, basic_auth: @auth, headers: { 'Content-Type' => 'application/json', 'Accept' => 'application/json' }, type: :json)
|
58
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
59
|
+
if parsed_response.present? && parsed_response['error'].present?
|
60
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
61
|
+
else
|
62
|
+
{ status: :success, code: 200, rules: parsed_response['rules'] }
|
72
63
|
end
|
64
|
+
rescue Exception => e
|
65
|
+
{ status: :error, code: 500, error: e.message }
|
73
66
|
end
|
74
|
-
|
75
|
-
#Get the full list of rules by tag
|
67
|
+
|
68
|
+
# Get the full list of rules by tag
|
76
69
|
def list_by_tag(tag)
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
{ status: :success, code: 200, rules: rules.select{ |rule| rule["tag"] == tag } }
|
85
|
-
end
|
86
|
-
rescue Exception => e
|
87
|
-
{ status: :error, code: 500, error: e.message }
|
70
|
+
response = self.class.get(rules_url, basic_auth: @auth)
|
71
|
+
parsed_response = safe_parsed_response(response.parsed_response)
|
72
|
+
if parsed_response.present? && parsed_response['error'].present?
|
73
|
+
{ status: :error, code: response.response.code, error: parsed_response['error']['message'] }
|
74
|
+
else
|
75
|
+
rules = parsed_response['rules']
|
76
|
+
{ status: :success, code: 200, rules: rules.select { |rule| rule['tag'] == tag } }
|
88
77
|
end
|
89
|
-
|
78
|
+
rescue Exception => e
|
79
|
+
{ status: :error, code: 500, error: e.message }
|
90
80
|
end
|
91
|
-
|
92
|
-
#delete all rules from PowerTrack
|
93
|
-
#http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
|
94
|
-
#Request Body Size Limit 1 MB (~5000 rules)
|
81
|
+
|
82
|
+
# delete all rules from PowerTrack
|
83
|
+
# http://support.gnip.com/apis/powertrack/api_reference.html#DeleteRules
|
84
|
+
# Request Body Size Limit 1 MB (~5000 rules)
|
95
85
|
def delete_all!
|
96
86
|
retry_times = 0
|
97
87
|
begin
|
98
|
-
rules_list =
|
99
|
-
(rules_list[:rules]||[]).in_groups_of(2, false).each do |group_of_rules|
|
100
|
-
|
88
|
+
rules_list = list
|
89
|
+
(rules_list[:rules] || []).in_groups_of(2, false).each do |group_of_rules|
|
90
|
+
remove("rules": group_of_rules)
|
101
91
|
end
|
102
92
|
sleep 0.05
|
103
|
-
rules_list =
|
104
|
-
if !(rules_list[:rules]||[]).size.zero?
|
105
|
-
|
93
|
+
rules_list = list
|
94
|
+
if !(rules_list[:rules] || []).size.zero?
|
95
|
+
delete_all!
|
106
96
|
else
|
107
|
-
|
97
|
+
{ status: :success, code: 200, rules: [] }
|
108
98
|
end
|
109
99
|
rescue Exception => e
|
110
100
|
retry_times += 1
|
111
101
|
if retry_times <= 3
|
112
102
|
retry
|
113
103
|
else
|
114
|
-
|
115
|
-
end
|
104
|
+
{ status: :error, code: 500, error: e.message }
|
105
|
+
end
|
116
106
|
end
|
117
|
-
|
118
107
|
end
|
119
|
-
|
108
|
+
|
120
109
|
private
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
110
|
+
|
111
|
+
def safe_parsed_response(parsed_response)
|
112
|
+
ret = parsed_response.present? ? (parsed_response.is_a?(String) ? JSON.parse(parsed_response).with_indifferent_access : parsed_response) : nil
|
113
|
+
ret
|
114
|
+
end
|
126
115
|
end
|
127
116
|
end
|
128
117
|
end
|
@@ -1,8 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
module GnipStream
|
3
|
-
|
4
5
|
class ErrorReconnect
|
5
|
-
|
6
6
|
def initialize(source_class, method_name)
|
7
7
|
@source_class = source_class
|
8
8
|
@method_name = method_name
|
@@ -12,7 +12,7 @@ module Gnip
|
|
12
12
|
def attempt_to_reconnect(error_message)
|
13
13
|
@error_message = error_message
|
14
14
|
if @reconnect_attempts < 5
|
15
|
-
@reconnect_attempts +=1
|
15
|
+
@reconnect_attempts += 1
|
16
16
|
sleep(2)
|
17
17
|
@source_class.send(@method_name)
|
18
18
|
else
|
@@ -23,7 +23,6 @@ module Gnip
|
|
23
23
|
def reconnect_failed_raise_error
|
24
24
|
raise @error_message
|
25
25
|
end
|
26
|
-
|
27
26
|
end
|
28
27
|
end
|
29
|
-
end
|
28
|
+
end
|
@@ -1,13 +1,12 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
|
-
class JsonDataBuffer
|
4
|
-
|
3
|
+
class JsonDataBuffer
|
5
4
|
attr_accessor :split_pattern, :check_pattern
|
6
|
-
|
5
|
+
|
7
6
|
def initialize(split_pattern, check_pattern)
|
8
7
|
@split_pattern = split_pattern
|
9
8
|
@check_pattern = check_pattern
|
10
|
-
@buffer =
|
9
|
+
@buffer = ''
|
11
10
|
end
|
12
11
|
|
13
12
|
def process(chunk)
|
@@ -21,11 +20,10 @@ module Gnip
|
|
21
20
|
activities = @buffer.split(split_pattern)
|
22
21
|
entries << activities.shift
|
23
22
|
@buffer = activities.join(split_pattern)
|
24
|
-
@buffer
|
23
|
+
@buffer += "\r\n" if !@buffer.empty? && new_line
|
25
24
|
end
|
26
|
-
entries.
|
25
|
+
entries.reject(&:empty?)
|
27
26
|
end
|
28
27
|
end
|
29
|
-
|
30
28
|
end
|
31
|
-
end
|
29
|
+
end
|
@@ -1,49 +1,47 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
3
|
class Replay < Stream
|
4
|
-
|
5
4
|
def initialize(client)
|
6
|
-
super #version is setted in the super
|
7
|
-
case
|
5
|
+
super # version is setted in the super
|
6
|
+
case version
|
8
7
|
when '1.0'
|
9
8
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/replay/track/#{client.replay_label}.json"
|
10
9
|
when '2.0'
|
11
10
|
@url = "https://gnip-stream.gnip.com/replay/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
12
11
|
else
|
13
|
-
raise Exception.new("version #{
|
12
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
14
13
|
end
|
15
14
|
end
|
16
|
-
|
15
|
+
|
17
16
|
def configure_handlers
|
18
|
-
|
19
|
-
|
17
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
18
|
+
on_connection_close { puts 'Gnip::GnipStream::Replay -> Connection closed' }
|
20
19
|
end
|
21
|
-
|
22
|
-
def consume(options={}, &block)
|
20
|
+
|
21
|
+
def consume(options = {}, &block)
|
23
22
|
@client_callback = block if block
|
24
|
-
|
25
|
-
|
23
|
+
on_message(&@client_callback)
|
24
|
+
connect(options)
|
26
25
|
end
|
27
|
-
|
26
|
+
|
28
27
|
def connect(options)
|
29
28
|
search_options = {}
|
30
29
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
31
30
|
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
32
|
-
stream_url = [
|
31
|
+
stream_url = [url, search_options.to_query].join('?')
|
33
32
|
EM.run do
|
34
33
|
http = EM::HttpRequest.new(stream_url, inactivity_timeout: 45, connection_timeout: 75).get(head: @headers)
|
35
34
|
http.stream { |chunk| process_chunk(chunk) }
|
36
|
-
http.callback {
|
37
|
-
handle_connection_close(http)
|
35
|
+
http.callback {
|
36
|
+
handle_connection_close(http)
|
38
37
|
EM.stop
|
39
38
|
}
|
40
|
-
http.errback {
|
39
|
+
http.errback {
|
41
40
|
handle_error(http)
|
42
41
|
EM.stop
|
43
42
|
}
|
44
43
|
end
|
45
44
|
end
|
46
|
-
|
47
45
|
end
|
48
46
|
end
|
49
|
-
end
|
47
|
+
end
|
@@ -4,40 +4,39 @@ require 'em-http-request'
|
|
4
4
|
module Gnip
|
5
5
|
module GnipStream
|
6
6
|
class Stream
|
7
|
-
|
8
7
|
EventMachine.threadpool_size = 5
|
9
8
|
|
10
9
|
attr_accessor :url, :backfill_client, :version
|
11
10
|
|
12
11
|
def initialize(client)
|
13
12
|
self.version = client.power_track_version
|
14
|
-
case
|
13
|
+
case version
|
15
14
|
when '1.0'
|
16
15
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/streams/track/#{client.label}.json"
|
17
16
|
when '2.0'
|
18
17
|
@url = "https://gnip-stream.twitter.com/stream/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
18
|
else
|
20
|
-
raise Exception.new("version #{
|
19
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
21
20
|
end
|
22
21
|
@backfill_client = client.backfill_client
|
23
22
|
@processor = JsonDataBuffer.new("\r\n", Regexp.new(/^\{.*\}\r\n/))
|
24
|
-
@headers = {'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed'}
|
23
|
+
@headers = { 'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed' }
|
25
24
|
@error_handler = ErrorReconnect.new(self, :consume)
|
26
25
|
@connection_close_handler = ErrorReconnect.new(self, :consume)
|
27
26
|
configure_handlers
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def configure_handlers
|
31
|
-
|
32
|
-
|
30
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
31
|
+
on_connection_close { @connection_close_handler.attempt_to_reconnect('Gnip Connection Closed') }
|
33
32
|
end
|
34
33
|
|
35
34
|
def consume(&block)
|
36
35
|
@client_callback = block if block
|
37
|
-
|
38
|
-
|
36
|
+
on_message(&@client_callback)
|
37
|
+
connect
|
39
38
|
end
|
40
|
-
|
39
|
+
|
41
40
|
def on_message(&block)
|
42
41
|
@on_message = block
|
43
42
|
end
|
@@ -53,14 +52,14 @@ module Gnip
|
|
53
52
|
def connect
|
54
53
|
EM.run do
|
55
54
|
options = {}
|
56
|
-
options = { query: {
|
57
|
-
http = EM::HttpRequest.new(
|
55
|
+
options = { query: { 'client' => backfill_client } } if backfill_client.present?
|
56
|
+
http = EM::HttpRequest.new(url, inactivity_timeout: 45, connection_timeout: 75).get({ head: @headers }.merge!(options))
|
58
57
|
http.stream { |chunk| process_chunk(chunk) }
|
59
|
-
http.callback {
|
60
|
-
handle_connection_close(http)
|
58
|
+
http.callback {
|
59
|
+
handle_connection_close(http)
|
61
60
|
EM.stop
|
62
61
|
}
|
63
|
-
http.errback {
|
62
|
+
http.errback {
|
64
63
|
handle_error(http)
|
65
64
|
EM.stop
|
66
65
|
}
|
@@ -81,7 +80,6 @@ module Gnip
|
|
81
80
|
def handle_connection_close(http_connection)
|
82
81
|
@on_connection_close.call(http_connection)
|
83
82
|
end
|
84
|
-
|
85
83
|
end
|
86
84
|
end
|
87
|
-
end
|
85
|
+
end
|
@@ -1,27 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Gnip
|
2
4
|
class PowerTrackClient
|
3
|
-
|
4
5
|
attr_accessor :publisher, :label, :account,
|
5
6
|
:username, :password,
|
6
7
|
:backfill_client, :replay_label
|
7
|
-
|
8
|
-
attr_reader :rules, :replay_rules, :full_archive, :stream, :replay, :power_track_version
|
9
|
-
|
8
|
+
|
9
|
+
attr_reader :rules, :replay_rules, :full_archive, :thirty_day, :stream, :replay, :power_track_version
|
10
|
+
|
10
11
|
def initialize(options = {})
|
11
12
|
@account = options[:account]
|
12
|
-
@publisher = options[:publisher]||
|
13
|
-
@label = options[:label]||
|
14
|
-
@replay_label = options[:replay_label]
|
13
|
+
@publisher = options[:publisher] || 'twitter'
|
14
|
+
@label = options[:label] || 'dev'
|
15
|
+
@replay_label = options[:replay_label] || @label
|
15
16
|
@username = options[:username]
|
16
17
|
@password = options[:password]
|
17
|
-
@backfill_client = options[:backfill_client]||nil
|
18
|
-
@power_track_version = options[:power_track_version]||'2.0'
|
18
|
+
@backfill_client = options[:backfill_client] || nil
|
19
|
+
@power_track_version = options[:power_track_version] || '2.0'
|
19
20
|
@rules = Gnip::GnipRules::Rules.new(self)
|
20
21
|
@replay_rules = Gnip::GnipRules::Rules.new(self, true)
|
21
22
|
@full_archive = Gnip::GnipFullArchive::FullArchive.new(self)
|
23
|
+
@thirty_day = Gnip::GnipFullArchive::ThirtyDay.new(self)
|
22
24
|
@stream = Gnip::GnipStream::Stream.new(self)
|
23
25
|
@replay = Gnip::GnipStream::Replay.new(self)
|
24
26
|
end
|
25
|
-
|
26
27
|
end
|
27
28
|
end
|
data/lib/gnip/version.rb
CHANGED
metadata
CHANGED
@@ -1,113 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gnip-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Duccio Giovannelli
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-11-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.9'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.9'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '10.0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '10.0'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: pry
|
14
|
+
name: activesupport
|
43
15
|
requirement: !ruby/object:Gem::Requirement
|
44
16
|
requirements:
|
45
|
-
- - "
|
17
|
+
- - ">="
|
46
18
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
49
21
|
prerelease: false
|
50
22
|
version_requirements: !ruby/object:Gem::Requirement
|
51
23
|
requirements:
|
52
|
-
- - "
|
24
|
+
- - ">="
|
53
25
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
26
|
+
version: '4.2'
|
55
27
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
28
|
+
name: em-http-request
|
57
29
|
requirement: !ruby/object:Gem::Requirement
|
58
30
|
requirements:
|
59
31
|
- - "~>"
|
60
32
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
62
|
-
type: :
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
63
35
|
prerelease: false
|
64
36
|
version_requirements: !ruby/object:Gem::Requirement
|
65
37
|
requirements:
|
66
38
|
- - "~>"
|
67
39
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
40
|
+
version: '1'
|
69
41
|
- !ruby/object:Gem::Dependency
|
70
42
|
name: httparty
|
71
43
|
requirement: !ruby/object:Gem::Requirement
|
72
44
|
requirements:
|
73
45
|
- - "~>"
|
74
46
|
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
47
|
+
version: '0.16'
|
76
48
|
type: :runtime
|
77
49
|
prerelease: false
|
78
50
|
version_requirements: !ruby/object:Gem::Requirement
|
79
51
|
requirements:
|
80
52
|
- - "~>"
|
81
53
|
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
54
|
+
version: '0.16'
|
83
55
|
- !ruby/object:Gem::Dependency
|
84
|
-
name:
|
56
|
+
name: rake
|
85
57
|
requirement: !ruby/object:Gem::Requirement
|
86
58
|
requirements:
|
87
59
|
- - "~>"
|
88
60
|
- !ruby/object:Gem::Version
|
89
|
-
version: '
|
90
|
-
type: :
|
61
|
+
version: '13.0'
|
62
|
+
type: :development
|
91
63
|
prerelease: false
|
92
64
|
version_requirements: !ruby/object:Gem::Requirement
|
93
65
|
requirements:
|
94
66
|
- - "~>"
|
95
67
|
- !ruby/object:Gem::Version
|
96
|
-
version: '
|
68
|
+
version: '13.0'
|
97
69
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
70
|
+
name: rspec
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
100
72
|
requirements:
|
101
73
|
- - "~>"
|
102
74
|
- !ruby/object:Gem::Version
|
103
|
-
version: '
|
104
|
-
type: :
|
75
|
+
version: '3'
|
76
|
+
type: :development
|
105
77
|
prerelease: false
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
107
79
|
requirements:
|
108
80
|
- - "~>"
|
109
81
|
- !ruby/object:Gem::Version
|
110
|
-
version: '
|
82
|
+
version: '3'
|
111
83
|
description:
|
112
84
|
email:
|
113
85
|
- giovannelli@extendi.it
|
@@ -129,6 +101,7 @@ files:
|
|
129
101
|
- gnip.gemspec
|
130
102
|
- lib/gnip.rb
|
131
103
|
- lib/gnip/gnip-full-archive/full_archive.rb
|
104
|
+
- lib/gnip/gnip-full-archive/thirty_day.rb
|
132
105
|
- lib/gnip/gnip-rules/rules.rb
|
133
106
|
- lib/gnip/gnip-stream/error_reconnect.rb
|
134
107
|
- lib/gnip/gnip-stream/json_data_bufffer.rb
|
@@ -155,8 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
128
|
- !ruby/object:Gem::Version
|
156
129
|
version: '0'
|
157
130
|
requirements: []
|
158
|
-
|
159
|
-
rubygems_version: 2.5.2
|
131
|
+
rubygems_version: 3.1.4
|
160
132
|
signing_key:
|
161
133
|
specification_version: 4
|
162
134
|
summary: A Ruby library for accessing the Gnip API. See https://gnip.com/ for full
|