gnip-client 0.2.11 → 0.2.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +2 -1
- data/Gemfile +1 -1
- data/Rakefile +1 -1
- data/bin/console +3 -3
- data/gnip.gemspec +5 -6
- data/lib/gnip.rb +2 -4
- data/lib/gnip/gnip-full-archive/full_archive.rb +13 -13
- data/lib/gnip/gnip-full-archive/thirty_day.rb +0 -2
- data/lib/gnip/gnip-rules/rules.rb +5 -5
- data/lib/gnip/gnip-stream/replay.rb +16 -18
- data/lib/gnip/gnip-stream/stream.rb +15 -17
- data/lib/gnip/version.rb +1 -1
- metadata +24 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 00cc97da9051a8751dfd2858070e4a3efa2a4c9ffa79006a74abc34ea8e572cc
|
4
|
+
data.tar.gz: d261740d92796c40c3afe31fc19c644314c8cf95a838774258bf5c2757152ca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbe3c18c8238b98f3a2d97a7573bd0cb3d32587a49a14f663d262c042ccfe501df5adfe7b69f3f3a5095a16cf56fac687d22cd627dbd60a25ccb9d295079e434
|
7
|
+
data.tar.gz: 6d798c3988a6bae2b9af93c9eaaaa4fe5e4b268d04380a0df5cb8c266b150c6f2492fafc360e3102e966daba339a2f03e35025216bc6298fad800faa9079b8fa
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.7.2
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/console
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'gnip'
|
5
5
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +10,5 @@ require "gnip"
|
|
10
10
|
# require "pry"
|
11
11
|
# Pry.start
|
12
12
|
|
13
|
-
require
|
13
|
+
require 'irb'
|
14
14
|
IRB.start
|
data/gnip.gemspec
CHANGED
@@ -19,12 +19,11 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.add_development_dependency 'bundler', '~> 1.9'
|
23
|
-
spec.add_development_dependency 'pry', '~> 0'
|
24
|
-
spec.add_development_dependency 'rake', '~> 10.0'
|
25
|
-
spec.add_development_dependency 'rspec', '~> 3'
|
26
|
-
|
27
22
|
spec.add_dependency 'activesupport', '>= 4.2'
|
28
23
|
spec.add_dependency 'em-http-request', '~> 1'
|
29
|
-
spec.add_dependency 'httparty', '~> 0.16
|
24
|
+
spec.add_dependency 'httparty', '~> 0.16'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3'
|
28
|
+
|
30
29
|
end
|
data/lib/gnip.rb
CHANGED
@@ -11,15 +11,13 @@ require 'gnip/gnip-stream/stream'
|
|
11
11
|
require 'gnip/gnip-stream/replay'
|
12
12
|
|
13
13
|
begin
|
14
|
-
require
|
14
|
+
require 'pry'
|
15
15
|
rescue LoadError => e
|
16
|
-
|
16
|
+
|
17
17
|
end
|
18
18
|
|
19
19
|
module Gnip
|
20
|
-
|
21
20
|
def self.format_date(datetime)
|
22
21
|
datetime.to_datetime.utc.strftime('%Y%m%d%H%M')
|
23
22
|
end
|
24
|
-
|
25
23
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module Gnip
|
4
4
|
module GnipFullArchive
|
5
5
|
class FullArchive
|
6
|
-
class InvalidRequestException < StandardError
|
6
|
+
class InvalidRequestException < StandardError
|
7
7
|
end
|
8
8
|
|
9
9
|
include HTTParty
|
@@ -15,7 +15,7 @@ module Gnip
|
|
15
15
|
def initialize(client)
|
16
16
|
@search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json"
|
17
17
|
@counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json"
|
18
|
-
@auth = {username: client.username, password: client.password}
|
18
|
+
@auth = { username: client.username, password: client.password }
|
19
19
|
end
|
20
20
|
|
21
21
|
# Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page
|
@@ -40,23 +40,23 @@ module Gnip
|
|
40
40
|
raise response.message unless parsed_response.present?
|
41
41
|
|
42
42
|
if parsed_response[:error].present?
|
43
|
-
response = {results: [],
|
43
|
+
response = { results: [],
|
44
44
|
next: nil,
|
45
45
|
url: url,
|
46
46
|
error: parsed_response[:error][:message],
|
47
|
-
code: response.code.to_i}
|
47
|
+
code: response.code.to_i }
|
48
48
|
else
|
49
|
-
response = {results: parsed_response[:results],
|
49
|
+
response = { results: parsed_response[:results],
|
50
50
|
url: url,
|
51
51
|
next: parsed_response[:next],
|
52
|
-
code: response.code.to_i}
|
52
|
+
code: response.code.to_i }
|
53
53
|
end
|
54
54
|
rescue StandardError => e
|
55
|
-
response = {results: [],
|
55
|
+
response = { results: [],
|
56
56
|
url: url,
|
57
57
|
next: nil,
|
58
58
|
error: e.message,
|
59
|
-
code: 500}
|
59
|
+
code: 500 }
|
60
60
|
end
|
61
61
|
response
|
62
62
|
end
|
@@ -85,16 +85,16 @@ module Gnip
|
|
85
85
|
raise gnip_call.response.message unless parsed_response.present?
|
86
86
|
|
87
87
|
if parsed_response[:error].present?
|
88
|
-
response = {results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
88
|
+
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
89
89
|
else
|
90
90
|
call_done = 1 # we have received a valid response
|
91
91
|
parsed_response[:results].each_with_index do |item, i|
|
92
92
|
parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
|
93
93
|
end
|
94
|
-
response = {results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
94
|
+
response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
95
95
|
end
|
96
96
|
rescue StandardError => e
|
97
|
-
response = {results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done}
|
97
|
+
response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
|
98
98
|
end
|
99
99
|
# If the next cursor is not present we fetched all the data
|
100
100
|
# It happens that twitter returns the same cursor, in that case we stop
|
@@ -112,8 +112,8 @@ module Gnip
|
|
112
112
|
def total(options = {})
|
113
113
|
extra = {}
|
114
114
|
response = total_by_time_period(options)
|
115
|
-
extra = {error: response[:error]} if response[:error].present?
|
116
|
-
{query: options[:query], total: response[:results].map {|item| item[:count]}.reduce(:+), calls: response[:calls]}.merge!(extra)
|
115
|
+
extra = { error: response[:error] } if response[:error].present?
|
116
|
+
{ query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
|
117
117
|
end
|
118
118
|
end
|
119
119
|
end
|
@@ -3,13 +3,11 @@
|
|
3
3
|
module Gnip
|
4
4
|
module GnipFullArchive
|
5
5
|
class ThirtyDay < FullArchive
|
6
|
-
|
7
6
|
def initialize(client)
|
8
7
|
@search_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}.json"
|
9
8
|
@counts_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}/counts.json"
|
10
9
|
@auth = { username: client.username, password: client.password }
|
11
10
|
end
|
12
|
-
|
13
11
|
end
|
14
12
|
end
|
15
13
|
end
|
@@ -94,7 +94,7 @@ module Gnip
|
|
94
94
|
if !(rules_list[:rules] || []).size.zero?
|
95
95
|
delete_all!
|
96
96
|
else
|
97
|
-
|
97
|
+
{ status: :success, code: 200, rules: [] }
|
98
98
|
end
|
99
99
|
rescue Exception => e
|
100
100
|
retry_times += 1
|
@@ -108,10 +108,10 @@ module Gnip
|
|
108
108
|
|
109
109
|
private
|
110
110
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
111
|
+
def safe_parsed_response(parsed_response)
|
112
|
+
ret = parsed_response.present? ? (parsed_response.is_a?(String) ? JSON.parse(parsed_response).with_indifferent_access : parsed_response) : nil
|
113
|
+
ret
|
114
|
+
end
|
115
115
|
end
|
116
116
|
end
|
117
117
|
end
|
@@ -1,49 +1,47 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
3
|
class Replay < Stream
|
4
|
-
|
5
4
|
def initialize(client)
|
6
|
-
super #version is setted in the super
|
7
|
-
case
|
5
|
+
super # version is setted in the super
|
6
|
+
case version
|
8
7
|
when '1.0'
|
9
8
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/replay/track/#{client.replay_label}.json"
|
10
9
|
when '2.0'
|
11
10
|
@url = "https://gnip-stream.gnip.com/replay/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
12
11
|
else
|
13
|
-
raise Exception.new("version #{
|
12
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
14
13
|
end
|
15
14
|
end
|
16
|
-
|
15
|
+
|
17
16
|
def configure_handlers
|
18
|
-
|
19
|
-
|
17
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
18
|
+
on_connection_close { puts 'Gnip::GnipStream::Replay -> Connection closed' }
|
20
19
|
end
|
21
|
-
|
22
|
-
def consume(options={}, &block)
|
20
|
+
|
21
|
+
def consume(options = {}, &block)
|
23
22
|
@client_callback = block if block
|
24
|
-
|
25
|
-
|
23
|
+
on_message(&@client_callback)
|
24
|
+
connect(options)
|
26
25
|
end
|
27
|
-
|
26
|
+
|
28
27
|
def connect(options)
|
29
28
|
search_options = {}
|
30
29
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
31
30
|
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
32
|
-
stream_url = [
|
31
|
+
stream_url = [url, search_options.to_query].join('?')
|
33
32
|
EM.run do
|
34
33
|
http = EM::HttpRequest.new(stream_url, inactivity_timeout: 45, connection_timeout: 75).get(head: @headers)
|
35
34
|
http.stream { |chunk| process_chunk(chunk) }
|
36
|
-
http.callback {
|
37
|
-
handle_connection_close(http)
|
35
|
+
http.callback {
|
36
|
+
handle_connection_close(http)
|
38
37
|
EM.stop
|
39
38
|
}
|
40
|
-
http.errback {
|
39
|
+
http.errback {
|
41
40
|
handle_error(http)
|
42
41
|
EM.stop
|
43
42
|
}
|
44
43
|
end
|
45
44
|
end
|
46
|
-
|
47
45
|
end
|
48
46
|
end
|
49
|
-
end
|
47
|
+
end
|
@@ -4,40 +4,39 @@ require 'em-http-request'
|
|
4
4
|
module Gnip
|
5
5
|
module GnipStream
|
6
6
|
class Stream
|
7
|
-
|
8
7
|
EventMachine.threadpool_size = 5
|
9
8
|
|
10
9
|
attr_accessor :url, :backfill_client, :version
|
11
10
|
|
12
11
|
def initialize(client)
|
13
12
|
self.version = client.power_track_version
|
14
|
-
case
|
13
|
+
case version
|
15
14
|
when '1.0'
|
16
15
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/streams/track/#{client.label}.json"
|
17
16
|
when '2.0'
|
18
17
|
@url = "https://gnip-stream.twitter.com/stream/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
18
|
else
|
20
|
-
raise Exception.new("version #{
|
19
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
21
20
|
end
|
22
21
|
@backfill_client = client.backfill_client
|
23
22
|
@processor = JsonDataBuffer.new("\r\n", Regexp.new(/^\{.*\}\r\n/))
|
24
|
-
@headers = {'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed'}
|
23
|
+
@headers = { 'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed' }
|
25
24
|
@error_handler = ErrorReconnect.new(self, :consume)
|
26
25
|
@connection_close_handler = ErrorReconnect.new(self, :consume)
|
27
26
|
configure_handlers
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def configure_handlers
|
31
|
-
|
32
|
-
|
30
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
31
|
+
on_connection_close { @connection_close_handler.attempt_to_reconnect('Gnip Connection Closed') }
|
33
32
|
end
|
34
33
|
|
35
34
|
def consume(&block)
|
36
35
|
@client_callback = block if block
|
37
|
-
|
38
|
-
|
36
|
+
on_message(&@client_callback)
|
37
|
+
connect
|
39
38
|
end
|
40
|
-
|
39
|
+
|
41
40
|
def on_message(&block)
|
42
41
|
@on_message = block
|
43
42
|
end
|
@@ -53,14 +52,14 @@ module Gnip
|
|
53
52
|
def connect
|
54
53
|
EM.run do
|
55
54
|
options = {}
|
56
|
-
options = { query: {
|
57
|
-
http = EM::HttpRequest.new(
|
55
|
+
options = { query: { 'client' => backfill_client } } if backfill_client.present?
|
56
|
+
http = EM::HttpRequest.new(url, inactivity_timeout: 45, connection_timeout: 75).get({ head: @headers }.merge!(options))
|
58
57
|
http.stream { |chunk| process_chunk(chunk) }
|
59
|
-
http.callback {
|
60
|
-
handle_connection_close(http)
|
58
|
+
http.callback {
|
59
|
+
handle_connection_close(http)
|
61
60
|
EM.stop
|
62
61
|
}
|
63
|
-
http.errback {
|
62
|
+
http.errback {
|
64
63
|
handle_error(http)
|
65
64
|
EM.stop
|
66
65
|
}
|
@@ -81,7 +80,6 @@ module Gnip
|
|
81
80
|
def handle_connection_close(http_connection)
|
82
81
|
@on_connection_close.call(http_connection)
|
83
82
|
end
|
84
|
-
|
85
83
|
end
|
86
84
|
end
|
87
|
-
end
|
85
|
+
end
|
data/lib/gnip/version.rb
CHANGED
metadata
CHANGED
@@ -1,113 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gnip-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Duccio Giovannelli
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-11-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
20
|
-
type: :
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '4.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: em-http-request
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
34
|
-
type: :
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: httparty
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
47
|
+
version: '0.16'
|
48
|
+
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0.16'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '13.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: activesupport
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '4.2'
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '4.2'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: em-http-request
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '1'
|
90
|
-
type: :runtime
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '1'
|
68
|
+
version: '13.0'
|
97
69
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
70
|
+
name: rspec
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
100
72
|
requirements:
|
101
73
|
- - "~>"
|
102
74
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
104
|
-
type: :
|
75
|
+
version: '3'
|
76
|
+
type: :development
|
105
77
|
prerelease: false
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
107
79
|
requirements:
|
108
80
|
- - "~>"
|
109
81
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
82
|
+
version: '3'
|
111
83
|
description:
|
112
84
|
email:
|
113
85
|
- giovannelli@extendi.it
|
@@ -156,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
128
|
- !ruby/object:Gem::Version
|
157
129
|
version: '0'
|
158
130
|
requirements: []
|
159
|
-
rubygems_version: 3.
|
131
|
+
rubygems_version: 3.1.4
|
160
132
|
signing_key:
|
161
133
|
specification_version: 4
|
162
134
|
summary: A Ruby library for accessing the Gnip API. See https://gnip.com/ for full
|