gnip-client 0.2.11 → 0.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +2 -1
- data/Gemfile +1 -1
- data/Rakefile +1 -1
- data/bin/console +3 -3
- data/gnip.gemspec +5 -6
- data/lib/gnip.rb +2 -4
- data/lib/gnip/gnip-full-archive/full_archive.rb +13 -13
- data/lib/gnip/gnip-full-archive/thirty_day.rb +0 -2
- data/lib/gnip/gnip-rules/rules.rb +5 -5
- data/lib/gnip/gnip-stream/replay.rb +16 -18
- data/lib/gnip/gnip-stream/stream.rb +15 -17
- data/lib/gnip/version.rb +1 -1
- metadata +24 -52
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 00cc97da9051a8751dfd2858070e4a3efa2a4c9ffa79006a74abc34ea8e572cc
|
4
|
+
data.tar.gz: d261740d92796c40c3afe31fc19c644314c8cf95a838774258bf5c2757152ca2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbe3c18c8238b98f3a2d97a7573bd0cb3d32587a49a14f663d262c042ccfe501df5adfe7b69f3f3a5095a16cf56fac687d22cd627dbd60a25ccb9d295079e434
|
7
|
+
data.tar.gz: 6d798c3988a6bae2b9af93c9eaaaa4fe5e4b268d04380a0df5cb8c266b150c6f2492fafc360e3102e966daba339a2f03e35025216bc6298fad800faa9079b8fa
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.7.2
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/Rakefile
CHANGED
@@ -1 +1 @@
|
|
1
|
-
require
|
1
|
+
require 'bundler/gem_tasks'
|
data/bin/console
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'gnip'
|
5
5
|
|
6
6
|
# You can add fixtures and/or initialization code here to make experimenting
|
7
7
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -10,5 +10,5 @@ require "gnip"
|
|
10
10
|
# require "pry"
|
11
11
|
# Pry.start
|
12
12
|
|
13
|
-
require
|
13
|
+
require 'irb'
|
14
14
|
IRB.start
|
data/gnip.gemspec
CHANGED
@@ -19,12 +19,11 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.add_development_dependency 'bundler', '~> 1.9'
|
23
|
-
spec.add_development_dependency 'pry', '~> 0'
|
24
|
-
spec.add_development_dependency 'rake', '~> 10.0'
|
25
|
-
spec.add_development_dependency 'rspec', '~> 3'
|
26
|
-
|
27
22
|
spec.add_dependency 'activesupport', '>= 4.2'
|
28
23
|
spec.add_dependency 'em-http-request', '~> 1'
|
29
|
-
spec.add_dependency 'httparty', '~> 0.16
|
24
|
+
spec.add_dependency 'httparty', '~> 0.16'
|
25
|
+
|
26
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3'
|
28
|
+
|
30
29
|
end
|
data/lib/gnip.rb
CHANGED
@@ -11,15 +11,13 @@ require 'gnip/gnip-stream/stream'
|
|
11
11
|
require 'gnip/gnip-stream/replay'
|
12
12
|
|
13
13
|
begin
|
14
|
-
require
|
14
|
+
require 'pry'
|
15
15
|
rescue LoadError => e
|
16
|
-
|
16
|
+
|
17
17
|
end
|
18
18
|
|
19
19
|
module Gnip
|
20
|
-
|
21
20
|
def self.format_date(datetime)
|
22
21
|
datetime.to_datetime.utc.strftime('%Y%m%d%H%M')
|
23
22
|
end
|
24
|
-
|
25
23
|
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module Gnip
|
4
4
|
module GnipFullArchive
|
5
5
|
class FullArchive
|
6
|
-
class InvalidRequestException < StandardError
|
6
|
+
class InvalidRequestException < StandardError
|
7
7
|
end
|
8
8
|
|
9
9
|
include HTTParty
|
@@ -15,7 +15,7 @@ module Gnip
|
|
15
15
|
def initialize(client)
|
16
16
|
@search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json"
|
17
17
|
@counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json"
|
18
|
-
@auth = {username: client.username, password: client.password}
|
18
|
+
@auth = { username: client.username, password: client.password }
|
19
19
|
end
|
20
20
|
|
21
21
|
# Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page
|
@@ -40,23 +40,23 @@ module Gnip
|
|
40
40
|
raise response.message unless parsed_response.present?
|
41
41
|
|
42
42
|
if parsed_response[:error].present?
|
43
|
-
response = {results: [],
|
43
|
+
response = { results: [],
|
44
44
|
next: nil,
|
45
45
|
url: url,
|
46
46
|
error: parsed_response[:error][:message],
|
47
|
-
code: response.code.to_i}
|
47
|
+
code: response.code.to_i }
|
48
48
|
else
|
49
|
-
response = {results: parsed_response[:results],
|
49
|
+
response = { results: parsed_response[:results],
|
50
50
|
url: url,
|
51
51
|
next: parsed_response[:next],
|
52
|
-
code: response.code.to_i}
|
52
|
+
code: response.code.to_i }
|
53
53
|
end
|
54
54
|
rescue StandardError => e
|
55
|
-
response = {results: [],
|
55
|
+
response = { results: [],
|
56
56
|
url: url,
|
57
57
|
next: nil,
|
58
58
|
error: e.message,
|
59
|
-
code: 500}
|
59
|
+
code: 500 }
|
60
60
|
end
|
61
61
|
response
|
62
62
|
end
|
@@ -85,16 +85,16 @@ module Gnip
|
|
85
85
|
raise gnip_call.response.message unless parsed_response.present?
|
86
86
|
|
87
87
|
if parsed_response[:error].present?
|
88
|
-
response = {results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
88
|
+
response = { results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
89
89
|
else
|
90
90
|
call_done = 1 # we have received a valid response
|
91
91
|
parsed_response[:results].each_with_index do |item, i|
|
92
92
|
parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
|
93
93
|
end
|
94
|
-
response = {results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
94
|
+
response = { results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1 }
|
95
95
|
end
|
96
96
|
rescue StandardError => e
|
97
|
-
response = {results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done}
|
97
|
+
response = { results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done }
|
98
98
|
end
|
99
99
|
# If the next cursor is not present we fetched all the data
|
100
100
|
# It happens that twitter returns the same cursor, in that case we stop
|
@@ -112,8 +112,8 @@ module Gnip
|
|
112
112
|
def total(options = {})
|
113
113
|
extra = {}
|
114
114
|
response = total_by_time_period(options)
|
115
|
-
extra = {error: response[:error]} if response[:error].present?
|
116
|
-
{query: options[:query], total: response[:results].map {|item| item[:count]}.reduce(:+), calls: response[:calls]}.merge!(extra)
|
115
|
+
extra = { error: response[:error] } if response[:error].present?
|
116
|
+
{ query: options[:query], total: response[:results].map { |item| item[:count] }.reduce(:+), calls: response[:calls] }.merge!(extra)
|
117
117
|
end
|
118
118
|
end
|
119
119
|
end
|
@@ -3,13 +3,11 @@
|
|
3
3
|
module Gnip
|
4
4
|
module GnipFullArchive
|
5
5
|
class ThirtyDay < FullArchive
|
6
|
-
|
7
6
|
def initialize(client)
|
8
7
|
@search_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}.json"
|
9
8
|
@counts_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}/counts.json"
|
10
9
|
@auth = { username: client.username, password: client.password }
|
11
10
|
end
|
12
|
-
|
13
11
|
end
|
14
12
|
end
|
15
13
|
end
|
@@ -94,7 +94,7 @@ module Gnip
|
|
94
94
|
if !(rules_list[:rules] || []).size.zero?
|
95
95
|
delete_all!
|
96
96
|
else
|
97
|
-
|
97
|
+
{ status: :success, code: 200, rules: [] }
|
98
98
|
end
|
99
99
|
rescue Exception => e
|
100
100
|
retry_times += 1
|
@@ -108,10 +108,10 @@ module Gnip
|
|
108
108
|
|
109
109
|
private
|
110
110
|
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
111
|
+
def safe_parsed_response(parsed_response)
|
112
|
+
ret = parsed_response.present? ? (parsed_response.is_a?(String) ? JSON.parse(parsed_response).with_indifferent_access : parsed_response) : nil
|
113
|
+
ret
|
114
|
+
end
|
115
115
|
end
|
116
116
|
end
|
117
117
|
end
|
@@ -1,49 +1,47 @@
|
|
1
1
|
module Gnip
|
2
2
|
module GnipStream
|
3
3
|
class Replay < Stream
|
4
|
-
|
5
4
|
def initialize(client)
|
6
|
-
super #version is setted in the super
|
7
|
-
case
|
5
|
+
super # version is setted in the super
|
6
|
+
case version
|
8
7
|
when '1.0'
|
9
8
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/replay/track/#{client.replay_label}.json"
|
10
9
|
when '2.0'
|
11
10
|
@url = "https://gnip-stream.gnip.com/replay/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.replay_label}.json"
|
12
11
|
else
|
13
|
-
raise Exception.new("version #{
|
12
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
14
13
|
end
|
15
14
|
end
|
16
|
-
|
15
|
+
|
17
16
|
def configure_handlers
|
18
|
-
|
19
|
-
|
17
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
18
|
+
on_connection_close { puts 'Gnip::GnipStream::Replay -> Connection closed' }
|
20
19
|
end
|
21
|
-
|
22
|
-
def consume(options={}, &block)
|
20
|
+
|
21
|
+
def consume(options = {}, &block)
|
23
22
|
@client_callback = block if block
|
24
|
-
|
25
|
-
|
23
|
+
on_message(&@client_callback)
|
24
|
+
connect(options)
|
26
25
|
end
|
27
|
-
|
26
|
+
|
28
27
|
def connect(options)
|
29
28
|
search_options = {}
|
30
29
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
31
30
|
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
32
|
-
stream_url = [
|
31
|
+
stream_url = [url, search_options.to_query].join('?')
|
33
32
|
EM.run do
|
34
33
|
http = EM::HttpRequest.new(stream_url, inactivity_timeout: 45, connection_timeout: 75).get(head: @headers)
|
35
34
|
http.stream { |chunk| process_chunk(chunk) }
|
36
|
-
http.callback {
|
37
|
-
handle_connection_close(http)
|
35
|
+
http.callback {
|
36
|
+
handle_connection_close(http)
|
38
37
|
EM.stop
|
39
38
|
}
|
40
|
-
http.errback {
|
39
|
+
http.errback {
|
41
40
|
handle_error(http)
|
42
41
|
EM.stop
|
43
42
|
}
|
44
43
|
end
|
45
44
|
end
|
46
|
-
|
47
45
|
end
|
48
46
|
end
|
49
|
-
end
|
47
|
+
end
|
@@ -4,40 +4,39 @@ require 'em-http-request'
|
|
4
4
|
module Gnip
|
5
5
|
module GnipStream
|
6
6
|
class Stream
|
7
|
-
|
8
7
|
EventMachine.threadpool_size = 5
|
9
8
|
|
10
9
|
attr_accessor :url, :backfill_client, :version
|
11
10
|
|
12
11
|
def initialize(client)
|
13
12
|
self.version = client.power_track_version
|
14
|
-
case
|
13
|
+
case version
|
15
14
|
when '1.0'
|
16
15
|
@url = "https://stream.gnip.com:443/accounts/#{client.account}/publishers/#{client.publisher}/streams/track/#{client.label}.json"
|
17
16
|
when '2.0'
|
18
17
|
@url = "https://gnip-stream.twitter.com/stream/powertrack/accounts/#{client.account}/publishers/#{client.publisher}/#{client.label}.json"
|
19
18
|
else
|
20
|
-
raise Exception.new("version #{
|
19
|
+
raise Exception.new("version #{version} is not supported from this gem.")
|
21
20
|
end
|
22
21
|
@backfill_client = client.backfill_client
|
23
22
|
@processor = JsonDataBuffer.new("\r\n", Regexp.new(/^\{.*\}\r\n/))
|
24
|
-
@headers = {'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed'}
|
23
|
+
@headers = { 'authorization' => [client.username, client.password], 'accept-encoding' => 'gzip, compressed' }
|
25
24
|
@error_handler = ErrorReconnect.new(self, :consume)
|
26
25
|
@connection_close_handler = ErrorReconnect.new(self, :consume)
|
27
26
|
configure_handlers
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def configure_handlers
|
31
|
-
|
32
|
-
|
30
|
+
on_error { |error| @error_handler.attempt_to_reconnect("Gnip Connection Error. Reason was: #{error.inspect}") }
|
31
|
+
on_connection_close { @connection_close_handler.attempt_to_reconnect('Gnip Connection Closed') }
|
33
32
|
end
|
34
33
|
|
35
34
|
def consume(&block)
|
36
35
|
@client_callback = block if block
|
37
|
-
|
38
|
-
|
36
|
+
on_message(&@client_callback)
|
37
|
+
connect
|
39
38
|
end
|
40
|
-
|
39
|
+
|
41
40
|
def on_message(&block)
|
42
41
|
@on_message = block
|
43
42
|
end
|
@@ -53,14 +52,14 @@ module Gnip
|
|
53
52
|
def connect
|
54
53
|
EM.run do
|
55
54
|
options = {}
|
56
|
-
options = { query: {
|
57
|
-
http = EM::HttpRequest.new(
|
55
|
+
options = { query: { 'client' => backfill_client } } if backfill_client.present?
|
56
|
+
http = EM::HttpRequest.new(url, inactivity_timeout: 45, connection_timeout: 75).get({ head: @headers }.merge!(options))
|
58
57
|
http.stream { |chunk| process_chunk(chunk) }
|
59
|
-
http.callback {
|
60
|
-
handle_connection_close(http)
|
58
|
+
http.callback {
|
59
|
+
handle_connection_close(http)
|
61
60
|
EM.stop
|
62
61
|
}
|
63
|
-
http.errback {
|
62
|
+
http.errback {
|
64
63
|
handle_error(http)
|
65
64
|
EM.stop
|
66
65
|
}
|
@@ -81,7 +80,6 @@ module Gnip
|
|
81
80
|
def handle_connection_close(http_connection)
|
82
81
|
@on_connection_close.call(http_connection)
|
83
82
|
end
|
84
|
-
|
85
83
|
end
|
86
84
|
end
|
87
|
-
end
|
85
|
+
end
|
data/lib/gnip/version.rb
CHANGED
metadata
CHANGED
@@ -1,113 +1,85 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gnip-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Duccio Giovannelli
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-11-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - "
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
20
|
-
type: :
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - "
|
24
|
+
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '4.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name:
|
28
|
+
name: em-http-request
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
34
|
-
type: :
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
40
|
+
version: '1'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: httparty
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '
|
48
|
-
type: :
|
47
|
+
version: '0.16'
|
48
|
+
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '
|
54
|
+
version: '0.16'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
56
|
+
name: rake
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
61
|
+
version: '13.0'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: activesupport
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '4.2'
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '4.2'
|
83
|
-
- !ruby/object:Gem::Dependency
|
84
|
-
name: em-http-request
|
85
|
-
requirement: !ruby/object:Gem::Requirement
|
86
|
-
requirements:
|
87
|
-
- - "~>"
|
88
|
-
- !ruby/object:Gem::Version
|
89
|
-
version: '1'
|
90
|
-
type: :runtime
|
91
|
-
prerelease: false
|
92
|
-
version_requirements: !ruby/object:Gem::Requirement
|
93
|
-
requirements:
|
94
|
-
- - "~>"
|
95
|
-
- !ruby/object:Gem::Version
|
96
|
-
version: '1'
|
68
|
+
version: '13.0'
|
97
69
|
- !ruby/object:Gem::Dependency
|
98
|
-
name:
|
70
|
+
name: rspec
|
99
71
|
requirement: !ruby/object:Gem::Requirement
|
100
72
|
requirements:
|
101
73
|
- - "~>"
|
102
74
|
- !ruby/object:Gem::Version
|
103
|
-
version:
|
104
|
-
type: :
|
75
|
+
version: '3'
|
76
|
+
type: :development
|
105
77
|
prerelease: false
|
106
78
|
version_requirements: !ruby/object:Gem::Requirement
|
107
79
|
requirements:
|
108
80
|
- - "~>"
|
109
81
|
- !ruby/object:Gem::Version
|
110
|
-
version:
|
82
|
+
version: '3'
|
111
83
|
description:
|
112
84
|
email:
|
113
85
|
- giovannelli@extendi.it
|
@@ -156,7 +128,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
156
128
|
- !ruby/object:Gem::Version
|
157
129
|
version: '0'
|
158
130
|
requirements: []
|
159
|
-
rubygems_version: 3.
|
131
|
+
rubygems_version: 3.1.4
|
160
132
|
signing_key:
|
161
133
|
specification_version: 4
|
162
134
|
summary: A Ruby library for accessing the Gnip API. See https://gnip.com/ for full
|