gnip-client 0.2.10 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +5 -1
- data/README.md +12 -3
- data/lib/gnip.rb +1 -0
- data/lib/gnip/gnip-full-archive/full_archive.rb +29 -28
- data/lib/gnip/gnip-full-archive/thirty_day.rb +15 -0
- data/lib/gnip/power_track_client.rb +2 -1
- data/lib/gnip/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7eb03023da58579a9c34244752346c71b14495e6f43641d3d1b5d5267b8c5ea7
|
4
|
+
data.tar.gz: 293cd273287d5462359c29bf98b7c60b2c738185fdfc0331d8e71a1f717d10f2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0dc8e474c8a08909b5e877194ec0227a73373750d075238c9901324eb8bef7ee3f8cd0d4c9dde174ff12ec97dc681ff129d62e7df26031ce657781bf2c500976
|
7
|
+
data.tar.gz: 46f1fa9899382657eaf757a3b240c1f8d9f47bf8e7c873ede169462c2d2318778fafe26b34da0bf63efa9168646b1ff55831aa5c95a2dc7cf1d52fbbc5eed5d8
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -48,12 +48,21 @@ client.replay_rules.list
|
|
48
48
|
client.replay_rules.delete_all!
|
49
49
|
|
50
50
|
```
|
51
|
+
|
51
52
|
**Full Archive search**
|
52
53
|
|
53
54
|
```ruby
|
54
|
-
client.full_archive.search(query: "hello", date_from:
|
55
|
-
client.full_archive.total_by_time_period(query: "hello", date_from:
|
56
|
-
client.full_archive.total(query: "hello", date_from:
|
55
|
+
client.full_archive.search(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
56
|
+
client.full_archive.total_by_time_period(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
57
|
+
client.full_archive.total(query: "hello", date_from: 2.months.ago, date_to: 20.hours.ago)
|
58
|
+
```
|
59
|
+
|
60
|
+
**30day search**
|
61
|
+
|
62
|
+
```ruby
|
63
|
+
client.thirty_day.search(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
64
|
+
client.thirty_day.total_by_time_period(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
65
|
+
client.thirty_day.total(query: "hello", date_from: 30.days.ago, date_to: 1.hour.ago)
|
57
66
|
```
|
58
67
|
|
59
68
|
**Stream**
|
data/lib/gnip.rb
CHANGED
@@ -4,6 +4,7 @@ require 'gnip/version'
|
|
4
4
|
require 'gnip/power_track_client'
|
5
5
|
require 'gnip/gnip-rules/rules'
|
6
6
|
require 'gnip/gnip-full-archive/full_archive'
|
7
|
+
require 'gnip/gnip-full-archive/thirty_day'
|
7
8
|
require 'gnip/gnip-stream/error_reconnect'
|
8
9
|
require 'gnip/gnip-stream/json_data_bufffer'
|
9
10
|
require 'gnip/gnip-stream/stream'
|
@@ -3,7 +3,8 @@
|
|
3
3
|
module Gnip
|
4
4
|
module GnipFullArchive
|
5
5
|
class FullArchive
|
6
|
-
class InvalidRequestException < StandardError;
|
6
|
+
class InvalidRequestException < StandardError;
|
7
|
+
end
|
7
8
|
|
8
9
|
include HTTParty
|
9
10
|
|
@@ -14,7 +15,7 @@ module Gnip
|
|
14
15
|
def initialize(client)
|
15
16
|
@search_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}.json"
|
16
17
|
@counts_url = "https://data-api.twitter.com/search/fullarchive/accounts/#{client.account}/#{client.label}/counts.json"
|
17
|
-
@auth = {
|
18
|
+
@auth = {username: client.username, password: client.password}
|
18
19
|
end
|
19
20
|
|
20
21
|
# Search using the full-archive search endpoint return an hash containing up to 500 results and the cursor to the next page
|
@@ -26,9 +27,9 @@ module Gnip
|
|
26
27
|
def search(options = {})
|
27
28
|
search_options = {}
|
28
29
|
search_options[:query] = options[:query] || ''
|
29
|
-
search_options[:maxResults]
|
30
|
-
search_options[:fromDate]
|
31
|
-
search_options[:toDate]
|
30
|
+
search_options[:maxResults] = options[:per_page] || 500
|
31
|
+
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
32
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
32
33
|
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
33
34
|
url = [search_url, search_options.to_query].join('?')
|
34
35
|
begin
|
@@ -39,23 +40,23 @@ module Gnip
|
|
39
40
|
raise response.message unless parsed_response.present?
|
40
41
|
|
41
42
|
if parsed_response[:error].present?
|
42
|
-
response = {
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
43
|
+
response = {results: [],
|
44
|
+
next: nil,
|
45
|
+
url: url,
|
46
|
+
error: parsed_response[:error][:message],
|
47
|
+
code: response.code.to_i}
|
47
48
|
else
|
48
|
-
response = {
|
49
|
-
|
50
|
-
|
51
|
-
|
49
|
+
response = {results: parsed_response[:results],
|
50
|
+
url: url,
|
51
|
+
next: parsed_response[:next],
|
52
|
+
code: response.code.to_i}
|
52
53
|
end
|
53
54
|
rescue StandardError => e
|
54
|
-
response = {
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
55
|
+
response = {results: [],
|
56
|
+
url: url,
|
57
|
+
next: nil,
|
58
|
+
error: e.message,
|
59
|
+
code: 500}
|
59
60
|
end
|
60
61
|
response
|
61
62
|
end
|
@@ -66,11 +67,11 @@ module Gnip
|
|
66
67
|
def total_by_time_period(options = {})
|
67
68
|
response = options[:response] || {}
|
68
69
|
search_options = {}
|
69
|
-
search_options[:query]
|
70
|
-
search_options[:bucket]
|
70
|
+
search_options[:query] = options[:query] || ''
|
71
|
+
search_options[:bucket] = options[:bucket] || 'day'
|
71
72
|
search_options[:fromDate] = Gnip.format_date(options[:date_from]) if options[:date_from]
|
72
|
-
search_options[:toDate]
|
73
|
-
search_options[:next]
|
73
|
+
search_options[:toDate] = Gnip.format_date(options[:date_to]) if options[:date_to]
|
74
|
+
search_options[:next] = options[:next_cursor] if options[:next_cursor]
|
74
75
|
|
75
76
|
url = [counts_url, search_options.to_query].join('?')
|
76
77
|
call_done = 0
|
@@ -84,16 +85,16 @@ module Gnip
|
|
84
85
|
raise gnip_call.response.message unless parsed_response.present?
|
85
86
|
|
86
87
|
if parsed_response[:error].present?
|
87
|
-
response = {
|
88
|
+
response = {results: [], next: nil, error: parsed_response[:error][:message], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
88
89
|
else
|
89
90
|
call_done = 1 # we have received a valid response
|
90
91
|
parsed_response[:results].each_with_index do |item, i|
|
91
92
|
parsed_response[:results][i] = item.merge(timePeriod: DateTime.parse(item[:timePeriod]).to_s)
|
92
93
|
end
|
93
|
-
response = {
|
94
|
+
response = {results: (response[:results] || []) + parsed_response[:results], next: parsed_response[:next], code: gnip_call.response.code.to_i, calls: (response[:calls] || 0) + 1}
|
94
95
|
end
|
95
96
|
rescue StandardError => e
|
96
|
-
response = {
|
97
|
+
response = {results: [], next: nil, error: e.message, code: 500, calls: (response[:calls] || 0) + call_done}
|
97
98
|
end
|
98
99
|
# If the next cursor is not present we fetched all the data
|
99
100
|
# It happens that twitter returns the same cursor, in that case we stop
|
@@ -111,8 +112,8 @@ module Gnip
|
|
111
112
|
def total(options = {})
|
112
113
|
extra = {}
|
113
114
|
response = total_by_time_period(options)
|
114
|
-
extra = {
|
115
|
-
{
|
115
|
+
extra = {error: response[:error]} if response[:error].present?
|
116
|
+
{query: options[:query], total: response[:results].map {|item| item[:count]}.reduce(:+), calls: response[:calls]}.merge!(extra)
|
116
117
|
end
|
117
118
|
end
|
118
119
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gnip
|
4
|
+
module GnipFullArchive
|
5
|
+
class ThirtyDay < FullArchive
|
6
|
+
|
7
|
+
def initialize(client)
|
8
|
+
@search_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}.json"
|
9
|
+
@counts_url = "https://gnip-api.twitter.com/search/30day/accounts/#{client.account}/#{client.label}/counts.json"
|
10
|
+
@auth = { username: client.username, password: client.password }
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -6,7 +6,7 @@ module Gnip
|
|
6
6
|
:username, :password,
|
7
7
|
:backfill_client, :replay_label
|
8
8
|
|
9
|
-
attr_reader :rules, :replay_rules, :full_archive, :stream, :replay, :power_track_version
|
9
|
+
attr_reader :rules, :replay_rules, :full_archive, :thirty_day, :stream, :replay, :power_track_version
|
10
10
|
|
11
11
|
def initialize(options = {})
|
12
12
|
@account = options[:account]
|
@@ -20,6 +20,7 @@ module Gnip
|
|
20
20
|
@rules = Gnip::GnipRules::Rules.new(self)
|
21
21
|
@replay_rules = Gnip::GnipRules::Rules.new(self, true)
|
22
22
|
@full_archive = Gnip::GnipFullArchive::FullArchive.new(self)
|
23
|
+
@thirty_day = Gnip::GnipFullArchive::ThirtyDay.new(self)
|
23
24
|
@stream = Gnip::GnipStream::Stream.new(self)
|
24
25
|
@replay = Gnip::GnipStream::Replay.new(self)
|
25
26
|
end
|
data/lib/gnip/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gnip-client
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Duccio Giovannelli
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -129,6 +129,7 @@ files:
|
|
129
129
|
- gnip.gemspec
|
130
130
|
- lib/gnip.rb
|
131
131
|
- lib/gnip/gnip-full-archive/full_archive.rb
|
132
|
+
- lib/gnip/gnip-full-archive/thirty_day.rb
|
132
133
|
- lib/gnip/gnip-rules/rules.rb
|
133
134
|
- lib/gnip/gnip-stream/error_reconnect.rb
|
134
135
|
- lib/gnip/gnip-stream/json_data_bufffer.rb
|