weeter 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -1
- data/.travis.yml +1 -1
- data/lib/weeter/limitator.rb +26 -5
- data/lib/weeter/plugins/lib/redis.rb +5 -6
- data/lib/weeter/plugins/notification/http.rb +9 -1
- data/lib/weeter/plugins/notification/resque.rb +18 -4
- data/lib/weeter/plugins/notification_plugin.rb +8 -5
- data/lib/weeter/plugins/subscription/redis.rb +6 -2
- data/lib/weeter/twitter/tweet_consumer.rb +32 -11
- data/lib/weeter/twitter/tweet_item.rb +18 -1
- data/lib/weeter/version.rb +1 -1
- data/spec/weeter/limitator_spec.rb +69 -18
- data/spec/weeter/twitter/tweet_consumer_spec.rb +40 -12
- data/spec/weeter/twitter/tweet_item_spec.rb +29 -9
- data/weeter.gemspec +1 -0
- metadata +35 -24
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/lib/weeter/limitator.rb
CHANGED
@@ -2,10 +2,15 @@ require 'active_support/core_ext/numeric/time'
|
|
2
2
|
|
3
3
|
module Weeter
|
4
4
|
class Limitator
|
5
|
+
Result = Struct.new(:status, :limited_keys)
|
6
|
+
|
7
|
+
DO_NOT_LIMIT = :do_not_limit
|
8
|
+
INITIATE_LIMITING = :initiate_limiting
|
9
|
+
CONTINUE_LIMITING = :continue_limiting
|
5
10
|
|
6
11
|
module UNLIMITED
|
7
|
-
def self.
|
8
|
-
|
12
|
+
def self.limit_status(*args)
|
13
|
+
DO_NOT_LIMIT
|
9
14
|
end
|
10
15
|
end
|
11
16
|
|
@@ -27,7 +32,7 @@ module Weeter
|
|
27
32
|
attr_accessor :lookup, :window, :max
|
28
33
|
|
29
34
|
def initialize(options = {})
|
30
|
-
self.window= TimeWindow.new({
|
35
|
+
self.window = TimeWindow.new({
|
31
36
|
start: self.now,
|
32
37
|
duration: options.fetch(:duration)
|
33
38
|
})
|
@@ -37,14 +42,26 @@ module Weeter
|
|
37
42
|
flush
|
38
43
|
end
|
39
44
|
|
40
|
-
def
|
45
|
+
def process(*keys)
|
41
46
|
ensure_correct_window
|
42
47
|
|
43
48
|
keys.each do |key|
|
44
49
|
increment(key)
|
45
50
|
end
|
46
51
|
|
47
|
-
|
52
|
+
result = Result.new
|
53
|
+
limited_keys = keys.select { |key| exceeds_max?(key) }
|
54
|
+
if limited_keys.any?
|
55
|
+
result[:limited_keys] = limited_keys
|
56
|
+
if limited_keys.any? { |key| exceeds_max_by_one?(key) }
|
57
|
+
result[:status] = INITIATE_LIMITING
|
58
|
+
else
|
59
|
+
result[:status] = CONTINUE_LIMITING
|
60
|
+
end
|
61
|
+
else
|
62
|
+
result[:status] = DO_NOT_LIMIT
|
63
|
+
end
|
64
|
+
result
|
48
65
|
end
|
49
66
|
|
50
67
|
protected
|
@@ -61,6 +78,10 @@ module Weeter
|
|
61
78
|
lookup[key] > max
|
62
79
|
end
|
63
80
|
|
81
|
+
def exceeds_max_by_one?(key)
|
82
|
+
lookup[key] == max + 1
|
83
|
+
end
|
84
|
+
|
64
85
|
def ensure_correct_window
|
65
86
|
return unless window.over?(now)
|
66
87
|
|
@@ -5,13 +5,12 @@ module Weeter
|
|
5
5
|
module Net
|
6
6
|
module Redis
|
7
7
|
def create_redis_client
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
8
|
+
redis = EM::Hiredis.connect(@config.redis_uri)
|
9
|
+
redis.callback { Weeter.logger.info "Connected to Redis" }
|
10
|
+
redis.errback { |message| Weeter.logger.err "Failed to connect to Redis: #{message}" }
|
11
|
+
redis
|
13
12
|
end
|
14
13
|
end
|
15
14
|
end
|
16
15
|
end
|
17
|
-
end
|
16
|
+
end
|
@@ -20,7 +20,15 @@ module Weeter
|
|
20
20
|
Weeter.logger.info("Deleting tweet #{id} for user #{user_id}")
|
21
21
|
Weeter::Plugins::Net::OauthHttp.delete(@config, @config.delete_url, {:id => id, :twitter_user_id => user_id})
|
22
22
|
end
|
23
|
+
|
24
|
+
def notify_missed_tweets(tweet_item)
|
25
|
+
Weeter.logger.info("Weeter was limited by Twitter. #{tweet_item.missed_tweets_count} tweets missed.")
|
26
|
+
end
|
27
|
+
|
28
|
+
def notify_rate_limiting_initiated(tweet_item, limited_keys)
|
29
|
+
Weeter.logger.info("Initiated rate limiting with tweet: #{tweet_item.to_json}")
|
30
|
+
end
|
23
31
|
end
|
24
32
|
end
|
25
33
|
end
|
26
|
-
end
|
34
|
+
end
|
@@ -3,7 +3,7 @@ module Weeter
|
|
3
3
|
module Notification
|
4
4
|
class Resque
|
5
5
|
include Weeter::Plugins::Net::Redis
|
6
|
-
|
6
|
+
|
7
7
|
def initialize(client_app_config)
|
8
8
|
@config = client_app_config
|
9
9
|
end
|
@@ -20,12 +20,26 @@ module Weeter
|
|
20
20
|
enqueue(resque_job)
|
21
21
|
end
|
22
22
|
|
23
|
+
def notify_missed_tweets(tweet_item)
|
24
|
+
resque_job = %Q|{"class":"WeeterMissedTweetsJob","args":[#{tweet_item.to_json}]}|
|
25
|
+
Weeter.logger.info("Notifying of missed tweets (#{tweet_item.missed_tweets_count}).")
|
26
|
+
enqueue(resque_job)
|
27
|
+
end
|
28
|
+
|
29
|
+
def notify_rate_limiting_initiated(tweet_item, limited_keys)
|
30
|
+
payload = tweet_item.to_hash.merge(:limited_keys => limited_keys)
|
31
|
+
payload_json = MultiJson.encode(payload)
|
32
|
+
resque_job = %Q|{"class":"WeeterRateLimitingInitiatedJob","args":[#{payload_json}]}|
|
33
|
+
Weeter.logger.info("Initiated rate limiting with tweet: #{payload_json}")
|
34
|
+
enqueue(resque_job)
|
35
|
+
end
|
36
|
+
|
23
37
|
protected
|
24
|
-
|
38
|
+
|
25
39
|
def redis
|
26
40
|
@redis ||= create_redis_client
|
27
41
|
end
|
28
|
-
|
42
|
+
|
29
43
|
def enqueue(job)
|
30
44
|
redis.rpush(queue_key, job)
|
31
45
|
end
|
@@ -36,4 +50,4 @@ module Weeter
|
|
36
50
|
end
|
37
51
|
end
|
38
52
|
end
|
39
|
-
end
|
53
|
+
end
|
@@ -7,13 +7,16 @@ require 'active_support/core_ext/module/delegation'
|
|
7
7
|
module Weeter
|
8
8
|
module Plugins
|
9
9
|
class NotificationPlugin
|
10
|
-
delegate :publish_tweet,
|
11
|
-
|
12
|
-
|
10
|
+
delegate :publish_tweet,
|
11
|
+
:delete_tweet,
|
12
|
+
:notify_rate_limiting_initiated,
|
13
|
+
:notify_missed_tweets,
|
14
|
+
:to => :configured_plugin
|
15
|
+
|
13
16
|
def initialize(client_app_config)
|
14
17
|
@config = client_app_config
|
15
18
|
end
|
16
|
-
|
19
|
+
|
17
20
|
protected
|
18
21
|
def configured_plugin
|
19
22
|
@configured_plugin ||= begin
|
@@ -23,4 +26,4 @@ module Weeter
|
|
23
26
|
end
|
24
27
|
end
|
25
28
|
end
|
26
|
-
end
|
29
|
+
end
|
@@ -11,20 +11,24 @@ module Weeter
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def get_initial_filters(&block)
|
14
|
-
redis.get(@config.subscriptions_key) do |value|
|
14
|
+
deferred_get = redis.get(@config.subscriptions_key) do |value|
|
15
15
|
if value.nil?
|
16
16
|
raise "Expected to find subscription data at redis key #{@config.subscriptions_key}"
|
17
17
|
end
|
18
18
|
yield MultiJson.decode(value)
|
19
19
|
end
|
20
|
+
deferred_get.errback do |message|
|
21
|
+
Weeter.logger.error(message)
|
22
|
+
end
|
20
23
|
end
|
21
24
|
|
22
25
|
def listen_for_filter_update(tweet_consumer)
|
23
26
|
pub_sub_redis.subscribe(@config.subscriptions_changed_channel)
|
24
27
|
pub_sub_redis.on(:message) do |channel, message|
|
25
28
|
Weeter.logger.info [:message, channel, message]
|
26
|
-
Weeter.logger.info("
|
29
|
+
Weeter.logger.info("Retrieving updated filters from redis")
|
27
30
|
get_initial_filters do |filter_params|
|
31
|
+
Weeter.logger.info("Triggering reconnect Twitter stream with new filters")
|
28
32
|
tweet_consumer.reconnect(filter_params)
|
29
33
|
end
|
30
34
|
end
|
@@ -4,8 +4,13 @@ require 'multi_json'
|
|
4
4
|
module Weeter
|
5
5
|
module Twitter
|
6
6
|
class TweetConsumer
|
7
|
+
extend ::Forwardable
|
7
8
|
|
8
|
-
attr_reader :limiter
|
9
|
+
attr_reader :limiter, :notifier
|
10
|
+
def_delegators :@notifier, :notify_missed_tweets,
|
11
|
+
:notify_rate_limiting_initiated,
|
12
|
+
:delete_tweet,
|
13
|
+
:publish_tweet
|
9
14
|
|
10
15
|
def initialize(twitter_config, notifier, limiter, subscriptions_limit = nil)
|
11
16
|
@config = twitter_config
|
@@ -18,31 +23,31 @@ module Weeter
|
|
18
23
|
filter_params = limit_filter_params(filter_params) if @subscriptions_limit
|
19
24
|
filter_params = clean_filter_params(filter_params)
|
20
25
|
|
26
|
+
|
21
27
|
connect_options = {
|
22
28
|
ssl: true,
|
23
29
|
params: filter_params,
|
24
30
|
method: 'POST'
|
25
31
|
}.merge(@config.auth_options)
|
26
32
|
|
33
|
+
Weeter.logger.info("Connecting to Twitter stream...")
|
27
34
|
@stream = ::Twitter::JSONStream.connect(connect_options)
|
28
35
|
|
29
36
|
@stream.each_item do |item|
|
30
37
|
begin
|
31
38
|
tweet_item = TweetItem.new(MultiJson.decode(item))
|
32
39
|
|
33
|
-
if tweet_item.
|
34
|
-
|
40
|
+
if tweet_item.limit_notice?
|
41
|
+
notify_missed_tweets(tweet_item)
|
42
|
+
elsif tweet_item.deletion?
|
43
|
+
delete_tweet(tweet_item)
|
35
44
|
elsif tweet_item.publishable?
|
36
|
-
|
37
|
-
rate_limit_tweet(tweet_item)
|
38
|
-
else
|
39
|
-
@notifier.publish_tweet(tweet_item)
|
40
|
-
end
|
45
|
+
publish_or_rate_limit(tweet_item)
|
41
46
|
else
|
42
47
|
ignore_tweet(tweet_item)
|
43
48
|
end
|
44
49
|
rescue => ex
|
45
|
-
Weeter.logger.error("Twitter stream tweet exception: #{ex.class.name}: #{ex.message}")
|
50
|
+
Weeter.logger.error("Twitter stream tweet exception: #{ex.class.name}: #{ex.message} #{tweet_item.to_json}")
|
46
51
|
end
|
47
52
|
end
|
48
53
|
|
@@ -57,6 +62,7 @@ module Weeter
|
|
57
62
|
|
58
63
|
def reconnect(filter_params)
|
59
64
|
@stream.stop
|
65
|
+
@stream.unbind
|
60
66
|
connect(filter_params)
|
61
67
|
end
|
62
68
|
|
@@ -97,15 +103,17 @@ module Weeter
|
|
97
103
|
return {} if p.nil?
|
98
104
|
cleaned_params = {}
|
99
105
|
cleaned_params['follow'] = p['follow'] if (p['follow'] || []).any?
|
100
|
-
cleaned_params['follow'] = cleaned_params['follow'].map(&:to_i)
|
106
|
+
cleaned_params['follow'] = cleaned_params['follow'].map(&:to_i) if cleaned_params['follow']
|
101
107
|
cleaned_params['track'] = p['track'] if (p['track'] || []).any?
|
102
108
|
cleaned_params
|
103
109
|
end
|
104
110
|
|
105
111
|
def ignore_tweet(tweet_item)
|
112
|
+
return if tweet_item.disconnect_notice?
|
106
113
|
id = tweet_item['id_str']
|
107
114
|
text = tweet_item['text']
|
108
|
-
|
115
|
+
user = tweet_item['user']
|
116
|
+
user_id = user['id_str'] if user
|
109
117
|
Weeter.logger.info("Ignoring tweet #{id} from user #{user_id}: #{text}")
|
110
118
|
end
|
111
119
|
|
@@ -116,6 +124,19 @@ module Weeter
|
|
116
124
|
|
117
125
|
Weeter.logger.info("Rate Limiting tweet #{id} from user #{user_id}: #{text}")
|
118
126
|
end
|
127
|
+
|
128
|
+
def publish_or_rate_limit(tweet_item)
|
129
|
+
limit_result = limiter.process(*tweet_item.limiting_facets)
|
130
|
+
case limit_result.status
|
131
|
+
when Weeter::Limitator::INITIATE_LIMITING
|
132
|
+
notify_rate_limiting_initiated(tweet_item, limit_result.limited_keys)
|
133
|
+
rate_limit_tweet(tweet_item)
|
134
|
+
when Weeter::Limitator::CONTINUE_LIMITING
|
135
|
+
rate_limit_tweet(tweet_item)
|
136
|
+
when Weeter::Limitator::DO_NOT_LIMIT
|
137
|
+
publish_tweet(tweet_item)
|
138
|
+
end
|
139
|
+
end
|
119
140
|
end
|
120
141
|
end
|
121
142
|
end
|
@@ -20,7 +20,20 @@ module Weeter
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def publishable?
|
23
|
-
!retweeted? && !reply?
|
23
|
+
!retweeted? && !reply? && !disconnect_notice? && !limit_notice?
|
24
|
+
end
|
25
|
+
|
26
|
+
def disconnect_notice?
|
27
|
+
!@tweet_hash['disconnect'].nil?
|
28
|
+
end
|
29
|
+
|
30
|
+
def limit_notice?
|
31
|
+
!@tweet_hash['limit'].nil?
|
32
|
+
end
|
33
|
+
|
34
|
+
def missed_tweets_count
|
35
|
+
return nil unless limit_notice?
|
36
|
+
@tweet_hash['limit']['track']
|
24
37
|
end
|
25
38
|
|
26
39
|
def [](val)
|
@@ -31,6 +44,10 @@ module Weeter
|
|
31
44
|
MultiJson.encode(@tweet_hash)
|
32
45
|
end
|
33
46
|
|
47
|
+
def to_hash
|
48
|
+
@tweet_hash
|
49
|
+
end
|
50
|
+
|
34
51
|
def limiting_facets
|
35
52
|
self['entities']['hashtags'].map do |tag|
|
36
53
|
tag['text'].downcase.chomp
|
data/lib/weeter/version.rb
CHANGED
@@ -17,46 +17,57 @@ describe Weeter::Limitator do
|
|
17
17
|
it { limitator.should be }
|
18
18
|
end
|
19
19
|
|
20
|
-
describe '#
|
20
|
+
describe '#limit_status' do
|
21
21
|
|
22
22
|
subject do
|
23
|
-
limitator.
|
23
|
+
limitator.process(*keys)
|
24
24
|
end
|
25
25
|
|
26
26
|
context 'max: 0' do
|
27
27
|
let(:max) { 0 }
|
28
|
-
|
28
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
29
|
+
its(:limited_keys) { should == keys }
|
29
30
|
|
30
31
|
context 'no keys' do
|
31
32
|
let(:keys) { [] }
|
32
|
-
|
33
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
34
|
+
its(:limited_keys) { should == nil }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'two keys' do
|
38
|
+
let(:keys) { ['key', 'key2'] }
|
39
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
40
|
+
its(:limited_keys) { should == keys }
|
33
41
|
end
|
34
42
|
end
|
35
43
|
|
36
44
|
context 'max: 1' do
|
37
45
|
let(:max) { 1 }
|
38
46
|
|
39
|
-
|
47
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
48
|
+
its(:limited_keys) { should == nil }
|
40
49
|
|
41
50
|
context 'two keys within max' do
|
42
51
|
let(:keys) { ['key', 'key2'] }
|
43
52
|
|
44
|
-
|
53
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
45
54
|
end
|
46
55
|
|
47
56
|
context 'no keys' do
|
48
57
|
let(:keys) { [] }
|
49
|
-
|
58
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
59
|
+
its(:limited_keys) { should == nil }
|
50
60
|
end
|
51
61
|
|
52
|
-
context 'one key outside max' do
|
62
|
+
context 'one key just outside max' do
|
53
63
|
before do
|
54
64
|
max.times do
|
55
|
-
limitator.
|
65
|
+
limitator.process(*keys)
|
56
66
|
end
|
57
67
|
end
|
58
68
|
|
59
|
-
|
69
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
70
|
+
its(:limited_keys) { should == keys }
|
60
71
|
|
61
72
|
context 'outside duration' do
|
62
73
|
let(:some_time_after_duration) do
|
@@ -67,31 +78,71 @@ describe Weeter::Limitator do
|
|
67
78
|
limitator.stub(:now).and_return(some_time_after_duration)
|
68
79
|
end
|
69
80
|
|
70
|
-
|
81
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
82
|
+
its(:limited_keys) { should == nil }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
context 'two keys just past max' do
|
87
|
+
let(:keys) { ['key', 'key2'] }
|
88
|
+
|
89
|
+
before do
|
90
|
+
limitator.process(*keys)
|
71
91
|
end
|
92
|
+
|
93
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
94
|
+
its(:limited_keys) { should == keys }
|
72
95
|
end
|
73
96
|
|
74
|
-
context 'two keys
|
97
|
+
context 'two keys past max' do
|
75
98
|
let(:keys) { ['key', 'key2'] }
|
76
99
|
|
77
100
|
before do
|
78
|
-
limitator.
|
101
|
+
limitator.process(*keys)
|
102
|
+
limitator.process(*keys)
|
79
103
|
end
|
80
104
|
|
81
|
-
|
105
|
+
its(:status) { should == Weeter::Limitator::CONTINUE_LIMITING }
|
106
|
+
its(:limited_keys) { should == keys }
|
82
107
|
end
|
83
108
|
|
84
|
-
context 'one key
|
109
|
+
context 'one key just past max: 1, one key within max: 1' do
|
85
110
|
let(:max) { 1 }
|
86
111
|
let(:keys) { ['key', 'key2'] }
|
87
112
|
|
88
113
|
before do
|
89
|
-
limitator.
|
114
|
+
limitator.process(keys.first)
|
90
115
|
end
|
91
116
|
|
92
|
-
|
117
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
118
|
+
its(:limited_keys) { should == [keys.first] }
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'one key past max: 1, one key within max: 1' do
|
122
|
+
let(:max) { 1 }
|
123
|
+
let(:keys) { ['key', 'key2'] }
|
124
|
+
|
125
|
+
before do
|
126
|
+
limitator.process(keys.first)
|
127
|
+
limitator.process(keys.first)
|
128
|
+
end
|
129
|
+
|
130
|
+
its(:status) { should == Weeter::Limitator::CONTINUE_LIMITING }
|
131
|
+
its(:limited_keys) { should == [keys.first] }
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'one key past max: 1, one key just past max: 1' do
|
135
|
+
let(:max) { 1 }
|
136
|
+
let(:keys) { ['key', 'key2'] }
|
137
|
+
|
138
|
+
before do
|
139
|
+
limitator.process(*[keys.first])
|
140
|
+
limitator.process(*[keys.first, keys.last])
|
141
|
+
end
|
142
|
+
|
143
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
144
|
+
its(:limited_keys) { should == keys }
|
93
145
|
end
|
94
146
|
end
|
95
147
|
end
|
96
148
|
end
|
97
|
-
|
@@ -10,7 +10,7 @@ describe Weeter::Twitter::TweetConsumer do
|
|
10
10
|
|
11
11
|
describe "auth" do
|
12
12
|
it 'should use connect to JSON stream with auth options for the configuration' do
|
13
|
-
@mock_stream = mock('JSONStream', :each_item => nil, :on_error => nil, :on_max_reconnects => nil)
|
13
|
+
@mock_stream = mock('JSONStream', :each_item => nil, :on_error => nil, :on_max_reconnects => nil, :on_close => nil)
|
14
14
|
Twitter::JSONStream.stub!(:connect).and_return(@mock_stream)
|
15
15
|
|
16
16
|
Weeter::Configuration::TwitterConfig.instance.stub!(:auth_options).and_return(:foo => :bar)
|
@@ -91,13 +91,22 @@ describe Weeter::Twitter::TweetConsumer do
|
|
91
91
|
|
92
92
|
describe "connecting to twitter" do
|
93
93
|
|
94
|
+
let(:tweet_values) {
|
95
|
+
[@tweet_hash]
|
96
|
+
}
|
97
|
+
let(:mock_stream) {
|
98
|
+
mock_stream = mock('JSONStream', :on_error => nil, :on_max_reconnects => nil, :on_close => nil)
|
99
|
+
each_item_stub = mock_stream.stub!(:each_item)
|
100
|
+
tweet_values.each do |t|
|
101
|
+
each_item_stub.and_yield(MultiJson.encode(t))
|
102
|
+
end
|
103
|
+
mock_stream
|
104
|
+
}
|
94
105
|
before(:each) do
|
95
106
|
@filter_params = {'follow' => ['1','2','3']}
|
96
107
|
Weeter::Configuration::TwitterConfig.instance.stub!(:auth_options).and_return(:foo => :bar)
|
97
|
-
@
|
98
|
-
|
99
|
-
@mock_stream.stub!(:each_item).and_yield(MultiJson.encode(@tweet_values))
|
100
|
-
Twitter::JSONStream.stub!(:connect).and_return(@mock_stream)
|
108
|
+
@tweet_hash = {'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => "1"}}
|
109
|
+
Twitter::JSONStream.stub!(:connect).and_return(mock_stream)
|
101
110
|
@client_proxy = mock('NotificationPlugin', :publish_tweet => nil)
|
102
111
|
@consumer = Weeter::Twitter::TweetConsumer.new(Weeter::Configuration::TwitterConfig.instance, @client_proxy, limiter)
|
103
112
|
end
|
@@ -107,7 +116,7 @@ describe Weeter::Twitter::TweetConsumer do
|
|
107
116
|
end
|
108
117
|
|
109
118
|
it "should instantiate a TweetItem" do
|
110
|
-
tweet_item = Weeter::TweetItem.new(@
|
119
|
+
tweet_item = Weeter::TweetItem.new(@tweet_hash)
|
111
120
|
Weeter::TweetItem.should_receive(:new).with({'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => "1"}}).and_return(tweet_item)
|
112
121
|
end
|
113
122
|
|
@@ -117,22 +126,41 @@ describe Weeter::Twitter::TweetConsumer do
|
|
117
126
|
end
|
118
127
|
|
119
128
|
it "should publish new tweet if publishable" do
|
120
|
-
mock_tweet = mock('tweet', :deletion? => false, :publishable? => true, :limiting_facets => [])
|
121
|
-
|
129
|
+
mock_tweet = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => [])
|
130
|
+
Weeter::TweetItem.stub!(:new).and_return(mock_tweet)
|
122
131
|
@client_proxy.should_receive(:publish_tweet).with(mock_tweet)
|
123
132
|
end
|
124
133
|
|
125
134
|
it "should not publish unpublishable tweets" do
|
126
|
-
mock_tweet = mock('tweet', :deletion? => false, :publishable? => false, :[] => '', :limiting_facets => [])
|
127
|
-
|
135
|
+
mock_tweet = mock('tweet', :deletion? => false, :publishable? => false, :limit_notice? => false, :[] => '', :limiting_facets => [])
|
136
|
+
Weeter::TweetItem.stub!(:new).and_return mock_tweet
|
128
137
|
@client_proxy.should_not_receive(:publish_tweet).with(mock_tweet)
|
129
138
|
end
|
130
139
|
|
131
140
|
it "should delete deletion tweets" do
|
132
|
-
mock_tweet = mock('tweet', :deletion? => true, :publishable? => false, :limiting_facets => [])
|
133
|
-
|
141
|
+
mock_tweet = mock('tweet', :deletion? => true, :publishable? => false, :limit_notice? => false, :limiting_facets => [])
|
142
|
+
Weeter::TweetItem.stub!(:new).and_return mock_tweet
|
134
143
|
@client_proxy.should_receive(:delete_tweet).with(mock_tweet)
|
135
144
|
end
|
145
|
+
|
146
|
+
it "should notify when stream is limited by Twitter" do
|
147
|
+
tweet_item = Weeter::TweetItem.new({'limit' => { 'track' => 65 } })
|
148
|
+
Weeter::TweetItem.stub!(:new).and_return(tweet_item)
|
149
|
+
@client_proxy.should_receive(:notify_missed_tweets).with(tweet_item)
|
150
|
+
end
|
151
|
+
|
152
|
+
context "when weeter is initiating rate-limiting on a facet" do
|
153
|
+
let(:tweet_values) {
|
154
|
+
[@tweet_hash, @tweet_hash]
|
155
|
+
}
|
156
|
+
it "should notify that rate limiting is being initiated" do
|
157
|
+
tweet_item1 = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => ['key'], :[] => '1')
|
158
|
+
tweet_item2 = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => ['key'], :[] => '2')
|
159
|
+
Weeter::TweetItem.stub!(:new).and_return(tweet_item1, tweet_item2)
|
160
|
+
|
161
|
+
@client_proxy.should_receive(:notify_rate_limiting_initiated).with(tweet_item2, ['key'])
|
162
|
+
end
|
163
|
+
end
|
136
164
|
end
|
137
165
|
|
138
166
|
end
|
@@ -1,6 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Weeter::TweetItem do
|
4
|
+
let(:tweet_json) {
|
5
|
+
{'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => '1'}}
|
6
|
+
}
|
4
7
|
|
5
8
|
describe "deletion?" do
|
6
9
|
it "should be true if it is a deletion request" do
|
@@ -16,37 +19,54 @@ describe Weeter::TweetItem do
|
|
16
19
|
|
17
20
|
describe "publishable" do
|
18
21
|
|
19
|
-
before do
|
20
|
-
@tweet_json = {'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => '1'}}
|
21
|
-
end
|
22
22
|
|
23
23
|
it "should be publishable if not a reply or a retweet" do
|
24
|
-
item = Weeter::TweetItem.new(
|
24
|
+
item = Weeter::TweetItem.new(tweet_json)
|
25
25
|
item.should be_publishable
|
26
26
|
end
|
27
27
|
|
28
28
|
it "should not be publishable if implicitly retweeted" do
|
29
|
-
item = Weeter::TweetItem.new(
|
29
|
+
item = Weeter::TweetItem.new(tweet_json.merge({'text' => 'RT @joe Hey'}))
|
30
30
|
item.should_not be_publishable
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should not be publishable if explicitly retweeted" do
|
34
|
-
item = Weeter::TweetItem.new(
|
34
|
+
item = Weeter::TweetItem.new(tweet_json.merge('retweeted_status' => {'id_str' => '111', 'text' => 'Hey', 'user' => {'id_str' => "1"}}))
|
35
35
|
item.should_not be_publishable
|
36
36
|
end
|
37
37
|
|
38
38
|
it "should not be publishable if implicit reply" do
|
39
|
-
item = Weeter::TweetItem.new(
|
39
|
+
item = Weeter::TweetItem.new(tweet_json.merge('text' => '@joe Hey'))
|
40
40
|
item.should_not be_publishable
|
41
41
|
end
|
42
42
|
|
43
43
|
it "should not be publishable if explicit reply" do
|
44
|
-
item = Weeter::TweetItem.new(
|
44
|
+
item = Weeter::TweetItem.new(tweet_json.merge('text' => '@joe Hey', 'in_reply_to_user_id_str' => '1'))
|
45
|
+
item.should_not be_publishable
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should not be publishable if disconnect message" do
|
49
|
+
item = Weeter::TweetItem.new({"disconnect" => {"code" => 7,"stream_name" => "YappBox-statuses668638","reason" => "admin logout"}})
|
45
50
|
item.should_not be_publishable
|
46
51
|
end
|
47
52
|
|
48
53
|
end
|
49
54
|
|
55
|
+
describe "limit_notice?" do
|
56
|
+
it "should be true if it's a limit notice" do
|
57
|
+
item = Weeter::TweetItem.new({ 'limit' => { 'track' => 65 }})
|
58
|
+
item.should be_limit_notice
|
59
|
+
item.missed_tweets_count.should == 65
|
60
|
+
end
|
61
|
+
it "should not be true if it's a limit notice" do
|
62
|
+
item = Weeter::TweetItem.new(tweet_json)
|
63
|
+
item.should_not be_limit_notice
|
64
|
+
lambda {
|
65
|
+
item.missed_tweets_count
|
66
|
+
}.should_not raise_error
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
50
70
|
describe "json attributes" do
|
51
71
|
|
52
72
|
it "should delegate hash calls to its json" do
|
@@ -63,4 +83,4 @@ describe Weeter::TweetItem do
|
|
63
83
|
|
64
84
|
|
65
85
|
|
66
|
-
end
|
86
|
+
end
|
data/weeter.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: weeter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -11,11 +11,11 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2012-
|
14
|
+
date: 2012-12-04 00:00:00.000000000Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: eventmachine
|
18
|
-
requirement: &
|
18
|
+
requirement: &2161838440 !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
21
|
- - ! '>='
|
@@ -23,10 +23,10 @@ dependencies:
|
|
23
23
|
version: '0'
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
|
-
version_requirements: *
|
26
|
+
version_requirements: *2161838440
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: eventmachine_httpserver
|
29
|
-
requirement: &
|
29
|
+
requirement: &2161836940 !ruby/object:Gem::Requirement
|
30
30
|
none: false
|
31
31
|
requirements:
|
32
32
|
- - ! '>='
|
@@ -34,10 +34,10 @@ dependencies:
|
|
34
34
|
version: 0.2.1
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *2161836940
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: em-hiredis
|
40
|
-
requirement: &
|
40
|
+
requirement: &2161835100 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
@@ -45,10 +45,10 @@ dependencies:
|
|
45
45
|
version: 0.1.0
|
46
46
|
type: :runtime
|
47
47
|
prerelease: false
|
48
|
-
version_requirements: *
|
48
|
+
version_requirements: *2161835100
|
49
49
|
- !ruby/object:Gem::Dependency
|
50
50
|
name: multi_json
|
51
|
-
requirement: &
|
51
|
+
requirement: &2161833640 !ruby/object:Gem::Requirement
|
52
52
|
none: false
|
53
53
|
requirements:
|
54
54
|
- - ! '>='
|
@@ -56,10 +56,10 @@ dependencies:
|
|
56
56
|
version: 1.0.2
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
|
-
version_requirements: *
|
59
|
+
version_requirements: *2161833640
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: hashie
|
62
|
-
requirement: &
|
62
|
+
requirement: &2161832500 !ruby/object:Gem::Requirement
|
63
63
|
none: false
|
64
64
|
requirements:
|
65
65
|
- - ! '>='
|
@@ -67,10 +67,10 @@ dependencies:
|
|
67
67
|
version: 1.1.0
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
|
-
version_requirements: *
|
70
|
+
version_requirements: *2161832500
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
72
|
name: em-http-request
|
73
|
-
requirement: &
|
73
|
+
requirement: &2161832040 !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
76
|
- - ! '>='
|
@@ -78,10 +78,10 @@ dependencies:
|
|
78
78
|
version: 1.0.0
|
79
79
|
type: :runtime
|
80
80
|
prerelease: false
|
81
|
-
version_requirements: *
|
81
|
+
version_requirements: *2161832040
|
82
82
|
- !ruby/object:Gem::Dependency
|
83
83
|
name: i18n
|
84
|
-
requirement: &
|
84
|
+
requirement: &2161831220 !ruby/object:Gem::Requirement
|
85
85
|
none: false
|
86
86
|
requirements:
|
87
87
|
- - ~>
|
@@ -89,10 +89,10 @@ dependencies:
|
|
89
89
|
version: 0.6.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
|
-
version_requirements: *
|
92
|
+
version_requirements: *2161831220
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
94
|
name: activesupport
|
95
|
-
requirement: &
|
95
|
+
requirement: &2161830060 !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
98
98
|
- - ! '>='
|
@@ -100,10 +100,10 @@ dependencies:
|
|
100
100
|
version: 3.1.1
|
101
101
|
type: :runtime
|
102
102
|
prerelease: false
|
103
|
-
version_requirements: *
|
103
|
+
version_requirements: *2161830060
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
name: simple_oauth
|
106
|
-
requirement: &
|
106
|
+
requirement: &2161829240 !ruby/object:Gem::Requirement
|
107
107
|
none: false
|
108
108
|
requirements:
|
109
109
|
- - ~>
|
@@ -111,10 +111,10 @@ dependencies:
|
|
111
111
|
version: 0.1.5
|
112
112
|
type: :runtime
|
113
113
|
prerelease: false
|
114
|
-
version_requirements: *
|
114
|
+
version_requirements: *2161829240
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: lukemelia-twitter-stream
|
117
|
-
requirement: &
|
117
|
+
requirement: &2161828460 !ruby/object:Gem::Requirement
|
118
118
|
none: false
|
119
119
|
requirements:
|
120
120
|
- - ~>
|
@@ -122,10 +122,10 @@ dependencies:
|
|
122
122
|
version: 0.1.15
|
123
123
|
type: :runtime
|
124
124
|
prerelease: false
|
125
|
-
version_requirements: *
|
125
|
+
version_requirements: *2161828460
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: rspec
|
128
|
-
requirement: &
|
128
|
+
requirement: &2161827700 !ruby/object:Gem::Requirement
|
129
129
|
none: false
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
@@ -133,7 +133,18 @@ dependencies:
|
|
133
133
|
version: 2.6.0
|
134
134
|
type: :development
|
135
135
|
prerelease: false
|
136
|
-
version_requirements: *
|
136
|
+
version_requirements: *2161827700
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: ZenTest
|
139
|
+
requirement: &2161826960 !ruby/object:Gem::Requirement
|
140
|
+
none: false
|
141
|
+
requirements:
|
142
|
+
- - ! '>='
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
type: :development
|
146
|
+
prerelease: false
|
147
|
+
version_requirements: *2161826960
|
137
148
|
description: Weeter subscribes to a set of twitter users or search terms using Twitter's
|
138
149
|
streaming API, and notifies your app with each new tweet.
|
139
150
|
email:
|