weeter 0.11.0 → 0.13.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +3 -1
- data/.travis.yml +1 -1
- data/lib/weeter/limitator.rb +26 -5
- data/lib/weeter/plugins/lib/redis.rb +5 -6
- data/lib/weeter/plugins/notification/http.rb +9 -1
- data/lib/weeter/plugins/notification/resque.rb +18 -4
- data/lib/weeter/plugins/notification_plugin.rb +8 -5
- data/lib/weeter/plugins/subscription/redis.rb +6 -2
- data/lib/weeter/twitter/tweet_consumer.rb +32 -11
- data/lib/weeter/twitter/tweet_item.rb +18 -1
- data/lib/weeter/version.rb +1 -1
- data/spec/weeter/limitator_spec.rb +69 -18
- data/spec/weeter/twitter/tweet_consumer_spec.rb +40 -12
- data/spec/weeter/twitter/tweet_item_spec.rb +29 -9
- data/weeter.gemspec +1 -0
- metadata +35 -24
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
data/lib/weeter/limitator.rb
CHANGED
@@ -2,10 +2,15 @@ require 'active_support/core_ext/numeric/time'
|
|
2
2
|
|
3
3
|
module Weeter
|
4
4
|
class Limitator
|
5
|
+
Result = Struct.new(:status, :limited_keys)
|
6
|
+
|
7
|
+
DO_NOT_LIMIT = :do_not_limit
|
8
|
+
INITIATE_LIMITING = :initiate_limiting
|
9
|
+
CONTINUE_LIMITING = :continue_limiting
|
5
10
|
|
6
11
|
module UNLIMITED
|
7
|
-
def self.
|
8
|
-
|
12
|
+
def self.limit_status(*args)
|
13
|
+
DO_NOT_LIMIT
|
9
14
|
end
|
10
15
|
end
|
11
16
|
|
@@ -27,7 +32,7 @@ module Weeter
|
|
27
32
|
attr_accessor :lookup, :window, :max
|
28
33
|
|
29
34
|
def initialize(options = {})
|
30
|
-
self.window= TimeWindow.new({
|
35
|
+
self.window = TimeWindow.new({
|
31
36
|
start: self.now,
|
32
37
|
duration: options.fetch(:duration)
|
33
38
|
})
|
@@ -37,14 +42,26 @@ module Weeter
|
|
37
42
|
flush
|
38
43
|
end
|
39
44
|
|
40
|
-
def
|
45
|
+
def process(*keys)
|
41
46
|
ensure_correct_window
|
42
47
|
|
43
48
|
keys.each do |key|
|
44
49
|
increment(key)
|
45
50
|
end
|
46
51
|
|
47
|
-
|
52
|
+
result = Result.new
|
53
|
+
limited_keys = keys.select { |key| exceeds_max?(key) }
|
54
|
+
if limited_keys.any?
|
55
|
+
result[:limited_keys] = limited_keys
|
56
|
+
if limited_keys.any? { |key| exceeds_max_by_one?(key) }
|
57
|
+
result[:status] = INITIATE_LIMITING
|
58
|
+
else
|
59
|
+
result[:status] = CONTINUE_LIMITING
|
60
|
+
end
|
61
|
+
else
|
62
|
+
result[:status] = DO_NOT_LIMIT
|
63
|
+
end
|
64
|
+
result
|
48
65
|
end
|
49
66
|
|
50
67
|
protected
|
@@ -61,6 +78,10 @@ module Weeter
|
|
61
78
|
lookup[key] > max
|
62
79
|
end
|
63
80
|
|
81
|
+
def exceeds_max_by_one?(key)
|
82
|
+
lookup[key] == max + 1
|
83
|
+
end
|
84
|
+
|
64
85
|
def ensure_correct_window
|
65
86
|
return unless window.over?(now)
|
66
87
|
|
@@ -5,13 +5,12 @@ module Weeter
|
|
5
5
|
module Net
|
6
6
|
module Redis
|
7
7
|
def create_redis_client
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
8
|
+
redis = EM::Hiredis.connect(@config.redis_uri)
|
9
|
+
redis.callback { Weeter.logger.info "Connected to Redis" }
|
10
|
+
redis.errback { |message| Weeter.logger.err "Failed to connect to Redis: #{message}" }
|
11
|
+
redis
|
13
12
|
end
|
14
13
|
end
|
15
14
|
end
|
16
15
|
end
|
17
|
-
end
|
16
|
+
end
|
@@ -20,7 +20,15 @@ module Weeter
|
|
20
20
|
Weeter.logger.info("Deleting tweet #{id} for user #{user_id}")
|
21
21
|
Weeter::Plugins::Net::OauthHttp.delete(@config, @config.delete_url, {:id => id, :twitter_user_id => user_id})
|
22
22
|
end
|
23
|
+
|
24
|
+
def notify_missed_tweets(tweet_item)
|
25
|
+
Weeter.logger.info("Weeter was limited by Twitter. #{tweet_item.missed_tweets_count} tweets missed.")
|
26
|
+
end
|
27
|
+
|
28
|
+
def notify_rate_limiting_initiated(tweet_item, limited_keys)
|
29
|
+
Weeter.logger.info("Initiated rate limiting with tweet: #{tweet_item.to_json}")
|
30
|
+
end
|
23
31
|
end
|
24
32
|
end
|
25
33
|
end
|
26
|
-
end
|
34
|
+
end
|
@@ -3,7 +3,7 @@ module Weeter
|
|
3
3
|
module Notification
|
4
4
|
class Resque
|
5
5
|
include Weeter::Plugins::Net::Redis
|
6
|
-
|
6
|
+
|
7
7
|
def initialize(client_app_config)
|
8
8
|
@config = client_app_config
|
9
9
|
end
|
@@ -20,12 +20,26 @@ module Weeter
|
|
20
20
|
enqueue(resque_job)
|
21
21
|
end
|
22
22
|
|
23
|
+
def notify_missed_tweets(tweet_item)
|
24
|
+
resque_job = %Q|{"class":"WeeterMissedTweetsJob","args":[#{tweet_item.to_json}]}|
|
25
|
+
Weeter.logger.info("Notifying of missed tweets (#{tweet_item.missed_tweets_count}).")
|
26
|
+
enqueue(resque_job)
|
27
|
+
end
|
28
|
+
|
29
|
+
def notify_rate_limiting_initiated(tweet_item, limited_keys)
|
30
|
+
payload = tweet_item.to_hash.merge(:limited_keys => limited_keys)
|
31
|
+
payload_json = MultiJson.encode(payload)
|
32
|
+
resque_job = %Q|{"class":"WeeterRateLimitingInitiatedJob","args":[#{payload_json}]}|
|
33
|
+
Weeter.logger.info("Initiated rate limiting with tweet: #{payload_json}")
|
34
|
+
enqueue(resque_job)
|
35
|
+
end
|
36
|
+
|
23
37
|
protected
|
24
|
-
|
38
|
+
|
25
39
|
def redis
|
26
40
|
@redis ||= create_redis_client
|
27
41
|
end
|
28
|
-
|
42
|
+
|
29
43
|
def enqueue(job)
|
30
44
|
redis.rpush(queue_key, job)
|
31
45
|
end
|
@@ -36,4 +50,4 @@ module Weeter
|
|
36
50
|
end
|
37
51
|
end
|
38
52
|
end
|
39
|
-
end
|
53
|
+
end
|
@@ -7,13 +7,16 @@ require 'active_support/core_ext/module/delegation'
|
|
7
7
|
module Weeter
|
8
8
|
module Plugins
|
9
9
|
class NotificationPlugin
|
10
|
-
delegate :publish_tweet,
|
11
|
-
|
12
|
-
|
10
|
+
delegate :publish_tweet,
|
11
|
+
:delete_tweet,
|
12
|
+
:notify_rate_limiting_initiated,
|
13
|
+
:notify_missed_tweets,
|
14
|
+
:to => :configured_plugin
|
15
|
+
|
13
16
|
def initialize(client_app_config)
|
14
17
|
@config = client_app_config
|
15
18
|
end
|
16
|
-
|
19
|
+
|
17
20
|
protected
|
18
21
|
def configured_plugin
|
19
22
|
@configured_plugin ||= begin
|
@@ -23,4 +26,4 @@ module Weeter
|
|
23
26
|
end
|
24
27
|
end
|
25
28
|
end
|
26
|
-
end
|
29
|
+
end
|
@@ -11,20 +11,24 @@ module Weeter
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def get_initial_filters(&block)
|
14
|
-
redis.get(@config.subscriptions_key) do |value|
|
14
|
+
deferred_get = redis.get(@config.subscriptions_key) do |value|
|
15
15
|
if value.nil?
|
16
16
|
raise "Expected to find subscription data at redis key #{@config.subscriptions_key}"
|
17
17
|
end
|
18
18
|
yield MultiJson.decode(value)
|
19
19
|
end
|
20
|
+
deferred_get.errback do |message|
|
21
|
+
Weeter.logger.error(message)
|
22
|
+
end
|
20
23
|
end
|
21
24
|
|
22
25
|
def listen_for_filter_update(tweet_consumer)
|
23
26
|
pub_sub_redis.subscribe(@config.subscriptions_changed_channel)
|
24
27
|
pub_sub_redis.on(:message) do |channel, message|
|
25
28
|
Weeter.logger.info [:message, channel, message]
|
26
|
-
Weeter.logger.info("
|
29
|
+
Weeter.logger.info("Retrieving updated filters from redis")
|
27
30
|
get_initial_filters do |filter_params|
|
31
|
+
Weeter.logger.info("Triggering reconnect Twitter stream with new filters")
|
28
32
|
tweet_consumer.reconnect(filter_params)
|
29
33
|
end
|
30
34
|
end
|
@@ -4,8 +4,13 @@ require 'multi_json'
|
|
4
4
|
module Weeter
|
5
5
|
module Twitter
|
6
6
|
class TweetConsumer
|
7
|
+
extend ::Forwardable
|
7
8
|
|
8
|
-
attr_reader :limiter
|
9
|
+
attr_reader :limiter, :notifier
|
10
|
+
def_delegators :@notifier, :notify_missed_tweets,
|
11
|
+
:notify_rate_limiting_initiated,
|
12
|
+
:delete_tweet,
|
13
|
+
:publish_tweet
|
9
14
|
|
10
15
|
def initialize(twitter_config, notifier, limiter, subscriptions_limit = nil)
|
11
16
|
@config = twitter_config
|
@@ -18,31 +23,31 @@ module Weeter
|
|
18
23
|
filter_params = limit_filter_params(filter_params) if @subscriptions_limit
|
19
24
|
filter_params = clean_filter_params(filter_params)
|
20
25
|
|
26
|
+
|
21
27
|
connect_options = {
|
22
28
|
ssl: true,
|
23
29
|
params: filter_params,
|
24
30
|
method: 'POST'
|
25
31
|
}.merge(@config.auth_options)
|
26
32
|
|
33
|
+
Weeter.logger.info("Connecting to Twitter stream...")
|
27
34
|
@stream = ::Twitter::JSONStream.connect(connect_options)
|
28
35
|
|
29
36
|
@stream.each_item do |item|
|
30
37
|
begin
|
31
38
|
tweet_item = TweetItem.new(MultiJson.decode(item))
|
32
39
|
|
33
|
-
if tweet_item.
|
34
|
-
|
40
|
+
if tweet_item.limit_notice?
|
41
|
+
notify_missed_tweets(tweet_item)
|
42
|
+
elsif tweet_item.deletion?
|
43
|
+
delete_tweet(tweet_item)
|
35
44
|
elsif tweet_item.publishable?
|
36
|
-
|
37
|
-
rate_limit_tweet(tweet_item)
|
38
|
-
else
|
39
|
-
@notifier.publish_tweet(tweet_item)
|
40
|
-
end
|
45
|
+
publish_or_rate_limit(tweet_item)
|
41
46
|
else
|
42
47
|
ignore_tweet(tweet_item)
|
43
48
|
end
|
44
49
|
rescue => ex
|
45
|
-
Weeter.logger.error("Twitter stream tweet exception: #{ex.class.name}: #{ex.message}")
|
50
|
+
Weeter.logger.error("Twitter stream tweet exception: #{ex.class.name}: #{ex.message} #{tweet_item.to_json}")
|
46
51
|
end
|
47
52
|
end
|
48
53
|
|
@@ -57,6 +62,7 @@ module Weeter
|
|
57
62
|
|
58
63
|
def reconnect(filter_params)
|
59
64
|
@stream.stop
|
65
|
+
@stream.unbind
|
60
66
|
connect(filter_params)
|
61
67
|
end
|
62
68
|
|
@@ -97,15 +103,17 @@ module Weeter
|
|
97
103
|
return {} if p.nil?
|
98
104
|
cleaned_params = {}
|
99
105
|
cleaned_params['follow'] = p['follow'] if (p['follow'] || []).any?
|
100
|
-
cleaned_params['follow'] = cleaned_params['follow'].map(&:to_i)
|
106
|
+
cleaned_params['follow'] = cleaned_params['follow'].map(&:to_i) if cleaned_params['follow']
|
101
107
|
cleaned_params['track'] = p['track'] if (p['track'] || []).any?
|
102
108
|
cleaned_params
|
103
109
|
end
|
104
110
|
|
105
111
|
def ignore_tweet(tweet_item)
|
112
|
+
return if tweet_item.disconnect_notice?
|
106
113
|
id = tweet_item['id_str']
|
107
114
|
text = tweet_item['text']
|
108
|
-
|
115
|
+
user = tweet_item['user']
|
116
|
+
user_id = user['id_str'] if user
|
109
117
|
Weeter.logger.info("Ignoring tweet #{id} from user #{user_id}: #{text}")
|
110
118
|
end
|
111
119
|
|
@@ -116,6 +124,19 @@ module Weeter
|
|
116
124
|
|
117
125
|
Weeter.logger.info("Rate Limiting tweet #{id} from user #{user_id}: #{text}")
|
118
126
|
end
|
127
|
+
|
128
|
+
def publish_or_rate_limit(tweet_item)
|
129
|
+
limit_result = limiter.process(*tweet_item.limiting_facets)
|
130
|
+
case limit_result.status
|
131
|
+
when Weeter::Limitator::INITIATE_LIMITING
|
132
|
+
notify_rate_limiting_initiated(tweet_item, limit_result.limited_keys)
|
133
|
+
rate_limit_tweet(tweet_item)
|
134
|
+
when Weeter::Limitator::CONTINUE_LIMITING
|
135
|
+
rate_limit_tweet(tweet_item)
|
136
|
+
when Weeter::Limitator::DO_NOT_LIMIT
|
137
|
+
publish_tweet(tweet_item)
|
138
|
+
end
|
139
|
+
end
|
119
140
|
end
|
120
141
|
end
|
121
142
|
end
|
@@ -20,7 +20,20 @@ module Weeter
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def publishable?
|
23
|
-
!retweeted? && !reply?
|
23
|
+
!retweeted? && !reply? && !disconnect_notice? && !limit_notice?
|
24
|
+
end
|
25
|
+
|
26
|
+
def disconnect_notice?
|
27
|
+
!@tweet_hash['disconnect'].nil?
|
28
|
+
end
|
29
|
+
|
30
|
+
def limit_notice?
|
31
|
+
!@tweet_hash['limit'].nil?
|
32
|
+
end
|
33
|
+
|
34
|
+
def missed_tweets_count
|
35
|
+
return nil unless limit_notice?
|
36
|
+
@tweet_hash['limit']['track']
|
24
37
|
end
|
25
38
|
|
26
39
|
def [](val)
|
@@ -31,6 +44,10 @@ module Weeter
|
|
31
44
|
MultiJson.encode(@tweet_hash)
|
32
45
|
end
|
33
46
|
|
47
|
+
def to_hash
|
48
|
+
@tweet_hash
|
49
|
+
end
|
50
|
+
|
34
51
|
def limiting_facets
|
35
52
|
self['entities']['hashtags'].map do |tag|
|
36
53
|
tag['text'].downcase.chomp
|
data/lib/weeter/version.rb
CHANGED
@@ -17,46 +17,57 @@ describe Weeter::Limitator do
|
|
17
17
|
it { limitator.should be }
|
18
18
|
end
|
19
19
|
|
20
|
-
describe '#
|
20
|
+
describe '#limit_status' do
|
21
21
|
|
22
22
|
subject do
|
23
|
-
limitator.
|
23
|
+
limitator.process(*keys)
|
24
24
|
end
|
25
25
|
|
26
26
|
context 'max: 0' do
|
27
27
|
let(:max) { 0 }
|
28
|
-
|
28
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
29
|
+
its(:limited_keys) { should == keys }
|
29
30
|
|
30
31
|
context 'no keys' do
|
31
32
|
let(:keys) { [] }
|
32
|
-
|
33
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
34
|
+
its(:limited_keys) { should == nil }
|
35
|
+
end
|
36
|
+
|
37
|
+
context 'two keys' do
|
38
|
+
let(:keys) { ['key', 'key2'] }
|
39
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
40
|
+
its(:limited_keys) { should == keys }
|
33
41
|
end
|
34
42
|
end
|
35
43
|
|
36
44
|
context 'max: 1' do
|
37
45
|
let(:max) { 1 }
|
38
46
|
|
39
|
-
|
47
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
48
|
+
its(:limited_keys) { should == nil }
|
40
49
|
|
41
50
|
context 'two keys within max' do
|
42
51
|
let(:keys) { ['key', 'key2'] }
|
43
52
|
|
44
|
-
|
53
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
45
54
|
end
|
46
55
|
|
47
56
|
context 'no keys' do
|
48
57
|
let(:keys) { [] }
|
49
|
-
|
58
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
59
|
+
its(:limited_keys) { should == nil }
|
50
60
|
end
|
51
61
|
|
52
|
-
context 'one key outside max' do
|
62
|
+
context 'one key just outside max' do
|
53
63
|
before do
|
54
64
|
max.times do
|
55
|
-
limitator.
|
65
|
+
limitator.process(*keys)
|
56
66
|
end
|
57
67
|
end
|
58
68
|
|
59
|
-
|
69
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
70
|
+
its(:limited_keys) { should == keys }
|
60
71
|
|
61
72
|
context 'outside duration' do
|
62
73
|
let(:some_time_after_duration) do
|
@@ -67,31 +78,71 @@ describe Weeter::Limitator do
|
|
67
78
|
limitator.stub(:now).and_return(some_time_after_duration)
|
68
79
|
end
|
69
80
|
|
70
|
-
|
81
|
+
its(:status) { should == Weeter::Limitator::DO_NOT_LIMIT }
|
82
|
+
its(:limited_keys) { should == nil }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
context 'two keys just past max' do
|
87
|
+
let(:keys) { ['key', 'key2'] }
|
88
|
+
|
89
|
+
before do
|
90
|
+
limitator.process(*keys)
|
71
91
|
end
|
92
|
+
|
93
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
94
|
+
its(:limited_keys) { should == keys }
|
72
95
|
end
|
73
96
|
|
74
|
-
context 'two keys
|
97
|
+
context 'two keys past max' do
|
75
98
|
let(:keys) { ['key', 'key2'] }
|
76
99
|
|
77
100
|
before do
|
78
|
-
limitator.
|
101
|
+
limitator.process(*keys)
|
102
|
+
limitator.process(*keys)
|
79
103
|
end
|
80
104
|
|
81
|
-
|
105
|
+
its(:status) { should == Weeter::Limitator::CONTINUE_LIMITING }
|
106
|
+
its(:limited_keys) { should == keys }
|
82
107
|
end
|
83
108
|
|
84
|
-
context 'one key
|
109
|
+
context 'one key just past max: 1, one key within max: 1' do
|
85
110
|
let(:max) { 1 }
|
86
111
|
let(:keys) { ['key', 'key2'] }
|
87
112
|
|
88
113
|
before do
|
89
|
-
limitator.
|
114
|
+
limitator.process(keys.first)
|
90
115
|
end
|
91
116
|
|
92
|
-
|
117
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
118
|
+
its(:limited_keys) { should == [keys.first] }
|
119
|
+
end
|
120
|
+
|
121
|
+
context 'one key past max: 1, one key within max: 1' do
|
122
|
+
let(:max) { 1 }
|
123
|
+
let(:keys) { ['key', 'key2'] }
|
124
|
+
|
125
|
+
before do
|
126
|
+
limitator.process(keys.first)
|
127
|
+
limitator.process(keys.first)
|
128
|
+
end
|
129
|
+
|
130
|
+
its(:status) { should == Weeter::Limitator::CONTINUE_LIMITING }
|
131
|
+
its(:limited_keys) { should == [keys.first] }
|
132
|
+
end
|
133
|
+
|
134
|
+
context 'one key past max: 1, one key just past max: 1' do
|
135
|
+
let(:max) { 1 }
|
136
|
+
let(:keys) { ['key', 'key2'] }
|
137
|
+
|
138
|
+
before do
|
139
|
+
limitator.process(*[keys.first])
|
140
|
+
limitator.process(*[keys.first, keys.last])
|
141
|
+
end
|
142
|
+
|
143
|
+
its(:status) { should == Weeter::Limitator::INITIATE_LIMITING }
|
144
|
+
its(:limited_keys) { should == keys }
|
93
145
|
end
|
94
146
|
end
|
95
147
|
end
|
96
148
|
end
|
97
|
-
|
@@ -10,7 +10,7 @@ describe Weeter::Twitter::TweetConsumer do
|
|
10
10
|
|
11
11
|
describe "auth" do
|
12
12
|
it 'should use connect to JSON stream with auth options for the configuration' do
|
13
|
-
@mock_stream = mock('JSONStream', :each_item => nil, :on_error => nil, :on_max_reconnects => nil)
|
13
|
+
@mock_stream = mock('JSONStream', :each_item => nil, :on_error => nil, :on_max_reconnects => nil, :on_close => nil)
|
14
14
|
Twitter::JSONStream.stub!(:connect).and_return(@mock_stream)
|
15
15
|
|
16
16
|
Weeter::Configuration::TwitterConfig.instance.stub!(:auth_options).and_return(:foo => :bar)
|
@@ -91,13 +91,22 @@ describe Weeter::Twitter::TweetConsumer do
|
|
91
91
|
|
92
92
|
describe "connecting to twitter" do
|
93
93
|
|
94
|
+
let(:tweet_values) {
|
95
|
+
[@tweet_hash]
|
96
|
+
}
|
97
|
+
let(:mock_stream) {
|
98
|
+
mock_stream = mock('JSONStream', :on_error => nil, :on_max_reconnects => nil, :on_close => nil)
|
99
|
+
each_item_stub = mock_stream.stub!(:each_item)
|
100
|
+
tweet_values.each do |t|
|
101
|
+
each_item_stub.and_yield(MultiJson.encode(t))
|
102
|
+
end
|
103
|
+
mock_stream
|
104
|
+
}
|
94
105
|
before(:each) do
|
95
106
|
@filter_params = {'follow' => ['1','2','3']}
|
96
107
|
Weeter::Configuration::TwitterConfig.instance.stub!(:auth_options).and_return(:foo => :bar)
|
97
|
-
@
|
98
|
-
|
99
|
-
@mock_stream.stub!(:each_item).and_yield(MultiJson.encode(@tweet_values))
|
100
|
-
Twitter::JSONStream.stub!(:connect).and_return(@mock_stream)
|
108
|
+
@tweet_hash = {'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => "1"}}
|
109
|
+
Twitter::JSONStream.stub!(:connect).and_return(mock_stream)
|
101
110
|
@client_proxy = mock('NotificationPlugin', :publish_tweet => nil)
|
102
111
|
@consumer = Weeter::Twitter::TweetConsumer.new(Weeter::Configuration::TwitterConfig.instance, @client_proxy, limiter)
|
103
112
|
end
|
@@ -107,7 +116,7 @@ describe Weeter::Twitter::TweetConsumer do
|
|
107
116
|
end
|
108
117
|
|
109
118
|
it "should instantiate a TweetItem" do
|
110
|
-
tweet_item = Weeter::TweetItem.new(@
|
119
|
+
tweet_item = Weeter::TweetItem.new(@tweet_hash)
|
111
120
|
Weeter::TweetItem.should_receive(:new).with({'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => "1"}}).and_return(tweet_item)
|
112
121
|
end
|
113
122
|
|
@@ -117,22 +126,41 @@ describe Weeter::Twitter::TweetConsumer do
|
|
117
126
|
end
|
118
127
|
|
119
128
|
it "should publish new tweet if publishable" do
|
120
|
-
mock_tweet = mock('tweet', :deletion? => false, :publishable? => true, :limiting_facets => [])
|
121
|
-
|
129
|
+
mock_tweet = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => [])
|
130
|
+
Weeter::TweetItem.stub!(:new).and_return(mock_tweet)
|
122
131
|
@client_proxy.should_receive(:publish_tweet).with(mock_tweet)
|
123
132
|
end
|
124
133
|
|
125
134
|
it "should not publish unpublishable tweets" do
|
126
|
-
mock_tweet = mock('tweet', :deletion? => false, :publishable? => false, :[] => '', :limiting_facets => [])
|
127
|
-
|
135
|
+
mock_tweet = mock('tweet', :deletion? => false, :publishable? => false, :limit_notice? => false, :[] => '', :limiting_facets => [])
|
136
|
+
Weeter::TweetItem.stub!(:new).and_return mock_tweet
|
128
137
|
@client_proxy.should_not_receive(:publish_tweet).with(mock_tweet)
|
129
138
|
end
|
130
139
|
|
131
140
|
it "should delete deletion tweets" do
|
132
|
-
mock_tweet = mock('tweet', :deletion? => true, :publishable? => false, :limiting_facets => [])
|
133
|
-
|
141
|
+
mock_tweet = mock('tweet', :deletion? => true, :publishable? => false, :limit_notice? => false, :limiting_facets => [])
|
142
|
+
Weeter::TweetItem.stub!(:new).and_return mock_tweet
|
134
143
|
@client_proxy.should_receive(:delete_tweet).with(mock_tweet)
|
135
144
|
end
|
145
|
+
|
146
|
+
it "should notify when stream is limited by Twitter" do
|
147
|
+
tweet_item = Weeter::TweetItem.new({'limit' => { 'track' => 65 } })
|
148
|
+
Weeter::TweetItem.stub!(:new).and_return(tweet_item)
|
149
|
+
@client_proxy.should_receive(:notify_missed_tweets).with(tweet_item)
|
150
|
+
end
|
151
|
+
|
152
|
+
context "when weeter is initiating rate-limiting on a facet" do
|
153
|
+
let(:tweet_values) {
|
154
|
+
[@tweet_hash, @tweet_hash]
|
155
|
+
}
|
156
|
+
it "should notify that rate limiting is being initiated" do
|
157
|
+
tweet_item1 = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => ['key'], :[] => '1')
|
158
|
+
tweet_item2 = mock('tweet', :deletion? => false, :publishable? => true, :limit_notice? => false, :limiting_facets => ['key'], :[] => '2')
|
159
|
+
Weeter::TweetItem.stub!(:new).and_return(tweet_item1, tweet_item2)
|
160
|
+
|
161
|
+
@client_proxy.should_receive(:notify_rate_limiting_initiated).with(tweet_item2, ['key'])
|
162
|
+
end
|
163
|
+
end
|
136
164
|
end
|
137
165
|
|
138
166
|
end
|
@@ -1,6 +1,9 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe Weeter::TweetItem do
|
4
|
+
let(:tweet_json) {
|
5
|
+
{'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => '1'}}
|
6
|
+
}
|
4
7
|
|
5
8
|
describe "deletion?" do
|
6
9
|
it "should be true if it is a deletion request" do
|
@@ -16,37 +19,54 @@ describe Weeter::TweetItem do
|
|
16
19
|
|
17
20
|
describe "publishable" do
|
18
21
|
|
19
|
-
before do
|
20
|
-
@tweet_json = {'text' => "Hey", 'id_str' => "123", 'user' => {'id_str' => '1'}}
|
21
|
-
end
|
22
22
|
|
23
23
|
it "should be publishable if not a reply or a retweet" do
|
24
|
-
item = Weeter::TweetItem.new(
|
24
|
+
item = Weeter::TweetItem.new(tweet_json)
|
25
25
|
item.should be_publishable
|
26
26
|
end
|
27
27
|
|
28
28
|
it "should not be publishable if implicitly retweeted" do
|
29
|
-
item = Weeter::TweetItem.new(
|
29
|
+
item = Weeter::TweetItem.new(tweet_json.merge({'text' => 'RT @joe Hey'}))
|
30
30
|
item.should_not be_publishable
|
31
31
|
end
|
32
32
|
|
33
33
|
it "should not be publishable if explicitly retweeted" do
|
34
|
-
item = Weeter::TweetItem.new(
|
34
|
+
item = Weeter::TweetItem.new(tweet_json.merge('retweeted_status' => {'id_str' => '111', 'text' => 'Hey', 'user' => {'id_str' => "1"}}))
|
35
35
|
item.should_not be_publishable
|
36
36
|
end
|
37
37
|
|
38
38
|
it "should not be publishable if implicit reply" do
|
39
|
-
item = Weeter::TweetItem.new(
|
39
|
+
item = Weeter::TweetItem.new(tweet_json.merge('text' => '@joe Hey'))
|
40
40
|
item.should_not be_publishable
|
41
41
|
end
|
42
42
|
|
43
43
|
it "should not be publishable if explicit reply" do
|
44
|
-
item = Weeter::TweetItem.new(
|
44
|
+
item = Weeter::TweetItem.new(tweet_json.merge('text' => '@joe Hey', 'in_reply_to_user_id_str' => '1'))
|
45
|
+
item.should_not be_publishable
|
46
|
+
end
|
47
|
+
|
48
|
+
it "should not be publishable if disconnect message" do
|
49
|
+
item = Weeter::TweetItem.new({"disconnect" => {"code" => 7,"stream_name" => "YappBox-statuses668638","reason" => "admin logout"}})
|
45
50
|
item.should_not be_publishable
|
46
51
|
end
|
47
52
|
|
48
53
|
end
|
49
54
|
|
55
|
+
describe "limit_notice?" do
|
56
|
+
it "should be true if it's a limit notice" do
|
57
|
+
item = Weeter::TweetItem.new({ 'limit' => { 'track' => 65 }})
|
58
|
+
item.should be_limit_notice
|
59
|
+
item.missed_tweets_count.should == 65
|
60
|
+
end
|
61
|
+
it "should not be true if it's a limit notice" do
|
62
|
+
item = Weeter::TweetItem.new(tweet_json)
|
63
|
+
item.should_not be_limit_notice
|
64
|
+
lambda {
|
65
|
+
item.missed_tweets_count
|
66
|
+
}.should_not raise_error
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
50
70
|
describe "json attributes" do
|
51
71
|
|
52
72
|
it "should delegate hash calls to its json" do
|
@@ -63,4 +83,4 @@ describe Weeter::TweetItem do
|
|
63
83
|
|
64
84
|
|
65
85
|
|
66
|
-
end
|
86
|
+
end
|
data/weeter.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: weeter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.13.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -11,11 +11,11 @@ authors:
|
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
date: 2012-
|
14
|
+
date: 2012-12-04 00:00:00.000000000Z
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: eventmachine
|
18
|
-
requirement: &
|
18
|
+
requirement: &2161838440 !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
21
|
- - ! '>='
|
@@ -23,10 +23,10 @@ dependencies:
|
|
23
23
|
version: '0'
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
|
-
version_requirements: *
|
26
|
+
version_requirements: *2161838440
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: eventmachine_httpserver
|
29
|
-
requirement: &
|
29
|
+
requirement: &2161836940 !ruby/object:Gem::Requirement
|
30
30
|
none: false
|
31
31
|
requirements:
|
32
32
|
- - ! '>='
|
@@ -34,10 +34,10 @@ dependencies:
|
|
34
34
|
version: 0.2.1
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *2161836940
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: em-hiredis
|
40
|
-
requirement: &
|
40
|
+
requirement: &2161835100 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
@@ -45,10 +45,10 @@ dependencies:
|
|
45
45
|
version: 0.1.0
|
46
46
|
type: :runtime
|
47
47
|
prerelease: false
|
48
|
-
version_requirements: *
|
48
|
+
version_requirements: *2161835100
|
49
49
|
- !ruby/object:Gem::Dependency
|
50
50
|
name: multi_json
|
51
|
-
requirement: &
|
51
|
+
requirement: &2161833640 !ruby/object:Gem::Requirement
|
52
52
|
none: false
|
53
53
|
requirements:
|
54
54
|
- - ! '>='
|
@@ -56,10 +56,10 @@ dependencies:
|
|
56
56
|
version: 1.0.2
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
|
-
version_requirements: *
|
59
|
+
version_requirements: *2161833640
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: hashie
|
62
|
-
requirement: &
|
62
|
+
requirement: &2161832500 !ruby/object:Gem::Requirement
|
63
63
|
none: false
|
64
64
|
requirements:
|
65
65
|
- - ! '>='
|
@@ -67,10 +67,10 @@ dependencies:
|
|
67
67
|
version: 1.1.0
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
|
-
version_requirements: *
|
70
|
+
version_requirements: *2161832500
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
72
|
name: em-http-request
|
73
|
-
requirement: &
|
73
|
+
requirement: &2161832040 !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
76
|
- - ! '>='
|
@@ -78,10 +78,10 @@ dependencies:
|
|
78
78
|
version: 1.0.0
|
79
79
|
type: :runtime
|
80
80
|
prerelease: false
|
81
|
-
version_requirements: *
|
81
|
+
version_requirements: *2161832040
|
82
82
|
- !ruby/object:Gem::Dependency
|
83
83
|
name: i18n
|
84
|
-
requirement: &
|
84
|
+
requirement: &2161831220 !ruby/object:Gem::Requirement
|
85
85
|
none: false
|
86
86
|
requirements:
|
87
87
|
- - ~>
|
@@ -89,10 +89,10 @@ dependencies:
|
|
89
89
|
version: 0.6.0
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
|
-
version_requirements: *
|
92
|
+
version_requirements: *2161831220
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
94
|
name: activesupport
|
95
|
-
requirement: &
|
95
|
+
requirement: &2161830060 !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
98
98
|
- - ! '>='
|
@@ -100,10 +100,10 @@ dependencies:
|
|
100
100
|
version: 3.1.1
|
101
101
|
type: :runtime
|
102
102
|
prerelease: false
|
103
|
-
version_requirements: *
|
103
|
+
version_requirements: *2161830060
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
name: simple_oauth
|
106
|
-
requirement: &
|
106
|
+
requirement: &2161829240 !ruby/object:Gem::Requirement
|
107
107
|
none: false
|
108
108
|
requirements:
|
109
109
|
- - ~>
|
@@ -111,10 +111,10 @@ dependencies:
|
|
111
111
|
version: 0.1.5
|
112
112
|
type: :runtime
|
113
113
|
prerelease: false
|
114
|
-
version_requirements: *
|
114
|
+
version_requirements: *2161829240
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: lukemelia-twitter-stream
|
117
|
-
requirement: &
|
117
|
+
requirement: &2161828460 !ruby/object:Gem::Requirement
|
118
118
|
none: false
|
119
119
|
requirements:
|
120
120
|
- - ~>
|
@@ -122,10 +122,10 @@ dependencies:
|
|
122
122
|
version: 0.1.15
|
123
123
|
type: :runtime
|
124
124
|
prerelease: false
|
125
|
-
version_requirements: *
|
125
|
+
version_requirements: *2161828460
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: rspec
|
128
|
-
requirement: &
|
128
|
+
requirement: &2161827700 !ruby/object:Gem::Requirement
|
129
129
|
none: false
|
130
130
|
requirements:
|
131
131
|
- - ~>
|
@@ -133,7 +133,18 @@ dependencies:
|
|
133
133
|
version: 2.6.0
|
134
134
|
type: :development
|
135
135
|
prerelease: false
|
136
|
-
version_requirements: *
|
136
|
+
version_requirements: *2161827700
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: ZenTest
|
139
|
+
requirement: &2161826960 !ruby/object:Gem::Requirement
|
140
|
+
none: false
|
141
|
+
requirements:
|
142
|
+
- - ! '>='
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
145
|
+
type: :development
|
146
|
+
prerelease: false
|
147
|
+
version_requirements: *2161826960
|
137
148
|
description: Weeter subscribes to a set of twitter users or search terms using Twitter's
|
138
149
|
streaming API, and notifies your app with each new tweet.
|
139
150
|
email:
|