twitter_with_auto_pagination 0.8.12 → 0.9.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/lib/twitter_with_auto_pagination/analysis/api.rb +9 -0
  3. data/lib/twitter_with_auto_pagination/{rest/uncategorized.rb → analysis/timelines.rb} +2 -2
  4. data/lib/twitter_with_auto_pagination/cache.rb +78 -0
  5. data/lib/twitter_with_auto_pagination/caching_and_logging.rb +80 -0
  6. data/lib/twitter_with_auto_pagination/client.rb +57 -0
  7. data/lib/twitter_with_auto_pagination/collector.rb +18 -0
  8. data/lib/twitter_with_auto_pagination/log_subscriber.rb +59 -26
  9. data/lib/twitter_with_auto_pagination/logger.rb +10 -0
  10. data/lib/twitter_with_auto_pagination/parallel.rb +25 -0
  11. data/lib/twitter_with_auto_pagination/rate_limit.rb +56 -0
  12. data/lib/twitter_with_auto_pagination/rest/api.rb +16 -14
  13. data/lib/twitter_with_auto_pagination/rest/extension/friends_and_followers.rb +0 -7
  14. data/lib/twitter_with_auto_pagination/rest/favorites.rb +13 -7
  15. data/lib/twitter_with_auto_pagination/rest/friends_and_followers.rb +33 -70
  16. data/lib/twitter_with_auto_pagination/rest/lists.rb +10 -9
  17. data/lib/twitter_with_auto_pagination/rest/search.rb +16 -6
  18. data/lib/twitter_with_auto_pagination/rest/timelines.rb +13 -26
  19. data/lib/twitter_with_auto_pagination/rest/users.rb +27 -40
  20. data/lib/twitter_with_auto_pagination/rest/utils.rb +29 -149
  21. data/lib/twitter_with_auto_pagination/serializer.rb +27 -0
  22. data/lib/twitter_with_auto_pagination.rb +2 -38
  23. data/spec/helper.rb +167 -36
  24. data/spec/twitter_with_auto_pagination/cache_spec.rb +71 -0
  25. data/spec/twitter_with_auto_pagination/client_spec.rb +7 -145
  26. data/spec/twitter_with_auto_pagination/parallel_spec.rb +23 -0
  27. data/spec/twitter_with_auto_pagination/rest/favorites_spec.rb +58 -0
  28. data/spec/twitter_with_auto_pagination/rest/friends_and_followers_spec.rb +161 -0
  29. data/spec/twitter_with_auto_pagination/rest/lists_spec.rb +39 -0
  30. data/spec/twitter_with_auto_pagination/rest/search_spec.rb +42 -0
  31. data/spec/twitter_with_auto_pagination/rest/timelines_spec.rb +82 -0
  32. data/spec/twitter_with_auto_pagination/rest/users_spec.rb +143 -0
  33. data/twitter_with_auto_pagination.gemspec +1 -1
  34. metadata +28 -3
data/spec/helper.rb CHANGED
@@ -1,68 +1,199 @@
1
+ require 'dotenv/load'
1
2
  require 'twitter_with_auto_pagination'
2
3
  require 'rspec'
3
- require 'stringio'
4
- require 'tempfile'
5
- require 'timecop'
6
- require 'webmock/rspec'
7
-
8
- require_relative 'support/media_object_examples'
9
4
 
10
5
  RSpec.configure do |config|
6
+ config.before(:suite) do
7
+ $fetch_count = $request_count = 0
8
+
9
+ set_trace_func proc { |event, file, line, id, binding, klass|
10
+ if klass == TwitterWithAutoPagination::Cache && id == :fetch && event == 'call'
11
+ $fetch_called = true
12
+ $fetch_count += 1
13
+ end
14
+ if klass == Twitter::REST::Request && id == :perform && event == 'call'
15
+ $request_called = true
16
+ $request_count += 1
17
+ end
18
+ }
19
+ end
20
+
21
+ config.define_derived_metadata do |meta|
22
+ meta[:aggregate_failures] = true unless meta.key?(:aggregate_failures)
23
+ end
24
+
11
25
  config.expect_with :rspec do |c|
12
26
  c.syntax = :expect
13
27
  end
14
28
  end
15
29
 
16
- BASE_URL = 'https://api.twitter.com'.freeze
30
+ RSpec::Matchers.define_negated_matcher :not_change, :change
31
+
32
+ RSpec::Matchers.define :fetch do
33
+ match do |actual|
34
+ before = $fetch_called
35
+ actual.call
36
+ @count = 1 if @count.nil?
37
+ @actual_count = $fetch_count
38
+ result = !before && $fetch_called && @actual_count == @count
39
+
40
+ $fetch_called = false
41
+ $fetch_count = 0
42
+
43
+ result
44
+ end
45
+
46
+ chain(:once) { @count = 1 }
47
+ chain(:twice) { @count = 2 }
48
+ chain(:exactly) { |count| @count = count }
49
+ chain(:times) {}
50
+
51
+ failure_message do |actual|
52
+ "expected to call #fetch #{@count} times but called #{@actual_count} times"
53
+ end
54
+
55
+ supports_block_expectations
56
+ end
57
+
58
+ RSpec::Matchers.define :request do
59
+ match do |actual|
60
+ before = $request_called
61
+ actual.call
62
+ @count = 1 if @count.nil?
63
+ @actual_count = $request_count
64
+ result = !before && $request_called && @actual_count == @count
65
+
66
+ $request_called = false
67
+ $request_count = 0
68
+
69
+ result
70
+ end
71
+
72
+ chain(:once) { @count = 1 }
73
+ chain(:twice) { @count = 2 }
74
+ chain(:exactly) { |count| @count = count }
75
+ chain(:times) {}
76
+
77
+ failure_message do |actual|
78
+ "expected to call #request #{@count} times but called #{@actual_count} times"
79
+ end
80
+
81
+ supports_block_expectations
82
+ end
83
+
84
+ RSpec::Matchers.define :not_fetch do
85
+ match do |actual|
86
+ before = $fetch_called
87
+ actual.call
88
+ after = $fetch_called
89
+ $fetch_called = false
90
+ !before && !after
91
+ end
17
92
 
18
- def a_delete(path)
19
- a_request(:delete, BASE_URL + path)
93
+ failure_message { |actual| 'expected not to fetch' }
94
+ supports_block_expectations
20
95
  end
21
96
 
22
- def a_get(path)
23
- a_request(:get, BASE_URL + path)
97
+ RSpec::Matchers.define :not_request do
98
+ match do |actual|
99
+ before = $request_called
100
+ actual.call
101
+ after = $request_called
102
+ $request_called = false
103
+ !before && !after
104
+ end
105
+
106
+ failure_message { |actual| 'expected not to request' }
107
+ supports_block_expectations
24
108
  end
25
109
 
26
- def a_post(path)
27
- a_request(:post, BASE_URL + path)
110
+ RSpec::Matchers.define :match_twitter do |expected|
111
+ match do |actual|
112
+ if expected.is_a?(Array)
113
+ if expected[0].is_a?(Integer)
114
+ actual == expected
115
+ else
116
+ actual.map { |r| r[:id] } == expected.map { |r| r[:id] }
117
+ end
118
+ elsif expected.is_a?(Hash)
119
+ actual[:id] == expected[:id]
120
+ else
121
+ actual == expected
122
+ end
123
+ end
28
124
  end
29
125
 
30
- def a_put(path)
31
- a_request(:put, BASE_URL + path)
126
+ RSpec.shared_examples 'continuous calls' do
127
+ it 'fetches the result from a cache for the second time' do
128
+ result1 = result2 = nil
129
+ expect { result1 = client.send(name, *params) }.to fetch & request
130
+ expect { result2 = client.send(name, *params) }.to fetch & not_request
131
+ expect(result1).to match_twitter(result2)
132
+ end
32
133
  end
33
134
 
34
- def stub_delete(path)
35
- stub_request(:delete, BASE_URL + path)
135
+ RSpec.shared_examples 'cache: false is specified' do
136
+ it 'sends http requests' do
137
+ expect { client.send(name, *params) }.to fetch & request
138
+ expect { client.send(name, *params, cache: false) }.to not_fetch & request
139
+ end
36
140
  end
37
141
 
38
- def stub_get(path)
39
- stub_request(:get, BASE_URL + path)
142
+ RSpec.shared_examples 'when a value is changed' do
143
+ it 'sends http requests' do
144
+ expect { client.send(name, *params) }.to fetch & request
145
+ expect { client.send(name, *params2) }.to fetch & request
146
+ end
40
147
  end
41
148
 
42
- def stub_post(path)
43
- stub_request(:post, BASE_URL + path)
149
+ RSpec.shared_examples 'when options are changed' do
150
+ it 'sends http requests' do
151
+ expect { client.send(name, *params) }.to fetch & request
152
+ expect { client.send(name, *params, hello: :world) }.to fetch & request
153
+ end
154
+ end
155
+
156
+ RSpec.shared_examples 'when a client is changed, it shares a cache' do
157
+ it 'shares a cache' do
158
+ expect { client.send(name, *params) }.to fetch & request
159
+ expect { client2.send(name, *params) }.to fetch & not_request
160
+ end
161
+ end
162
+
163
+ RSpec.shared_examples 'when a client is changed, it does not share a cache' do
164
+ it 'does not share a cache' do
165
+ expect { client.send(name, *params) }.to fetch & request
166
+ expect { client2.send(name, *params) }.to fetch & request
167
+ end
44
168
  end
45
169
 
46
- def stub_put(path)
47
- stub_request(:put, BASE_URL + path)
170
+ RSpec.shared_examples 'when one param is specified, it raises an exception' do
171
+ it 'raises an exception' do
172
+ expect { client.send(name, id) }.to raise_error(ArgumentError)
173
+ end
48
174
  end
49
175
 
50
- def fixture_path
51
- File.expand_path('../fixtures', __FILE__)
176
+ RSpec.shared_examples 'when any params is not specified, it raises an exception' do
177
+ it 'raises an exception' do
178
+ expect { client.send(name) }.to raise_error(ArgumentError)
179
+ end
52
180
  end
53
181
 
54
- def fixture(file)
55
- File.new(fixture_path + '/' + file)
182
+ RSpec.shared_examples 'when any params is not specified, it returns a same result as a result with one param' do
183
+ it 'returns a same result as a result with one param' do
184
+ result1 = result2 = nil
185
+ expect { result1 = client.send(name) }.to fetch & request
186
+ expect { result2 = client.send(name, id) }.to fetch & request
187
+ expect(result1).to match_twitter(result2)
188
+ end
56
189
  end
57
190
 
58
- def capture_warning
59
- begin
60
- old_stderr = $stderr
61
- $stderr = StringIO.new
62
- yield
63
- result = $stderr.string
64
- ensure
65
- $stderr = old_stderr
191
+ RSpec.shared_examples 'when count is specified' do |count|
192
+ it 'requests twice' do
193
+ result1 = result2 = nil
194
+ expect { result1 = client.send(name, *params, count: count) }.to fetch & request.twice
195
+ expect { result2 = client.send(name, *params, count: count) }.to fetch & not_request
196
+ expect(result1.size).to be > (count / 2)
197
+ expect(result1).to match_twitter(result2)
66
198
  end
67
- result
68
199
  end
@@ -0,0 +1,71 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::Cache do
4
+ let(:cache) { TwitterWithAutoPagination::Cache.new }
5
+
6
+ # describe '#fetch' do
7
+ #
8
+ # before do
9
+ # cache.client.clear
10
+ # end
11
+ #
12
+ # context 'with a stored value' do
13
+ # before do
14
+ # cache.client.write('key', 1)
15
+ # allow(cache).to receive(:normalize_key).and_return('key')
16
+ # end
17
+ # it 'calls a block' do
18
+ # expect { |b| cache.fetch('anything', 'user', &b)}.to yield_control
19
+ # end
20
+ # end
21
+ #
22
+ # context 'without any value' do
23
+ # before do
24
+ # allow(cache).to receive(:normalize_key).and_return('key')
25
+ # end
26
+ # it 'does not call a block' do
27
+ # expect { |b| cache.fetch('anything', 'user', &b)}.to_not yield_control
28
+ # end
29
+ # end
30
+ # end
31
+
32
+ describe '#normalize_key' do
33
+ it 'returns a key' do
34
+ expect(cache.send(:normalize_key, :search, 'a')).to match(/\Asearch:query:a/)
35
+ expect(cache.send(:normalize_key, :friendship?, %w(a b))).to match(/\Afriendship\?:from:a:to:b/)
36
+ expect(cache.send(:normalize_key, :anything, 1)).to match(/\Aanything:id:/)
37
+ expect(cache.send(:normalize_key, :anything, 'a')).to match(/\Aanything:screen_name:a:options:empty\z/)
38
+ expect(cache.send(:normalize_key, :anything, 'a', {count: 10})).to match(/\Aanything:screen_name:a:options:count:10\z/)
39
+ expect {cache.send(:normalize_key, :anything, nil)}.to raise_error(RuntimeError)
40
+ end
41
+
42
+ end
43
+
44
+ describe '#user_identifier' do
45
+ it 'returns a serialized string' do
46
+ expect(cache.send(:user_identifier, 1)).to eq('id:1')
47
+ expect(cache.send(:user_identifier, 'abc')).to eq('screen_name:abc')
48
+ expect(cache.send(:user_identifier, [1, 2])).to match(/\Aids:2-\w+\z/)
49
+ expect(cache.send(:user_identifier, %w(abc cde fgh))).to match(/\Ascreen_names:3-\w+\z/)
50
+ expect { cache.send(:user_identifier, nil) }.to raise_error(RuntimeError)
51
+ end
52
+ end
53
+
54
+ describe '#options_identifier' do
55
+ it 'returns serialized string' do
56
+ expect(cache.send(:options_identifier, a: 1, b: 2)).to match(/\Aoptions:a:1,b:2\z/)
57
+ end
58
+
59
+ context 'empty options' do
60
+ it 'returns serialized string ends with "empty"' do
61
+ expect(cache.send(:options_identifier, {})).to eq("options:empty")
62
+ end
63
+ end
64
+ end
65
+
66
+ describe '#hexdigest' do
67
+ it 'returns hexdigest' do
68
+ expect(cache.send(:hexdigest, 'hello')).to eq(Digest::MD5.hexdigest('hello'))
69
+ end
70
+ end
71
+ end
@@ -1,150 +1,12 @@
1
1
  require 'helper'
2
2
 
3
3
  describe TwitterWithAutoPagination::Client do
4
- let(:config) {
5
- {
6
- consumer_key: 'CK',
7
- consumer_secret: 'CS',
8
- access_token: 'AT',
9
- access_token_secret: 'ATS',
10
- }
11
- }
12
- let(:client) { TwitterWithAutoPagination::Client.new(config) }
13
-
14
- describe '#user?' do
15
- before do
16
- stub_get('/1.1/users/show.json').with(query: {screen_name: 'sferik'}).to_return(body: fixture('sferik.json'), headers: {content_type: 'application/json; charset=utf-8'})
17
- stub_get('/1.1/users/show.json').with(query: {screen_name: 'pengwynn'}).to_return(body: fixture('not_found.json'), status: 404, headers: {content_type: 'application/json; charset=utf-8'})
18
- end
19
- it 'requests the correct resource' do
20
- client.user?('sferik')
21
- expect(a_get('/1.1/users/show.json').with(query: {screen_name: 'sferik'})).to have_been_made
22
- end
23
- it 'returns true if user exists' do
24
- user = client.user?('sferik')
25
- expect(user).to be true
26
- end
27
- it 'returns false if user does not exist' do
28
- user = client.user?('pengwynn')
29
- expect(user).to be false
30
- end
4
+ let(:client) do
5
+ described_class.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
31
11
  end
32
-
33
- # describe '#initialize' do
34
- # let(:default_call_count) { 0 }
35
- #
36
- # it 'sets call_count to 0' do
37
- # expect(client.call_count).to eq(default_call_count)
38
- # end
39
- #
40
- # context 'without params' do
41
- # end
42
- #
43
- # context 'with params' do
44
- # end
45
- # end
46
- #
47
- # describe '#logger' do
48
- # it 'has logger' do
49
- # expect(client.logger).to be_truthy
50
- # end
51
- # end
52
- #
53
- # describe '#call_old_method' do
54
- # end
55
- #
56
- # describe '#collect_with_max_id' do
57
- # end
58
- #
59
- # describe '#collect_with_cursor' do
60
- # end
61
- #
62
- # describe '#file_cache_key' do
63
- # end
64
- #
65
- # describe '#namespaced_key' do
66
- # end
67
- #
68
- # describe '#encode_json' do
69
- # end
70
- #
71
- # describe '#decode_json' do
72
- # end
73
- #
74
- # describe '#fetch_cache_or_call_api' do
75
- # end
76
- #
77
- # describe '#user_timeline' do
78
- # it 'calls old_user_timeline' do
79
- # expect(client).to receive(:old_user_timeline)
80
- # client.user_timeline
81
- # end
82
- #
83
- # it 'calls collect_with_max_id' do
84
- # expect(client).to receive(:collect_with_max_id)
85
- # client.user_timeline
86
- # end
87
- # end
88
- #
89
- # describe '#user_photos' do
90
- # it 'calls user_timeline' do
91
- # expect(client).to receive(:user_timeline)
92
- # client.user_photos
93
- # end
94
- # end
95
- #
96
- # describe '#friends' do
97
- # it 'calls old_friends' do
98
- # expect(client).to receive(:old_friends)
99
- # client.friends
100
- # end
101
- #
102
- # it 'calls collect_with_cursor' do
103
- # expect(client).to receive(:collect_with_cursor)
104
- # client.friends
105
- # end
106
- # end
107
- #
108
- # describe '#followers' do
109
- # it 'calls old_followers' do
110
- # expect(client).to receive(:old_followers)
111
- # client.followers
112
- # end
113
- #
114
- # it 'calls collect_with_cursor' do
115
- # expect(client).to receive(:collect_with_cursor)
116
- # client.followers
117
- # end
118
- # end
119
- #
120
- # describe '#friend_ids' do
121
- # it 'calls old_friend_ids' do
122
- # expect(client).to receive(:old_friend_ids)
123
- # client.friend_ids
124
- # end
125
- #
126
- # it 'calls collect_with_cursor' do
127
- # expect(client).to receive(:collect_with_cursor)
128
- # client.friend_ids
129
- # end
130
- # end
131
- #
132
- # describe '#follower_ids' do
133
- # it 'calls old_follower_ids' do
134
- # expect(client).to receive(:old_follower_ids)
135
- # client.follower_ids
136
- # end
137
- #
138
- # it 'calls collect_with_cursor' do
139
- # expect(client).to receive(:collect_with_cursor)
140
- # client.follower_ids
141
- # end
142
- # end
143
- #
144
- # describe '#users' do
145
- # it 'calls old_users' do
146
- # expect(client).to receive(:old_users)
147
- # client.users([1, 2, 3])
148
- # end
149
- # end
150
12
  end
@@ -0,0 +1,23 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::Parallel do
4
+ let(:client) do
5
+ TwitterWithAutoPagination::Client.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
11
+ end
12
+
13
+ let(:id) { 58135830 }
14
+
15
+ describe '#parallel' do
16
+ it 'calls #users' do
17
+ expect(client).to receive(:users).with([id], any_args)
18
+ client.parallel do |batch|
19
+ batch.users([id])
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,58 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::REST::Favorites do
4
+ let(:client) do
5
+ TwitterWithAutoPagination::Client.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
11
+ end
12
+
13
+ let(:client2) do
14
+ TwitterWithAutoPagination::Client.new(
15
+ consumer_key: ENV['CK2'],
16
+ consumer_secret: ENV['CS2'],
17
+ access_token: ENV['AT2'],
18
+ access_token_secret: ENV['ATS2']
19
+ )
20
+ end
21
+
22
+ let(:id) { 58135830 }
23
+ let(:id2) { 22356250 }
24
+
25
+ before do
26
+ client.cache.clear
27
+ $fetch_called = $request_called = false
28
+ $fetch_count = $request_count = 0
29
+ end
30
+
31
+ describe '#favorites' do
32
+ let(:name) { :favorites }
33
+
34
+ context 'with one param' do
35
+ let(:params) { [id] }
36
+ let(:params2) { [id2] }
37
+
38
+ it_behaves_like 'continuous calls'
39
+ it_behaves_like 'cache: false is specified'
40
+ it_behaves_like 'when a value is changed'
41
+ it_behaves_like 'when options are changed'
42
+ it_behaves_like 'when a client is changed, it shares a cache'
43
+ it_behaves_like 'when count is specified', 200
44
+ end
45
+
46
+ context 'with no params' do
47
+ let(:params) { [] }
48
+
49
+ it_behaves_like 'continuous calls'
50
+ it_behaves_like 'cache: false is specified'
51
+ it_behaves_like 'when options are changed'
52
+ it_behaves_like 'when a client is changed, it does not share a cache'
53
+ it_behaves_like 'when count is specified', 200
54
+ end
55
+
56
+ it_behaves_like 'when any params is not specified, it returns a same result as a result with one param'
57
+ end
58
+ end