twitter_with_auto_pagination 0.8.12 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/lib/twitter_with_auto_pagination/analysis/api.rb +9 -0
  3. data/lib/twitter_with_auto_pagination/{rest/uncategorized.rb → analysis/timelines.rb} +2 -2
  4. data/lib/twitter_with_auto_pagination/cache.rb +78 -0
  5. data/lib/twitter_with_auto_pagination/caching_and_logging.rb +80 -0
  6. data/lib/twitter_with_auto_pagination/client.rb +57 -0
  7. data/lib/twitter_with_auto_pagination/collector.rb +18 -0
  8. data/lib/twitter_with_auto_pagination/log_subscriber.rb +59 -26
  9. data/lib/twitter_with_auto_pagination/logger.rb +10 -0
  10. data/lib/twitter_with_auto_pagination/parallel.rb +25 -0
  11. data/lib/twitter_with_auto_pagination/rate_limit.rb +56 -0
  12. data/lib/twitter_with_auto_pagination/rest/api.rb +16 -14
  13. data/lib/twitter_with_auto_pagination/rest/extension/friends_and_followers.rb +0 -7
  14. data/lib/twitter_with_auto_pagination/rest/favorites.rb +13 -7
  15. data/lib/twitter_with_auto_pagination/rest/friends_and_followers.rb +33 -70
  16. data/lib/twitter_with_auto_pagination/rest/lists.rb +10 -9
  17. data/lib/twitter_with_auto_pagination/rest/search.rb +16 -6
  18. data/lib/twitter_with_auto_pagination/rest/timelines.rb +13 -26
  19. data/lib/twitter_with_auto_pagination/rest/users.rb +27 -40
  20. data/lib/twitter_with_auto_pagination/rest/utils.rb +29 -149
  21. data/lib/twitter_with_auto_pagination/serializer.rb +27 -0
  22. data/lib/twitter_with_auto_pagination.rb +2 -38
  23. data/spec/helper.rb +167 -36
  24. data/spec/twitter_with_auto_pagination/cache_spec.rb +71 -0
  25. data/spec/twitter_with_auto_pagination/client_spec.rb +7 -145
  26. data/spec/twitter_with_auto_pagination/parallel_spec.rb +23 -0
  27. data/spec/twitter_with_auto_pagination/rest/favorites_spec.rb +58 -0
  28. data/spec/twitter_with_auto_pagination/rest/friends_and_followers_spec.rb +161 -0
  29. data/spec/twitter_with_auto_pagination/rest/lists_spec.rb +39 -0
  30. data/spec/twitter_with_auto_pagination/rest/search_spec.rb +42 -0
  31. data/spec/twitter_with_auto_pagination/rest/timelines_spec.rb +82 -0
  32. data/spec/twitter_with_auto_pagination/rest/users_spec.rb +143 -0
  33. data/twitter_with_auto_pagination.gemspec +1 -1
  34. metadata +28 -3
data/spec/helper.rb CHANGED
@@ -1,68 +1,199 @@
1
+ require 'dotenv/load'
1
2
  require 'twitter_with_auto_pagination'
2
3
  require 'rspec'
3
- require 'stringio'
4
- require 'tempfile'
5
- require 'timecop'
6
- require 'webmock/rspec'
7
-
8
- require_relative 'support/media_object_examples'
9
4
 
10
5
  RSpec.configure do |config|
6
+ config.before(:suite) do
7
+ $fetch_count = $request_count = 0
8
+
9
+ set_trace_func proc { |event, file, line, id, binding, klass|
10
+ if klass == TwitterWithAutoPagination::Cache && id == :fetch && event == 'call'
11
+ $fetch_called = true
12
+ $fetch_count += 1
13
+ end
14
+ if klass == Twitter::REST::Request && id == :perform && event == 'call'
15
+ $request_called = true
16
+ $request_count += 1
17
+ end
18
+ }
19
+ end
20
+
21
+ config.define_derived_metadata do |meta|
22
+ meta[:aggregate_failures] = true unless meta.key?(:aggregate_failures)
23
+ end
24
+
11
25
  config.expect_with :rspec do |c|
12
26
  c.syntax = :expect
13
27
  end
14
28
  end
15
29
 
16
- BASE_URL = 'https://api.twitter.com'.freeze
30
+ RSpec::Matchers.define_negated_matcher :not_change, :change
31
+
32
+ RSpec::Matchers.define :fetch do
33
+ match do |actual|
34
+ before = $fetch_called
35
+ actual.call
36
+ @count = 1 if @count.nil?
37
+ @actual_count = $fetch_count
38
+ result = !before && $fetch_called && @actual_count == @count
39
+
40
+ $fetch_called = false
41
+ $fetch_count = 0
42
+
43
+ result
44
+ end
45
+
46
+ chain(:once) { @count = 1 }
47
+ chain(:twice) { @count = 2 }
48
+ chain(:exactly) { |count| @count = count }
49
+ chain(:times) {}
50
+
51
+ failure_message do |actual|
52
+ "expected to call #fetch #{@count} times but called #{@actual_count} times"
53
+ end
54
+
55
+ supports_block_expectations
56
+ end
57
+
58
+ RSpec::Matchers.define :request do
59
+ match do |actual|
60
+ before = $request_called
61
+ actual.call
62
+ @count = 1 if @count.nil?
63
+ @actual_count = $request_count
64
+ result = !before && $request_called && @actual_count == @count
65
+
66
+ $request_called = false
67
+ $request_count = 0
68
+
69
+ result
70
+ end
71
+
72
+ chain(:once) { @count = 1 }
73
+ chain(:twice) { @count = 2 }
74
+ chain(:exactly) { |count| @count = count }
75
+ chain(:times) {}
76
+
77
+ failure_message do |actual|
78
+ "expected to call #request #{@count} times but called #{@actual_count} times"
79
+ end
80
+
81
+ supports_block_expectations
82
+ end
83
+
84
+ RSpec::Matchers.define :not_fetch do
85
+ match do |actual|
86
+ before = $fetch_called
87
+ actual.call
88
+ after = $fetch_called
89
+ $fetch_called = false
90
+ !before && !after
91
+ end
17
92
 
18
- def a_delete(path)
19
- a_request(:delete, BASE_URL + path)
93
+ failure_message { |actual| 'expected not to fetch' }
94
+ supports_block_expectations
20
95
  end
21
96
 
22
- def a_get(path)
23
- a_request(:get, BASE_URL + path)
97
+ RSpec::Matchers.define :not_request do
98
+ match do |actual|
99
+ before = $request_called
100
+ actual.call
101
+ after = $request_called
102
+ $request_called = false
103
+ !before && !after
104
+ end
105
+
106
+ failure_message { |actual| 'expected not to request' }
107
+ supports_block_expectations
24
108
  end
25
109
 
26
- def a_post(path)
27
- a_request(:post, BASE_URL + path)
110
+ RSpec::Matchers.define :match_twitter do |expected|
111
+ match do |actual|
112
+ if expected.is_a?(Array)
113
+ if expected[0].is_a?(Integer)
114
+ actual == expected
115
+ else
116
+ actual.map { |r| r[:id] } == expected.map { |r| r[:id] }
117
+ end
118
+ elsif expected.is_a?(Hash)
119
+ actual[:id] == expected[:id]
120
+ else
121
+ actual == expected
122
+ end
123
+ end
28
124
  end
29
125
 
30
- def a_put(path)
31
- a_request(:put, BASE_URL + path)
126
+ RSpec.shared_examples 'continuous calls' do
127
+ it 'fetches the result from a cache for the second time' do
128
+ result1 = result2 = nil
129
+ expect { result1 = client.send(name, *params) }.to fetch & request
130
+ expect { result2 = client.send(name, *params) }.to fetch & not_request
131
+ expect(result1).to match_twitter(result2)
132
+ end
32
133
  end
33
134
 
34
- def stub_delete(path)
35
- stub_request(:delete, BASE_URL + path)
135
+ RSpec.shared_examples 'cache: false is specified' do
136
+ it 'sends http requests' do
137
+ expect { client.send(name, *params) }.to fetch & request
138
+ expect { client.send(name, *params, cache: false) }.to not_fetch & request
139
+ end
36
140
  end
37
141
 
38
- def stub_get(path)
39
- stub_request(:get, BASE_URL + path)
142
+ RSpec.shared_examples 'when a value is changed' do
143
+ it 'sends http requests' do
144
+ expect { client.send(name, *params) }.to fetch & request
145
+ expect { client.send(name, *params2) }.to fetch & request
146
+ end
40
147
  end
41
148
 
42
- def stub_post(path)
43
- stub_request(:post, BASE_URL + path)
149
+ RSpec.shared_examples 'when options are changed' do
150
+ it 'sends http requests' do
151
+ expect { client.send(name, *params) }.to fetch & request
152
+ expect { client.send(name, *params, hello: :world) }.to fetch & request
153
+ end
154
+ end
155
+
156
+ RSpec.shared_examples 'when a client is changed, it shares a cache' do
157
+ it 'shares a cache' do
158
+ expect { client.send(name, *params) }.to fetch & request
159
+ expect { client2.send(name, *params) }.to fetch & not_request
160
+ end
161
+ end
162
+
163
+ RSpec.shared_examples 'when a client is changed, it does not share a cache' do
164
+ it 'does not share a cache' do
165
+ expect { client.send(name, *params) }.to fetch & request
166
+ expect { client2.send(name, *params) }.to fetch & request
167
+ end
44
168
  end
45
169
 
46
- def stub_put(path)
47
- stub_request(:put, BASE_URL + path)
170
+ RSpec.shared_examples 'when one param is specified, it raises an exception' do
171
+ it 'raises an exception' do
172
+ expect { client.send(name, id) }.to raise_error(ArgumentError)
173
+ end
48
174
  end
49
175
 
50
- def fixture_path
51
- File.expand_path('../fixtures', __FILE__)
176
+ RSpec.shared_examples 'when any params is not specified, it raises an exception' do
177
+ it 'raises an exception' do
178
+ expect { client.send(name) }.to raise_error(ArgumentError)
179
+ end
52
180
  end
53
181
 
54
- def fixture(file)
55
- File.new(fixture_path + '/' + file)
182
+ RSpec.shared_examples 'when any params is not specified, it returns a same result as a result with one param' do
183
+ it 'returns a same result as a result with one param' do
184
+ result1 = result2 = nil
185
+ expect { result1 = client.send(name) }.to fetch & request
186
+ expect { result2 = client.send(name, id) }.to fetch & request
187
+ expect(result1).to match_twitter(result2)
188
+ end
56
189
  end
57
190
 
58
- def capture_warning
59
- begin
60
- old_stderr = $stderr
61
- $stderr = StringIO.new
62
- yield
63
- result = $stderr.string
64
- ensure
65
- $stderr = old_stderr
191
+ RSpec.shared_examples 'when count is specified' do |count|
192
+ it 'requests twice' do
193
+ result1 = result2 = nil
194
+ expect { result1 = client.send(name, *params, count: count) }.to fetch & request.twice
195
+ expect { result2 = client.send(name, *params, count: count) }.to fetch & not_request
196
+ expect(result1.size).to be > (count / 2)
197
+ expect(result1).to match_twitter(result2)
66
198
  end
67
- result
68
199
  end
@@ -0,0 +1,71 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::Cache do
4
+ let(:cache) { TwitterWithAutoPagination::Cache.new }
5
+
6
+ # describe '#fetch' do
7
+ #
8
+ # before do
9
+ # cache.client.clear
10
+ # end
11
+ #
12
+ # context 'with a stored value' do
13
+ # before do
14
+ # cache.client.write('key', 1)
15
+ # allow(cache).to receive(:normalize_key).and_return('key')
16
+ # end
17
+ # it 'calls a block' do
18
+ # expect { |b| cache.fetch('anything', 'user', &b)}.to yield_control
19
+ # end
20
+ # end
21
+ #
22
+ # context 'without any value' do
23
+ # before do
24
+ # allow(cache).to receive(:normalize_key).and_return('key')
25
+ # end
26
+ # it 'does not call a block' do
27
+ # expect { |b| cache.fetch('anything', 'user', &b)}.to_not yield_control
28
+ # end
29
+ # end
30
+ # end
31
+
32
+ describe '#normalize_key' do
33
+ it 'returns a key' do
34
+ expect(cache.send(:normalize_key, :search, 'a')).to match(/\Asearch:query:a/)
35
+ expect(cache.send(:normalize_key, :friendship?, %w(a b))).to match(/\Afriendship\?:from:a:to:b/)
36
+ expect(cache.send(:normalize_key, :anything, 1)).to match(/\Aanything:id:/)
37
+ expect(cache.send(:normalize_key, :anything, 'a')).to match(/\Aanything:screen_name:a:options:empty\z/)
38
+ expect(cache.send(:normalize_key, :anything, 'a', {count: 10})).to match(/\Aanything:screen_name:a:options:count:10\z/)
39
+ expect {cache.send(:normalize_key, :anything, nil)}.to raise_error(RuntimeError)
40
+ end
41
+
42
+ end
43
+
44
+ describe '#user_identifier' do
45
+ it 'returns a serialized string' do
46
+ expect(cache.send(:user_identifier, 1)).to eq('id:1')
47
+ expect(cache.send(:user_identifier, 'abc')).to eq('screen_name:abc')
48
+ expect(cache.send(:user_identifier, [1, 2])).to match(/\Aids:2-\w+\z/)
49
+ expect(cache.send(:user_identifier, %w(abc cde fgh))).to match(/\Ascreen_names:3-\w+\z/)
50
+ expect { cache.send(:user_identifier, nil) }.to raise_error(RuntimeError)
51
+ end
52
+ end
53
+
54
+ describe '#options_identifier' do
55
+ it 'returns serialized string' do
56
+ expect(cache.send(:options_identifier, a: 1, b: 2)).to match(/\Aoptions:a:1,b:2\z/)
57
+ end
58
+
59
+ context 'empty options' do
60
+ it 'returns serialized string ends with "empty"' do
61
+ expect(cache.send(:options_identifier, {})).to eq("options:empty")
62
+ end
63
+ end
64
+ end
65
+
66
+ describe '#hexdigest' do
67
+ it 'returns hexdigest' do
68
+ expect(cache.send(:hexdigest, 'hello')).to eq(Digest::MD5.hexdigest('hello'))
69
+ end
70
+ end
71
+ end
@@ -1,150 +1,12 @@
1
1
  require 'helper'
2
2
 
3
3
  describe TwitterWithAutoPagination::Client do
4
- let(:config) {
5
- {
6
- consumer_key: 'CK',
7
- consumer_secret: 'CS',
8
- access_token: 'AT',
9
- access_token_secret: 'ATS',
10
- }
11
- }
12
- let(:client) { TwitterWithAutoPagination::Client.new(config) }
13
-
14
- describe '#user?' do
15
- before do
16
- stub_get('/1.1/users/show.json').with(query: {screen_name: 'sferik'}).to_return(body: fixture('sferik.json'), headers: {content_type: 'application/json; charset=utf-8'})
17
- stub_get('/1.1/users/show.json').with(query: {screen_name: 'pengwynn'}).to_return(body: fixture('not_found.json'), status: 404, headers: {content_type: 'application/json; charset=utf-8'})
18
- end
19
- it 'requests the correct resource' do
20
- client.user?('sferik')
21
- expect(a_get('/1.1/users/show.json').with(query: {screen_name: 'sferik'})).to have_been_made
22
- end
23
- it 'returns true if user exists' do
24
- user = client.user?('sferik')
25
- expect(user).to be true
26
- end
27
- it 'returns false if user does not exist' do
28
- user = client.user?('pengwynn')
29
- expect(user).to be false
30
- end
4
+ let(:client) do
5
+ described_class.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
31
11
  end
32
-
33
- # describe '#initialize' do
34
- # let(:default_call_count) { 0 }
35
- #
36
- # it 'sets call_count to 0' do
37
- # expect(client.call_count).to eq(default_call_count)
38
- # end
39
- #
40
- # context 'without params' do
41
- # end
42
- #
43
- # context 'with params' do
44
- # end
45
- # end
46
- #
47
- # describe '#logger' do
48
- # it 'has logger' do
49
- # expect(client.logger).to be_truthy
50
- # end
51
- # end
52
- #
53
- # describe '#call_old_method' do
54
- # end
55
- #
56
- # describe '#collect_with_max_id' do
57
- # end
58
- #
59
- # describe '#collect_with_cursor' do
60
- # end
61
- #
62
- # describe '#file_cache_key' do
63
- # end
64
- #
65
- # describe '#namespaced_key' do
66
- # end
67
- #
68
- # describe '#encode_json' do
69
- # end
70
- #
71
- # describe '#decode_json' do
72
- # end
73
- #
74
- # describe '#fetch_cache_or_call_api' do
75
- # end
76
- #
77
- # describe '#user_timeline' do
78
- # it 'calls old_user_timeline' do
79
- # expect(client).to receive(:old_user_timeline)
80
- # client.user_timeline
81
- # end
82
- #
83
- # it 'calls collect_with_max_id' do
84
- # expect(client).to receive(:collect_with_max_id)
85
- # client.user_timeline
86
- # end
87
- # end
88
- #
89
- # describe '#user_photos' do
90
- # it 'calls user_timeline' do
91
- # expect(client).to receive(:user_timeline)
92
- # client.user_photos
93
- # end
94
- # end
95
- #
96
- # describe '#friends' do
97
- # it 'calls old_friends' do
98
- # expect(client).to receive(:old_friends)
99
- # client.friends
100
- # end
101
- #
102
- # it 'calls collect_with_cursor' do
103
- # expect(client).to receive(:collect_with_cursor)
104
- # client.friends
105
- # end
106
- # end
107
- #
108
- # describe '#followers' do
109
- # it 'calls old_followers' do
110
- # expect(client).to receive(:old_followers)
111
- # client.followers
112
- # end
113
- #
114
- # it 'calls collect_with_cursor' do
115
- # expect(client).to receive(:collect_with_cursor)
116
- # client.followers
117
- # end
118
- # end
119
- #
120
- # describe '#friend_ids' do
121
- # it 'calls old_friend_ids' do
122
- # expect(client).to receive(:old_friend_ids)
123
- # client.friend_ids
124
- # end
125
- #
126
- # it 'calls collect_with_cursor' do
127
- # expect(client).to receive(:collect_with_cursor)
128
- # client.friend_ids
129
- # end
130
- # end
131
- #
132
- # describe '#follower_ids' do
133
- # it 'calls old_follower_ids' do
134
- # expect(client).to receive(:old_follower_ids)
135
- # client.follower_ids
136
- # end
137
- #
138
- # it 'calls collect_with_cursor' do
139
- # expect(client).to receive(:collect_with_cursor)
140
- # client.follower_ids
141
- # end
142
- # end
143
- #
144
- # describe '#users' do
145
- # it 'calls old_users' do
146
- # expect(client).to receive(:old_users)
147
- # client.users([1, 2, 3])
148
- # end
149
- # end
150
12
  end
@@ -0,0 +1,23 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::Parallel do
4
+ let(:client) do
5
+ TwitterWithAutoPagination::Client.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
11
+ end
12
+
13
+ let(:id) { 58135830 }
14
+
15
+ describe '#parallel' do
16
+ it 'calls #users' do
17
+ expect(client).to receive(:users).with([id], any_args)
18
+ client.parallel do |batch|
19
+ batch.users([id])
20
+ end
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,58 @@
1
+ require 'helper'
2
+
3
+ describe TwitterWithAutoPagination::REST::Favorites do
4
+ let(:client) do
5
+ TwitterWithAutoPagination::Client.new(
6
+ consumer_key: ENV['CK'],
7
+ consumer_secret: ENV['CS'],
8
+ access_token: ENV['AT'],
9
+ access_token_secret: ENV['ATS']
10
+ )
11
+ end
12
+
13
+ let(:client2) do
14
+ TwitterWithAutoPagination::Client.new(
15
+ consumer_key: ENV['CK2'],
16
+ consumer_secret: ENV['CS2'],
17
+ access_token: ENV['AT2'],
18
+ access_token_secret: ENV['ATS2']
19
+ )
20
+ end
21
+
22
+ let(:id) { 58135830 }
23
+ let(:id2) { 22356250 }
24
+
25
+ before do
26
+ client.cache.clear
27
+ $fetch_called = $request_called = false
28
+ $fetch_count = $request_count = 0
29
+ end
30
+
31
+ describe '#favorites' do
32
+ let(:name) { :favorites }
33
+
34
+ context 'with one param' do
35
+ let(:params) { [id] }
36
+ let(:params2) { [id2] }
37
+
38
+ it_behaves_like 'continuous calls'
39
+ it_behaves_like 'cache: false is specified'
40
+ it_behaves_like 'when a value is changed'
41
+ it_behaves_like 'when options are changed'
42
+ it_behaves_like 'when a client is changed, it shares a cache'
43
+ it_behaves_like 'when count is specified', 200
44
+ end
45
+
46
+ context 'with no params' do
47
+ let(:params) { [] }
48
+
49
+ it_behaves_like 'continuous calls'
50
+ it_behaves_like 'cache: false is specified'
51
+ it_behaves_like 'when options are changed'
52
+ it_behaves_like 'when a client is changed, it does not share a cache'
53
+ it_behaves_like 'when count is specified', 200
54
+ end
55
+
56
+ it_behaves_like 'when any params is not specified, it returns a same result as a result with one param'
57
+ end
58
+ end