twitter_with_auto_pagination 0.8.4 → 0.8.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/twitter_with_auto_pagination/log_subscriber.rb +25 -50
- data/lib/twitter_with_auto_pagination/rest/extension/clusters.rb +23 -7
- data/lib/twitter_with_auto_pagination/rest/favorites.rb +3 -4
- data/lib/twitter_with_auto_pagination/rest/friends_and_followers.rb +27 -21
- data/lib/twitter_with_auto_pagination/rest/lists.rb +2 -2
- data/lib/twitter_with_auto_pagination/rest/timelines.rb +14 -11
- data/lib/twitter_with_auto_pagination/rest/users.rb +3 -3
- data/lib/twitter_with_auto_pagination/rest/utils.rb +52 -83
- data/lib/twitter_with_auto_pagination.rb +3 -4
- data/twitter_with_auto_pagination.gemspec +1 -2
- metadata +2 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fc7f7eedec47178f6928767006f494d6f21a927
|
4
|
+
data.tar.gz: e63a240fed3bdf083405337a0f3e619c88d527fb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ee927c273eb2d858da46704ca57ac4796fbeff8ba9ddb340aaf9b6ab02413667dadc5608c13bc50b4f2948021e3c13676f965db8a64e24802998d65127b3bd4
|
7
|
+
data.tar.gz: a259c921a5b17090001381223a3f14ff54c2467dca7a343d0bec616c13e4739f243cdbe566de21b7b2d0d7e1c98084d8284254e3af4d7002f5964d0ec178d4a8
|
@@ -1,21 +1,38 @@
|
|
1
|
-
require 'active_support'
|
2
|
-
require 'active_support/core_ext'
|
3
|
-
|
4
|
-
|
5
1
|
module TwitterWithAutoPagination
|
6
2
|
class LogSubscriber < ActiveSupport::LogSubscriber
|
7
3
|
|
8
4
|
def initialize
|
9
5
|
super
|
10
|
-
|
6
|
+
end
|
7
|
+
|
8
|
+
def call(event)
|
9
|
+
return unless logger.debug?
|
10
|
+
|
11
|
+
payload = event.payload
|
12
|
+
name = "TW::#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
|
13
|
+
name = color(name, CYAN, true) # WHITE, RED, GREEN, BLUE, CYAN, MAGENTA, YELLOW
|
14
|
+
debug { "#{name} #{(payload.inspect)}" }
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def logger
|
20
|
+
Twitter::REST::Client.logger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class ASLogSubscriber < ActiveSupport::LogSubscriber
|
25
|
+
|
26
|
+
def initialize
|
27
|
+
super
|
11
28
|
end
|
12
29
|
|
13
30
|
def cache_any(event)
|
14
31
|
return unless logger.debug?
|
15
32
|
|
16
33
|
payload = event.payload
|
17
|
-
name
|
18
|
-
name =
|
34
|
+
name= "AS::#{payload.delete(:name)} (#{event.duration.round(1)}ms)"
|
35
|
+
name = color(name, MAGENTA, true)
|
19
36
|
debug { "#{name} #{(payload.inspect)}" }
|
20
37
|
end
|
21
38
|
|
@@ -28,50 +45,8 @@ module TwitterWithAutoPagination
|
|
28
45
|
METHOD
|
29
46
|
end
|
30
47
|
|
31
|
-
def call(event)
|
32
|
-
return unless logger.debug?
|
33
|
-
|
34
|
-
payload = event.payload
|
35
|
-
name = "#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
|
36
|
-
|
37
|
-
name = colorize_payload_name(name, payload[:name])
|
38
|
-
# sql = color(sql, sql_color(sql), true)
|
39
|
-
|
40
|
-
key = payload.delete(:key)
|
41
|
-
debug { "#{name}#{key.nil? ? '' : " #{key}"} #{(payload.inspect)}" }
|
42
|
-
end
|
43
|
-
|
44
48
|
private
|
45
49
|
|
46
|
-
def colorize_payload_name(name, payload_name, options = {})
|
47
|
-
if options[:AS]
|
48
|
-
color(name, MAGENTA, true)
|
49
|
-
else
|
50
|
-
color(name, CYAN, true)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def sql_color(sql)
|
55
|
-
case sql
|
56
|
-
when /\A\s*rollback/mi
|
57
|
-
RED
|
58
|
-
when /select .*for update/mi, /\A\s*lock/mi
|
59
|
-
WHITE
|
60
|
-
when /\A\s*select/i
|
61
|
-
BLUE
|
62
|
-
when /\A\s*insert/i
|
63
|
-
GREEN
|
64
|
-
when /\A\s*update/i
|
65
|
-
YELLOW
|
66
|
-
when /\A\s*delete/i
|
67
|
-
RED
|
68
|
-
when /transaction\s*\Z/i
|
69
|
-
CYAN
|
70
|
-
else
|
71
|
-
MAGENTA
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
50
|
def logger
|
76
51
|
Twitter::REST::Client.logger
|
77
52
|
end
|
@@ -79,4 +54,4 @@ module TwitterWithAutoPagination
|
|
79
54
|
end
|
80
55
|
|
81
56
|
TwitterWithAutoPagination::LogSubscriber.attach_to :twitter_with_auto_pagination
|
82
|
-
TwitterWithAutoPagination::
|
57
|
+
TwitterWithAutoPagination::ASLogSubscriber.attach_to :active_support
|
@@ -6,9 +6,9 @@ module TwitterWithAutoPagination
|
|
6
6
|
module Clusters
|
7
7
|
include TwitterWithAutoPagination::REST::Utils
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
def tweet_clusters(tweets, limit: 10)
|
10
|
+
return {} if tweets.blank?
|
11
|
+
text = tweets.map(&:text).join(' ')
|
12
12
|
|
13
13
|
if defined?(Rails)
|
14
14
|
exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
|
@@ -40,12 +40,30 @@ module TwitterWithAutoPagination
|
|
40
40
|
each { |w| frequency[w] += 1 }
|
41
41
|
|
42
42
|
# 複数個以上見付かった単語のみを残し、出現頻度順にソート
|
43
|
-
frequency.select { |_, v| 2 < v }.sort_by { |
|
43
|
+
frequency.select { |_, v| 2 < v }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit).to_h
|
44
44
|
end
|
45
45
|
|
46
|
-
|
46
|
+
def hashtag_clusters(tweets, limit: 10, debug: false)
|
47
|
+
puts "tweets: #{tweets.size}" if debug
|
48
|
+
return {} if tweets.blank?
|
49
|
+
|
50
|
+
tweets = tweets.select { |t| t.text && t.text.include?('#') }
|
51
|
+
puts "tweets with hashtag: #{tweets.size}" if debug
|
52
|
+
|
53
|
+
hashtags = tweets.map { |t| t.text.scan(/[##][A-Za-zA-Za-z一-鿆0-90-9ぁ-ヶヲ-゚ー]+/).map(&:strip) }.flatten
|
54
|
+
puts "hashtags: #{hashtags.size}" if debug
|
55
|
+
|
56
|
+
hashtags.each_with_object(Hash.new(0)) { |h, memo| memo[h] += 1 }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit).to_h
|
57
|
+
end
|
47
58
|
|
48
59
|
def list_clusters(user, shrink: false, each_member: 300, total_member: 1000, rate: 0.3, limit: 10, debug: false)
|
60
|
+
begin
|
61
|
+
require 'mecab'
|
62
|
+
rescue => e
|
63
|
+
puts "Add gem 'mecab' to your Gemfile."
|
64
|
+
return nil
|
65
|
+
end
|
66
|
+
|
49
67
|
begin
|
50
68
|
lists = memberships(user).sort_by { |li| li.member_count }
|
51
69
|
rescue => e
|
@@ -141,9 +159,7 @@ module TwitterWithAutoPagination
|
|
141
159
|
pipe_freq = count_by_word(candidates, delim: '|')
|
142
160
|
puts "words splitted by |: #{pipe_freq.to_a.slice(0, 10)}" if debug
|
143
161
|
|
144
|
-
require 'mecab'
|
145
162
|
tagger = MeCab::Tagger.new("-d #{`mecab-config --dicdir`.chomp}/mecab-ipadic-neologd/")
|
146
|
-
|
147
163
|
noun_freq = count_by_word(remains, tagger: tagger, exclude_words: profile_exclude_words)
|
148
164
|
puts "words tagged as noun: #{noun_freq.to_a.slice(0, 10)}" if debug
|
149
165
|
|
@@ -6,9 +6,8 @@ module TwitterWithAutoPagination
|
|
6
6
|
include TwitterWithAutoPagination::REST::Utils
|
7
7
|
|
8
8
|
def favorites(*args)
|
9
|
-
|
10
|
-
|
11
|
-
args[0] = verify_credentials.id if args.empty?
|
9
|
+
options = {count: 100, call_limit: 3}.merge(args.extract_options!)
|
10
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
12
11
|
instrument(__method__, nil, options) do
|
13
12
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
14
13
|
collect_with_max_id(method(__method__).super_method, *args, options).map { |s| s.attrs }
|
@@ -17,4 +16,4 @@ module TwitterWithAutoPagination
|
|
17
16
|
end
|
18
17
|
end
|
19
18
|
end
|
20
|
-
end
|
19
|
+
end
|
@@ -16,7 +16,7 @@ module TwitterWithAutoPagination
|
|
16
16
|
|
17
17
|
def friend_ids(*args)
|
18
18
|
options = {count: 5000, cursor: -1}.merge(args.extract_options!)
|
19
|
-
args[0] = verify_credentials.id if args.empty?
|
19
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
20
20
|
instrument(__method__, nil, options) do
|
21
21
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
22
22
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -26,7 +26,7 @@ module TwitterWithAutoPagination
|
|
26
26
|
|
27
27
|
def follower_ids(*args)
|
28
28
|
options = {count: 5000, cursor: -1}.merge(args.extract_options!)
|
29
|
-
args[0] = verify_credentials.id if args.empty?
|
29
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
30
30
|
instrument(__method__, nil, options) do
|
31
31
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
32
32
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -37,16 +37,18 @@ module TwitterWithAutoPagination
|
|
37
37
|
# specify reduce: false to use tweet for inactive_*
|
38
38
|
def friends(*args)
|
39
39
|
options = args.extract_options!
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
40
|
+
instrument(__method__, nil, options) do
|
41
|
+
if options.delete(:serial)
|
42
|
+
_friends_serially(*args, options)
|
43
|
+
else
|
44
|
+
_friends_parallelly(*args, options)
|
45
|
+
end
|
44
46
|
end
|
45
47
|
end
|
46
48
|
|
47
49
|
def _friends_serially(*args)
|
48
|
-
options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
|
49
|
-
args[0] = verify_credentials.id if args.empty?
|
50
|
+
options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :friends}.merge(args.extract_options!)
|
51
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
50
52
|
instrument(__method__, nil, options) do
|
51
53
|
fetch_cache_or_call_api(:friends, args[0], options) do
|
52
54
|
collect_with_cursor(method(:friends).super_method, *args, options).map { |u| u.to_hash }
|
@@ -55,25 +57,28 @@ module TwitterWithAutoPagination
|
|
55
57
|
end
|
56
58
|
|
57
59
|
def _friends_parallelly(*args)
|
58
|
-
options =
|
59
|
-
instrument(__method__, nil, options) do
|
60
|
-
|
60
|
+
options = args.extract_options!
|
61
|
+
instrument(__method__, nil, {super_operation: :friends}.merge(options)) do
|
62
|
+
opt = {super_operation: __method__}.merge(options)
|
63
|
+
users(friend_ids(*args, opt).map { |id| id.to_i }, opt)
|
61
64
|
end
|
62
65
|
end
|
63
66
|
|
64
67
|
# specify reduce: false to use tweet for inactive_*
|
65
68
|
def followers(*args)
|
66
69
|
options = args.extract_options!
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
70
|
+
instrument(__method__, nil, options) do
|
71
|
+
if options.delete(:serial)
|
72
|
+
_followers_serially(*args, options)
|
73
|
+
else
|
74
|
+
_followers_parallelly(*args, options)
|
75
|
+
end
|
71
76
|
end
|
72
77
|
end
|
73
78
|
|
74
79
|
def _followers_serially(*args)
|
75
|
-
options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
|
76
|
-
args[0] = verify_credentials.id if args.empty?
|
80
|
+
options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :followers}.merge(args.extract_options!)
|
81
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
77
82
|
instrument(__method__, nil, options) do
|
78
83
|
fetch_cache_or_call_api(:followers, args[0], options) do
|
79
84
|
collect_with_cursor(method(:followers).super_method, *args, options).map { |u| u.to_hash }
|
@@ -82,11 +87,12 @@ module TwitterWithAutoPagination
|
|
82
87
|
end
|
83
88
|
|
84
89
|
def _followers_parallelly(*args)
|
85
|
-
options =
|
86
|
-
instrument(__method__, nil, options) do
|
87
|
-
|
90
|
+
options = args.extract_options!
|
91
|
+
instrument(__method__, nil, {super_operation: :followers}.merge(options)) do
|
92
|
+
opt = {super_operation: __method__}.merge(options)
|
93
|
+
users(follower_ids(*args, opt).map { |id| id.to_i }, opt)
|
88
94
|
end
|
89
95
|
end
|
90
96
|
end
|
91
97
|
end
|
92
|
-
end
|
98
|
+
end
|
@@ -7,7 +7,7 @@ module TwitterWithAutoPagination
|
|
7
7
|
|
8
8
|
def memberships(*args)
|
9
9
|
options = {count: 1000, cursor: -1}.merge(args.extract_options!)
|
10
|
-
args[0] = verify_credentials.id if args.empty?
|
10
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
11
11
|
instrument(__method__, nil, options) do
|
12
12
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
13
13
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -25,4 +25,4 @@ module TwitterWithAutoPagination
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
end
|
28
|
-
end
|
28
|
+
end
|
@@ -6,32 +6,35 @@ module TwitterWithAutoPagination
|
|
6
6
|
include TwitterWithAutoPagination::REST::Utils
|
7
7
|
|
8
8
|
def home_timeline(*args)
|
9
|
+
mtd = __method__
|
9
10
|
options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
|
10
|
-
instrument(
|
11
|
-
fetch_cache_or_call_api(
|
12
|
-
collect_with_max_id(method(
|
11
|
+
instrument(mtd, nil, options) do
|
12
|
+
fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
|
13
|
+
collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
|
13
14
|
end
|
14
15
|
end
|
15
16
|
end
|
16
17
|
|
17
18
|
def user_timeline(*args)
|
19
|
+
mtd = __method__
|
18
20
|
options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
|
19
|
-
args[0] = verify_credentials.id if args.empty?
|
20
|
-
instrument(
|
21
|
-
fetch_cache_or_call_api(
|
22
|
-
collect_with_max_id(method(
|
21
|
+
args[0] = verify_credentials(super_operation: mtd).id if args.empty?
|
22
|
+
instrument(mtd, nil, options) do
|
23
|
+
fetch_cache_or_call_api(mtd, args[0], options) do
|
24
|
+
collect_with_max_id(method(mtd).super_method, *args, options).map { |s| s.attrs }
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
26
28
|
|
27
29
|
def mentions_timeline(*args)
|
30
|
+
mtd = __method__
|
28
31
|
options = {count: 200, include_rts: true, call_limit: 1}.merge(args.extract_options!)
|
29
|
-
instrument(
|
30
|
-
fetch_cache_or_call_api(
|
31
|
-
collect_with_max_id(method(
|
32
|
+
instrument(mtd, nil, options) do
|
33
|
+
fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
|
34
|
+
collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
|
32
35
|
end
|
33
36
|
end
|
34
37
|
end
|
35
38
|
end
|
36
39
|
end
|
37
|
-
end
|
40
|
+
end
|
@@ -17,7 +17,7 @@ module TwitterWithAutoPagination
|
|
17
17
|
|
18
18
|
def user?(*args)
|
19
19
|
options = args.extract_options!
|
20
|
-
args[0]
|
20
|
+
return false if args.empty? || args[0].nil? || !args[0].to_s.match(/\A([a-zA-Z0-9_]{1,20}|[1-9][0-9]*)\z/)
|
21
21
|
instrument(__method__, nil, options) do
|
22
22
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
23
23
|
call_api(method(__method__).super_method, *args, options)
|
@@ -27,7 +27,7 @@ module TwitterWithAutoPagination
|
|
27
27
|
|
28
28
|
def user(*args)
|
29
29
|
options = args.extract_options!
|
30
|
-
args[0] = verify_credentials.id if args.empty?
|
30
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
31
31
|
instrument(__method__, nil, options) do
|
32
32
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
33
33
|
call_api(method(__method__).super_method, *args, options).to_hash
|
@@ -59,4 +59,4 @@ module TwitterWithAutoPagination
|
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
62
|
-
end
|
62
|
+
end
|
@@ -4,37 +4,6 @@ require 'digest/md5'
|
|
4
4
|
module TwitterWithAutoPagination
|
5
5
|
module REST
|
6
6
|
module Utils
|
7
|
-
# for backward compatibility
|
8
|
-
def uid
|
9
|
-
@uid || user.id.to_i
|
10
|
-
end
|
11
|
-
|
12
|
-
def __uid
|
13
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
14
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
15
|
-
MESSAGE
|
16
|
-
uid
|
17
|
-
end
|
18
|
-
|
19
|
-
def __uid_i
|
20
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
21
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
22
|
-
MESSAGE
|
23
|
-
uid
|
24
|
-
end
|
25
|
-
|
26
|
-
# for backward compatibility
|
27
|
-
def screen_name
|
28
|
-
@screen_name || user.screen_name
|
29
|
-
end
|
30
|
-
|
31
|
-
def __screen_name
|
32
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
33
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
34
|
-
MESSAGE
|
35
|
-
screen_name
|
36
|
-
end
|
37
|
-
|
38
7
|
def uid_or_screen_name?(object)
|
39
8
|
object.kind_of?(String) || object.kind_of?(Integer)
|
40
9
|
end
|
@@ -49,23 +18,21 @@ module TwitterWithAutoPagination
|
|
49
18
|
end
|
50
19
|
|
51
20
|
def credentials_hash
|
52
|
-
|
53
|
-
Digest::MD5.hexdigest(str)
|
21
|
+
Digest::MD5.hexdigest(access_token + access_token_secret + consumer_key + consumer_secret)
|
54
22
|
end
|
55
23
|
|
56
24
|
def instrument(operation, key, options = nil)
|
57
|
-
payload = {operation: operation
|
25
|
+
payload = {operation: operation}
|
58
26
|
payload.merge!(options) if options.is_a?(Hash)
|
59
27
|
ActiveSupport::Notifications.instrument('call.twitter_with_auto_pagination', payload) { yield(payload) }
|
60
28
|
end
|
61
29
|
|
62
|
-
def call_api(
|
30
|
+
def call_api(method, *args)
|
63
31
|
api_options = args.extract_options!
|
32
|
+
self.call_count += 1
|
33
|
+
options = {method: method.name, call_count: self.call_count, args: [*args, api_options]}
|
64
34
|
begin
|
65
|
-
|
66
|
-
# TODO call without reduce, call_count
|
67
|
-
options = {method_name: method_obj.name, call_count: self.call_count, args: [*args, api_options]}
|
68
|
-
instrument('request', args[0], options) { method_obj.call(*args, api_options) }
|
35
|
+
instrument('request', nil, options) { method.call(*args, api_options) }
|
69
36
|
rescue Twitter::Error::TooManyRequests => e
|
70
37
|
logger.warn "#{__method__}: #{options.inspect} #{e.class} Retry after #{e.rate_limit.reset_in} seconds."
|
71
38
|
raise e
|
@@ -74,68 +41,74 @@ module TwitterWithAutoPagination
|
|
74
41
|
logger.warn "#{__method__}: #{options.inspect} #{e.class} #{e.message}"
|
75
42
|
raise e
|
76
43
|
rescue => e
|
77
|
-
logger.warn "
|
44
|
+
logger.warn "CATCH ME! #{__method__}: #{options.inspect} #{e.class} #{e.message}"
|
78
45
|
raise e
|
79
46
|
end
|
80
47
|
end
|
81
48
|
|
82
49
|
# user_timeline, search
|
83
|
-
def collect_with_max_id(
|
50
|
+
def collect_with_max_id(method, *args)
|
84
51
|
options = args.extract_options!
|
85
52
|
call_limit = options.delete(:call_limit) || 3
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
while last_response.any? && call_count < call_limit
|
92
|
-
options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
|
93
|
-
last_response = call_api(method_obj, *args, options)
|
53
|
+
return_data = []
|
54
|
+
call_num = 0
|
55
|
+
|
56
|
+
while call_num < call_limit
|
57
|
+
last_response = call_api(method, *args, options)
|
94
58
|
last_response = yield(last_response) if block_given?
|
59
|
+
call_num += 1
|
95
60
|
return_data += last_response
|
96
|
-
|
61
|
+
if last_response.nil? || last_response.empty?
|
62
|
+
break
|
63
|
+
else
|
64
|
+
options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
|
65
|
+
end
|
97
66
|
end
|
98
67
|
|
99
|
-
return_data
|
68
|
+
return_data
|
100
69
|
end
|
101
70
|
|
102
71
|
# friends, followers
|
103
|
-
def collect_with_cursor(
|
72
|
+
def collect_with_cursor(method, *args)
|
104
73
|
options = args.extract_options!
|
105
|
-
|
106
|
-
|
74
|
+
return_data = []
|
75
|
+
call_num = 0
|
107
76
|
|
108
|
-
while
|
109
|
-
|
110
|
-
|
77
|
+
while call_num < 30
|
78
|
+
last_response = call_api(method, *args, options).attrs
|
79
|
+
call_num += 1
|
111
80
|
return_data += (last_response[:users] || last_response[:ids] || last_response[:lists])
|
81
|
+
options[:cursor] = last_response[:next_cursor]
|
82
|
+
if options[:cursor].nil? || options[:cursor] == 0
|
83
|
+
break
|
84
|
+
end
|
112
85
|
end
|
113
86
|
|
114
87
|
return_data
|
115
88
|
end
|
116
89
|
|
117
|
-
def
|
90
|
+
def normalize_key(method, user, options = {})
|
118
91
|
delim = ':'
|
119
92
|
identifier =
|
120
93
|
case
|
121
|
-
when
|
122
|
-
"hash
|
123
|
-
when
|
94
|
+
when method == :verify_credentials
|
95
|
+
"token-hash#{delim}#{credentials_hash}"
|
96
|
+
when method == :search
|
124
97
|
"str#{delim}#{user.to_s}"
|
125
|
-
when
|
98
|
+
when method == :list_members
|
126
99
|
"list_id#{delim}#{user.to_s}"
|
127
|
-
when
|
100
|
+
when method == :mentions_timeline
|
128
101
|
"#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
|
129
|
-
when
|
102
|
+
when method == :home_timeline
|
130
103
|
"#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
|
131
|
-
when
|
104
|
+
when method.in?([:users, :replying]) && options[:super_operation].present?
|
132
105
|
case
|
133
106
|
when user.kind_of?(Array) && user.first.kind_of?(Integer)
|
134
107
|
"#{options[:super_operation]}-ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
|
135
108
|
when user.kind_of?(Array) && user.first.kind_of?(String)
|
136
109
|
"#{options[:super_operation]}-sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
|
137
110
|
else
|
138
|
-
raise "#{
|
111
|
+
raise "#{method.inspect} #{user.inspect}"
|
139
112
|
end
|
140
113
|
when user.kind_of?(Integer)
|
141
114
|
"id#{delim}#{user.to_s}"
|
@@ -148,10 +121,10 @@ module TwitterWithAutoPagination
|
|
148
121
|
when user.kind_of?(Twitter::User)
|
149
122
|
"user#{delim}#{user.id.to_s}"
|
150
123
|
else
|
151
|
-
raise "#{
|
124
|
+
raise "#{method.inspect} #{user.inspect}"
|
152
125
|
end
|
153
126
|
|
154
|
-
"#{
|
127
|
+
"#{method}#{delim}#{identifier}"
|
155
128
|
end
|
156
129
|
|
157
130
|
CODER = JSON
|
@@ -162,35 +135,31 @@ module TwitterWithAutoPagination
|
|
162
135
|
|
163
136
|
def decode(str)
|
164
137
|
obj = str.kind_of?(String) ? CODER.load(str) : str
|
165
|
-
|
138
|
+
to_mash(obj)
|
166
139
|
end
|
167
140
|
|
168
|
-
def
|
141
|
+
def to_mash(obj)
|
169
142
|
case
|
170
143
|
when obj.kind_of?(Array)
|
171
|
-
obj.map { |o|
|
144
|
+
obj.map { |o| to_mash(o) }
|
172
145
|
when obj.kind_of?(Hash)
|
173
|
-
Hashie::Mash.new(obj.map { |k, v| [k,
|
146
|
+
Hashie::Mash.new(obj.map { |k, v| [k, to_mash(v)] }.to_h)
|
174
147
|
else
|
175
148
|
obj
|
176
149
|
end
|
177
150
|
end
|
178
151
|
|
179
|
-
def fetch_cache_or_call_api(
|
180
|
-
key =
|
152
|
+
def fetch_cache_or_call_api(method, user, options = {})
|
153
|
+
key = normalize_key(method, user, options)
|
181
154
|
|
182
155
|
fetch_result =
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
|
187
|
-
block_result = yield
|
188
|
-
instrument('serialize', key, caller: method_name) { encode(block_result) }
|
189
|
-
end
|
156
|
+
cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
|
157
|
+
block_result = yield
|
158
|
+
instrument('serialize', nil, key: key, caller: method) { encode(block_result) }
|
190
159
|
end
|
191
160
|
|
192
|
-
instrument('deserialize', key, caller:
|
161
|
+
instrument('deserialize', nil, key: key, caller: method) { decode(fetch_result) }
|
193
162
|
end
|
194
163
|
end
|
195
164
|
end
|
196
|
-
end
|
165
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
require 'active_support/core_ext'
|
1
3
|
require 'twitter'
|
2
4
|
|
3
5
|
require 'twitter_with_auto_pagination/log_subscriber'
|
@@ -16,9 +18,6 @@ module Twitter
|
|
16
18
|
@cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache'))
|
17
19
|
@call_count = 0
|
18
20
|
|
19
|
-
@uid = options.has_key?(:uid) ? options.delete(:uid).to_i : nil
|
20
|
-
@screen_name = options.has_key?(:screen_name) ? options.delete(:screen_name).to_s : nil
|
21
|
-
|
22
21
|
logger =
|
23
22
|
if options.has_key?(:logger)
|
24
23
|
options.delete(:logger)
|
@@ -42,4 +41,4 @@ module Twitter
|
|
42
41
|
INDENT = 4
|
43
42
|
end
|
44
43
|
end
|
45
|
-
end
|
44
|
+
end
|
@@ -6,7 +6,6 @@ Gem::Specification.new do |spec|
|
|
6
6
|
spec.add_dependency 'activesupport'
|
7
7
|
spec.add_dependency 'hashie'
|
8
8
|
spec.add_dependency 'parallel'
|
9
|
-
spec.add_dependency 'mecab'
|
10
9
|
|
11
10
|
spec.add_development_dependency 'bundler'
|
12
11
|
|
@@ -23,5 +22,5 @@ Gem::Specification.new do |spec|
|
|
23
22
|
spec.required_ruby_version = '>= 2.3'
|
24
23
|
spec.summary = spec.description
|
25
24
|
spec.test_files = Dir.glob('spec/**/*')
|
26
|
-
spec.version = '0.8.
|
25
|
+
spec.version = '0.8.5'
|
27
26
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twitter_with_auto_pagination
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinohara Teruki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: twitter
|
@@ -66,20 +66,6 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: mecab
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
70
|
name: bundler
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|