twitter_with_auto_pagination 0.8.4 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/twitter_with_auto_pagination/log_subscriber.rb +25 -50
- data/lib/twitter_with_auto_pagination/rest/extension/clusters.rb +23 -7
- data/lib/twitter_with_auto_pagination/rest/favorites.rb +3 -4
- data/lib/twitter_with_auto_pagination/rest/friends_and_followers.rb +27 -21
- data/lib/twitter_with_auto_pagination/rest/lists.rb +2 -2
- data/lib/twitter_with_auto_pagination/rest/timelines.rb +14 -11
- data/lib/twitter_with_auto_pagination/rest/users.rb +3 -3
- data/lib/twitter_with_auto_pagination/rest/utils.rb +52 -83
- data/lib/twitter_with_auto_pagination.rb +3 -4
- data/twitter_with_auto_pagination.gemspec +1 -2
- metadata +2 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fc7f7eedec47178f6928767006f494d6f21a927
|
4
|
+
data.tar.gz: e63a240fed3bdf083405337a0f3e619c88d527fb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ee927c273eb2d858da46704ca57ac4796fbeff8ba9ddb340aaf9b6ab02413667dadc5608c13bc50b4f2948021e3c13676f965db8a64e24802998d65127b3bd4
|
7
|
+
data.tar.gz: a259c921a5b17090001381223a3f14ff54c2467dca7a343d0bec616c13e4739f243cdbe566de21b7b2d0d7e1c98084d8284254e3af4d7002f5964d0ec178d4a8
|
@@ -1,21 +1,38 @@
|
|
1
|
-
require 'active_support'
|
2
|
-
require 'active_support/core_ext'
|
3
|
-
|
4
|
-
|
5
1
|
module TwitterWithAutoPagination
|
6
2
|
class LogSubscriber < ActiveSupport::LogSubscriber
|
7
3
|
|
8
4
|
def initialize
|
9
5
|
super
|
10
|
-
|
6
|
+
end
|
7
|
+
|
8
|
+
def call(event)
|
9
|
+
return unless logger.debug?
|
10
|
+
|
11
|
+
payload = event.payload
|
12
|
+
name = "TW::#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
|
13
|
+
name = color(name, CYAN, true) # WHITE, RED, GREEN, BLUE, CYAN, MAGENTA, YELLOW
|
14
|
+
debug { "#{name} #{(payload.inspect)}" }
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def logger
|
20
|
+
Twitter::REST::Client.logger
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class ASLogSubscriber < ActiveSupport::LogSubscriber
|
25
|
+
|
26
|
+
def initialize
|
27
|
+
super
|
11
28
|
end
|
12
29
|
|
13
30
|
def cache_any(event)
|
14
31
|
return unless logger.debug?
|
15
32
|
|
16
33
|
payload = event.payload
|
17
|
-
name
|
18
|
-
name =
|
34
|
+
name= "AS::#{payload.delete(:name)} (#{event.duration.round(1)}ms)"
|
35
|
+
name = color(name, MAGENTA, true)
|
19
36
|
debug { "#{name} #{(payload.inspect)}" }
|
20
37
|
end
|
21
38
|
|
@@ -28,50 +45,8 @@ module TwitterWithAutoPagination
|
|
28
45
|
METHOD
|
29
46
|
end
|
30
47
|
|
31
|
-
def call(event)
|
32
|
-
return unless logger.debug?
|
33
|
-
|
34
|
-
payload = event.payload
|
35
|
-
name = "#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
|
36
|
-
|
37
|
-
name = colorize_payload_name(name, payload[:name])
|
38
|
-
# sql = color(sql, sql_color(sql), true)
|
39
|
-
|
40
|
-
key = payload.delete(:key)
|
41
|
-
debug { "#{name}#{key.nil? ? '' : " #{key}"} #{(payload.inspect)}" }
|
42
|
-
end
|
43
|
-
|
44
48
|
private
|
45
49
|
|
46
|
-
def colorize_payload_name(name, payload_name, options = {})
|
47
|
-
if options[:AS]
|
48
|
-
color(name, MAGENTA, true)
|
49
|
-
else
|
50
|
-
color(name, CYAN, true)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def sql_color(sql)
|
55
|
-
case sql
|
56
|
-
when /\A\s*rollback/mi
|
57
|
-
RED
|
58
|
-
when /select .*for update/mi, /\A\s*lock/mi
|
59
|
-
WHITE
|
60
|
-
when /\A\s*select/i
|
61
|
-
BLUE
|
62
|
-
when /\A\s*insert/i
|
63
|
-
GREEN
|
64
|
-
when /\A\s*update/i
|
65
|
-
YELLOW
|
66
|
-
when /\A\s*delete/i
|
67
|
-
RED
|
68
|
-
when /transaction\s*\Z/i
|
69
|
-
CYAN
|
70
|
-
else
|
71
|
-
MAGENTA
|
72
|
-
end
|
73
|
-
end
|
74
|
-
|
75
50
|
def logger
|
76
51
|
Twitter::REST::Client.logger
|
77
52
|
end
|
@@ -79,4 +54,4 @@ module TwitterWithAutoPagination
|
|
79
54
|
end
|
80
55
|
|
81
56
|
TwitterWithAutoPagination::LogSubscriber.attach_to :twitter_with_auto_pagination
|
82
|
-
TwitterWithAutoPagination::
|
57
|
+
TwitterWithAutoPagination::ASLogSubscriber.attach_to :active_support
|
@@ -6,9 +6,9 @@ module TwitterWithAutoPagination
|
|
6
6
|
module Clusters
|
7
7
|
include TwitterWithAutoPagination::REST::Utils
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
def tweet_clusters(tweets, limit: 10)
|
10
|
+
return {} if tweets.blank?
|
11
|
+
text = tweets.map(&:text).join(' ')
|
12
12
|
|
13
13
|
if defined?(Rails)
|
14
14
|
exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
|
@@ -40,12 +40,30 @@ module TwitterWithAutoPagination
|
|
40
40
|
each { |w| frequency[w] += 1 }
|
41
41
|
|
42
42
|
# 複数個以上見付かった単語のみを残し、出現頻度順にソート
|
43
|
-
frequency.select { |_, v| 2 < v }.sort_by { |
|
43
|
+
frequency.select { |_, v| 2 < v }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit).to_h
|
44
44
|
end
|
45
45
|
|
46
|
-
|
46
|
+
def hashtag_clusters(tweets, limit: 10, debug: false)
|
47
|
+
puts "tweets: #{tweets.size}" if debug
|
48
|
+
return {} if tweets.blank?
|
49
|
+
|
50
|
+
tweets = tweets.select { |t| t.text && t.text.include?('#') }
|
51
|
+
puts "tweets with hashtag: #{tweets.size}" if debug
|
52
|
+
|
53
|
+
hashtags = tweets.map { |t| t.text.scan(/[##][A-Za-zA-Za-z一-鿆0-90-9ぁ-ヶヲ-゚ー]+/).map(&:strip) }.flatten
|
54
|
+
puts "hashtags: #{hashtags.size}" if debug
|
55
|
+
|
56
|
+
hashtags.each_with_object(Hash.new(0)) { |h, memo| memo[h] += 1 }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit).to_h
|
57
|
+
end
|
47
58
|
|
48
59
|
def list_clusters(user, shrink: false, each_member: 300, total_member: 1000, rate: 0.3, limit: 10, debug: false)
|
60
|
+
begin
|
61
|
+
require 'mecab'
|
62
|
+
rescue => e
|
63
|
+
puts "Add gem 'mecab' to your Gemfile."
|
64
|
+
return nil
|
65
|
+
end
|
66
|
+
|
49
67
|
begin
|
50
68
|
lists = memberships(user).sort_by { |li| li.member_count }
|
51
69
|
rescue => e
|
@@ -141,9 +159,7 @@ module TwitterWithAutoPagination
|
|
141
159
|
pipe_freq = count_by_word(candidates, delim: '|')
|
142
160
|
puts "words splitted by |: #{pipe_freq.to_a.slice(0, 10)}" if debug
|
143
161
|
|
144
|
-
require 'mecab'
|
145
162
|
tagger = MeCab::Tagger.new("-d #{`mecab-config --dicdir`.chomp}/mecab-ipadic-neologd/")
|
146
|
-
|
147
163
|
noun_freq = count_by_word(remains, tagger: tagger, exclude_words: profile_exclude_words)
|
148
164
|
puts "words tagged as noun: #{noun_freq.to_a.slice(0, 10)}" if debug
|
149
165
|
|
@@ -6,9 +6,8 @@ module TwitterWithAutoPagination
|
|
6
6
|
include TwitterWithAutoPagination::REST::Utils
|
7
7
|
|
8
8
|
def favorites(*args)
|
9
|
-
|
10
|
-
|
11
|
-
args[0] = verify_credentials.id if args.empty?
|
9
|
+
options = {count: 100, call_limit: 3}.merge(args.extract_options!)
|
10
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
12
11
|
instrument(__method__, nil, options) do
|
13
12
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
14
13
|
collect_with_max_id(method(__method__).super_method, *args, options).map { |s| s.attrs }
|
@@ -17,4 +16,4 @@ module TwitterWithAutoPagination
|
|
17
16
|
end
|
18
17
|
end
|
19
18
|
end
|
20
|
-
end
|
19
|
+
end
|
@@ -16,7 +16,7 @@ module TwitterWithAutoPagination
|
|
16
16
|
|
17
17
|
def friend_ids(*args)
|
18
18
|
options = {count: 5000, cursor: -1}.merge(args.extract_options!)
|
19
|
-
args[0] = verify_credentials.id if args.empty?
|
19
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
20
20
|
instrument(__method__, nil, options) do
|
21
21
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
22
22
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -26,7 +26,7 @@ module TwitterWithAutoPagination
|
|
26
26
|
|
27
27
|
def follower_ids(*args)
|
28
28
|
options = {count: 5000, cursor: -1}.merge(args.extract_options!)
|
29
|
-
args[0] = verify_credentials.id if args.empty?
|
29
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
30
30
|
instrument(__method__, nil, options) do
|
31
31
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
32
32
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -37,16 +37,18 @@ module TwitterWithAutoPagination
|
|
37
37
|
# specify reduce: false to use tweet for inactive_*
|
38
38
|
def friends(*args)
|
39
39
|
options = args.extract_options!
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
40
|
+
instrument(__method__, nil, options) do
|
41
|
+
if options.delete(:serial)
|
42
|
+
_friends_serially(*args, options)
|
43
|
+
else
|
44
|
+
_friends_parallelly(*args, options)
|
45
|
+
end
|
44
46
|
end
|
45
47
|
end
|
46
48
|
|
47
49
|
def _friends_serially(*args)
|
48
|
-
options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
|
49
|
-
args[0] = verify_credentials.id if args.empty?
|
50
|
+
options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :friends}.merge(args.extract_options!)
|
51
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
50
52
|
instrument(__method__, nil, options) do
|
51
53
|
fetch_cache_or_call_api(:friends, args[0], options) do
|
52
54
|
collect_with_cursor(method(:friends).super_method, *args, options).map { |u| u.to_hash }
|
@@ -55,25 +57,28 @@ module TwitterWithAutoPagination
|
|
55
57
|
end
|
56
58
|
|
57
59
|
def _friends_parallelly(*args)
|
58
|
-
options =
|
59
|
-
instrument(__method__, nil, options) do
|
60
|
-
|
60
|
+
options = args.extract_options!
|
61
|
+
instrument(__method__, nil, {super_operation: :friends}.merge(options)) do
|
62
|
+
opt = {super_operation: __method__}.merge(options)
|
63
|
+
users(friend_ids(*args, opt).map { |id| id.to_i }, opt)
|
61
64
|
end
|
62
65
|
end
|
63
66
|
|
64
67
|
# specify reduce: false to use tweet for inactive_*
|
65
68
|
def followers(*args)
|
66
69
|
options = args.extract_options!
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
70
|
+
instrument(__method__, nil, options) do
|
71
|
+
if options.delete(:serial)
|
72
|
+
_followers_serially(*args, options)
|
73
|
+
else
|
74
|
+
_followers_parallelly(*args, options)
|
75
|
+
end
|
71
76
|
end
|
72
77
|
end
|
73
78
|
|
74
79
|
def _followers_serially(*args)
|
75
|
-
options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
|
76
|
-
args[0] = verify_credentials.id if args.empty?
|
80
|
+
options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :followers}.merge(args.extract_options!)
|
81
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
77
82
|
instrument(__method__, nil, options) do
|
78
83
|
fetch_cache_or_call_api(:followers, args[0], options) do
|
79
84
|
collect_with_cursor(method(:followers).super_method, *args, options).map { |u| u.to_hash }
|
@@ -82,11 +87,12 @@ module TwitterWithAutoPagination
|
|
82
87
|
end
|
83
88
|
|
84
89
|
def _followers_parallelly(*args)
|
85
|
-
options =
|
86
|
-
instrument(__method__, nil, options) do
|
87
|
-
|
90
|
+
options = args.extract_options!
|
91
|
+
instrument(__method__, nil, {super_operation: :followers}.merge(options)) do
|
92
|
+
opt = {super_operation: __method__}.merge(options)
|
93
|
+
users(follower_ids(*args, opt).map { |id| id.to_i }, opt)
|
88
94
|
end
|
89
95
|
end
|
90
96
|
end
|
91
97
|
end
|
92
|
-
end
|
98
|
+
end
|
@@ -7,7 +7,7 @@ module TwitterWithAutoPagination
|
|
7
7
|
|
8
8
|
def memberships(*args)
|
9
9
|
options = {count: 1000, cursor: -1}.merge(args.extract_options!)
|
10
|
-
args[0] = verify_credentials.id if args.empty?
|
10
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
11
11
|
instrument(__method__, nil, options) do
|
12
12
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
13
13
|
collect_with_cursor(method(__method__).super_method, *args, options)
|
@@ -25,4 +25,4 @@ module TwitterWithAutoPagination
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
end
|
28
|
-
end
|
28
|
+
end
|
@@ -6,32 +6,35 @@ module TwitterWithAutoPagination
|
|
6
6
|
include TwitterWithAutoPagination::REST::Utils
|
7
7
|
|
8
8
|
def home_timeline(*args)
|
9
|
+
mtd = __method__
|
9
10
|
options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
|
10
|
-
instrument(
|
11
|
-
fetch_cache_or_call_api(
|
12
|
-
collect_with_max_id(method(
|
11
|
+
instrument(mtd, nil, options) do
|
12
|
+
fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
|
13
|
+
collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
|
13
14
|
end
|
14
15
|
end
|
15
16
|
end
|
16
17
|
|
17
18
|
def user_timeline(*args)
|
19
|
+
mtd = __method__
|
18
20
|
options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
|
19
|
-
args[0] = verify_credentials.id if args.empty?
|
20
|
-
instrument(
|
21
|
-
fetch_cache_or_call_api(
|
22
|
-
collect_with_max_id(method(
|
21
|
+
args[0] = verify_credentials(super_operation: mtd).id if args.empty?
|
22
|
+
instrument(mtd, nil, options) do
|
23
|
+
fetch_cache_or_call_api(mtd, args[0], options) do
|
24
|
+
collect_with_max_id(method(mtd).super_method, *args, options).map { |s| s.attrs }
|
23
25
|
end
|
24
26
|
end
|
25
27
|
end
|
26
28
|
|
27
29
|
def mentions_timeline(*args)
|
30
|
+
mtd = __method__
|
28
31
|
options = {count: 200, include_rts: true, call_limit: 1}.merge(args.extract_options!)
|
29
|
-
instrument(
|
30
|
-
fetch_cache_or_call_api(
|
31
|
-
collect_with_max_id(method(
|
32
|
+
instrument(mtd, nil, options) do
|
33
|
+
fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
|
34
|
+
collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
|
32
35
|
end
|
33
36
|
end
|
34
37
|
end
|
35
38
|
end
|
36
39
|
end
|
37
|
-
end
|
40
|
+
end
|
@@ -17,7 +17,7 @@ module TwitterWithAutoPagination
|
|
17
17
|
|
18
18
|
def user?(*args)
|
19
19
|
options = args.extract_options!
|
20
|
-
args[0]
|
20
|
+
return false if args.empty? || args[0].nil? || !args[0].to_s.match(/\A([a-zA-Z0-9_]{1,20}|[1-9][0-9]*)\z/)
|
21
21
|
instrument(__method__, nil, options) do
|
22
22
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
23
23
|
call_api(method(__method__).super_method, *args, options)
|
@@ -27,7 +27,7 @@ module TwitterWithAutoPagination
|
|
27
27
|
|
28
28
|
def user(*args)
|
29
29
|
options = args.extract_options!
|
30
|
-
args[0] = verify_credentials.id if args.empty?
|
30
|
+
args[0] = verify_credentials(super_operation: __method__).id if args.empty?
|
31
31
|
instrument(__method__, nil, options) do
|
32
32
|
fetch_cache_or_call_api(__method__, args[0], options) do
|
33
33
|
call_api(method(__method__).super_method, *args, options).to_hash
|
@@ -59,4 +59,4 @@ module TwitterWithAutoPagination
|
|
59
59
|
end
|
60
60
|
end
|
61
61
|
end
|
62
|
-
end
|
62
|
+
end
|
@@ -4,37 +4,6 @@ require 'digest/md5'
|
|
4
4
|
module TwitterWithAutoPagination
|
5
5
|
module REST
|
6
6
|
module Utils
|
7
|
-
# for backward compatibility
|
8
|
-
def uid
|
9
|
-
@uid || user.id.to_i
|
10
|
-
end
|
11
|
-
|
12
|
-
def __uid
|
13
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
14
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
15
|
-
MESSAGE
|
16
|
-
uid
|
17
|
-
end
|
18
|
-
|
19
|
-
def __uid_i
|
20
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
21
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
22
|
-
MESSAGE
|
23
|
-
uid
|
24
|
-
end
|
25
|
-
|
26
|
-
# for backward compatibility
|
27
|
-
def screen_name
|
28
|
-
@screen_name || user.screen_name
|
29
|
-
end
|
30
|
-
|
31
|
-
def __screen_name
|
32
|
-
ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
|
33
|
-
`TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
|
34
|
-
MESSAGE
|
35
|
-
screen_name
|
36
|
-
end
|
37
|
-
|
38
7
|
def uid_or_screen_name?(object)
|
39
8
|
object.kind_of?(String) || object.kind_of?(Integer)
|
40
9
|
end
|
@@ -49,23 +18,21 @@ module TwitterWithAutoPagination
|
|
49
18
|
end
|
50
19
|
|
51
20
|
def credentials_hash
|
52
|
-
|
53
|
-
Digest::MD5.hexdigest(str)
|
21
|
+
Digest::MD5.hexdigest(access_token + access_token_secret + consumer_key + consumer_secret)
|
54
22
|
end
|
55
23
|
|
56
24
|
def instrument(operation, key, options = nil)
|
57
|
-
payload = {operation: operation
|
25
|
+
payload = {operation: operation}
|
58
26
|
payload.merge!(options) if options.is_a?(Hash)
|
59
27
|
ActiveSupport::Notifications.instrument('call.twitter_with_auto_pagination', payload) { yield(payload) }
|
60
28
|
end
|
61
29
|
|
62
|
-
def call_api(
|
30
|
+
def call_api(method, *args)
|
63
31
|
api_options = args.extract_options!
|
32
|
+
self.call_count += 1
|
33
|
+
options = {method: method.name, call_count: self.call_count, args: [*args, api_options]}
|
64
34
|
begin
|
65
|
-
|
66
|
-
# TODO call without reduce, call_count
|
67
|
-
options = {method_name: method_obj.name, call_count: self.call_count, args: [*args, api_options]}
|
68
|
-
instrument('request', args[0], options) { method_obj.call(*args, api_options) }
|
35
|
+
instrument('request', nil, options) { method.call(*args, api_options) }
|
69
36
|
rescue Twitter::Error::TooManyRequests => e
|
70
37
|
logger.warn "#{__method__}: #{options.inspect} #{e.class} Retry after #{e.rate_limit.reset_in} seconds."
|
71
38
|
raise e
|
@@ -74,68 +41,74 @@ module TwitterWithAutoPagination
|
|
74
41
|
logger.warn "#{__method__}: #{options.inspect} #{e.class} #{e.message}"
|
75
42
|
raise e
|
76
43
|
rescue => e
|
77
|
-
logger.warn "
|
44
|
+
logger.warn "CATCH ME! #{__method__}: #{options.inspect} #{e.class} #{e.message}"
|
78
45
|
raise e
|
79
46
|
end
|
80
47
|
end
|
81
48
|
|
82
49
|
# user_timeline, search
|
83
|
-
def collect_with_max_id(
|
50
|
+
def collect_with_max_id(method, *args)
|
84
51
|
options = args.extract_options!
|
85
52
|
call_limit = options.delete(:call_limit) || 3
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
while last_response.any? && call_count < call_limit
|
92
|
-
options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
|
93
|
-
last_response = call_api(method_obj, *args, options)
|
53
|
+
return_data = []
|
54
|
+
call_num = 0
|
55
|
+
|
56
|
+
while call_num < call_limit
|
57
|
+
last_response = call_api(method, *args, options)
|
94
58
|
last_response = yield(last_response) if block_given?
|
59
|
+
call_num += 1
|
95
60
|
return_data += last_response
|
96
|
-
|
61
|
+
if last_response.nil? || last_response.empty?
|
62
|
+
break
|
63
|
+
else
|
64
|
+
options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
|
65
|
+
end
|
97
66
|
end
|
98
67
|
|
99
|
-
return_data
|
68
|
+
return_data
|
100
69
|
end
|
101
70
|
|
102
71
|
# friends, followers
|
103
|
-
def collect_with_cursor(
|
72
|
+
def collect_with_cursor(method, *args)
|
104
73
|
options = args.extract_options!
|
105
|
-
|
106
|
-
|
74
|
+
return_data = []
|
75
|
+
call_num = 0
|
107
76
|
|
108
|
-
while
|
109
|
-
|
110
|
-
|
77
|
+
while call_num < 30
|
78
|
+
last_response = call_api(method, *args, options).attrs
|
79
|
+
call_num += 1
|
111
80
|
return_data += (last_response[:users] || last_response[:ids] || last_response[:lists])
|
81
|
+
options[:cursor] = last_response[:next_cursor]
|
82
|
+
if options[:cursor].nil? || options[:cursor] == 0
|
83
|
+
break
|
84
|
+
end
|
112
85
|
end
|
113
86
|
|
114
87
|
return_data
|
115
88
|
end
|
116
89
|
|
117
|
-
def
|
90
|
+
def normalize_key(method, user, options = {})
|
118
91
|
delim = ':'
|
119
92
|
identifier =
|
120
93
|
case
|
121
|
-
when
|
122
|
-
"hash
|
123
|
-
when
|
94
|
+
when method == :verify_credentials
|
95
|
+
"token-hash#{delim}#{credentials_hash}"
|
96
|
+
when method == :search
|
124
97
|
"str#{delim}#{user.to_s}"
|
125
|
-
when
|
98
|
+
when method == :list_members
|
126
99
|
"list_id#{delim}#{user.to_s}"
|
127
|
-
when
|
100
|
+
when method == :mentions_timeline
|
128
101
|
"#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
|
129
|
-
when
|
102
|
+
when method == :home_timeline
|
130
103
|
"#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
|
131
|
-
when
|
104
|
+
when method.in?([:users, :replying]) && options[:super_operation].present?
|
132
105
|
case
|
133
106
|
when user.kind_of?(Array) && user.first.kind_of?(Integer)
|
134
107
|
"#{options[:super_operation]}-ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
|
135
108
|
when user.kind_of?(Array) && user.first.kind_of?(String)
|
136
109
|
"#{options[:super_operation]}-sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
|
137
110
|
else
|
138
|
-
raise "#{
|
111
|
+
raise "#{method.inspect} #{user.inspect}"
|
139
112
|
end
|
140
113
|
when user.kind_of?(Integer)
|
141
114
|
"id#{delim}#{user.to_s}"
|
@@ -148,10 +121,10 @@ module TwitterWithAutoPagination
|
|
148
121
|
when user.kind_of?(Twitter::User)
|
149
122
|
"user#{delim}#{user.id.to_s}"
|
150
123
|
else
|
151
|
-
raise "#{
|
124
|
+
raise "#{method.inspect} #{user.inspect}"
|
152
125
|
end
|
153
126
|
|
154
|
-
"#{
|
127
|
+
"#{method}#{delim}#{identifier}"
|
155
128
|
end
|
156
129
|
|
157
130
|
CODER = JSON
|
@@ -162,35 +135,31 @@ module TwitterWithAutoPagination
|
|
162
135
|
|
163
136
|
def decode(str)
|
164
137
|
obj = str.kind_of?(String) ? CODER.load(str) : str
|
165
|
-
|
138
|
+
to_mash(obj)
|
166
139
|
end
|
167
140
|
|
168
|
-
def
|
141
|
+
def to_mash(obj)
|
169
142
|
case
|
170
143
|
when obj.kind_of?(Array)
|
171
|
-
obj.map { |o|
|
144
|
+
obj.map { |o| to_mash(o) }
|
172
145
|
when obj.kind_of?(Hash)
|
173
|
-
Hashie::Mash.new(obj.map { |k, v| [k,
|
146
|
+
Hashie::Mash.new(obj.map { |k, v| [k, to_mash(v)] }.to_h)
|
174
147
|
else
|
175
148
|
obj
|
176
149
|
end
|
177
150
|
end
|
178
151
|
|
179
|
-
def fetch_cache_or_call_api(
|
180
|
-
key =
|
152
|
+
def fetch_cache_or_call_api(method, user, options = {})
|
153
|
+
key = normalize_key(method, user, options)
|
181
154
|
|
182
155
|
fetch_result =
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
|
187
|
-
block_result = yield
|
188
|
-
instrument('serialize', key, caller: method_name) { encode(block_result) }
|
189
|
-
end
|
156
|
+
cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
|
157
|
+
block_result = yield
|
158
|
+
instrument('serialize', nil, key: key, caller: method) { encode(block_result) }
|
190
159
|
end
|
191
160
|
|
192
|
-
instrument('deserialize', key, caller:
|
161
|
+
instrument('deserialize', nil, key: key, caller: method) { decode(fetch_result) }
|
193
162
|
end
|
194
163
|
end
|
195
164
|
end
|
196
|
-
end
|
165
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'active_support'
|
2
|
+
require 'active_support/core_ext'
|
1
3
|
require 'twitter'
|
2
4
|
|
3
5
|
require 'twitter_with_auto_pagination/log_subscriber'
|
@@ -16,9 +18,6 @@ module Twitter
|
|
16
18
|
@cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache'))
|
17
19
|
@call_count = 0
|
18
20
|
|
19
|
-
@uid = options.has_key?(:uid) ? options.delete(:uid).to_i : nil
|
20
|
-
@screen_name = options.has_key?(:screen_name) ? options.delete(:screen_name).to_s : nil
|
21
|
-
|
22
21
|
logger =
|
23
22
|
if options.has_key?(:logger)
|
24
23
|
options.delete(:logger)
|
@@ -42,4 +41,4 @@ module Twitter
|
|
42
41
|
INDENT = 4
|
43
42
|
end
|
44
43
|
end
|
45
|
-
end
|
44
|
+
end
|
@@ -6,7 +6,6 @@ Gem::Specification.new do |spec|
|
|
6
6
|
spec.add_dependency 'activesupport'
|
7
7
|
spec.add_dependency 'hashie'
|
8
8
|
spec.add_dependency 'parallel'
|
9
|
-
spec.add_dependency 'mecab'
|
10
9
|
|
11
10
|
spec.add_development_dependency 'bundler'
|
12
11
|
|
@@ -23,5 +22,5 @@ Gem::Specification.new do |spec|
|
|
23
22
|
spec.required_ruby_version = '>= 2.3'
|
24
23
|
spec.summary = spec.description
|
25
24
|
spec.test_files = Dir.glob('spec/**/*')
|
26
|
-
spec.version = '0.8.
|
25
|
+
spec.version = '0.8.5'
|
27
26
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: twitter_with_auto_pagination
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinohara Teruki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: twitter
|
@@ -66,20 +66,6 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
-
- !ruby/object:Gem::Dependency
|
70
|
-
name: mecab
|
71
|
-
requirement: !ruby/object:Gem::Requirement
|
72
|
-
requirements:
|
73
|
-
- - ">="
|
74
|
-
- !ruby/object:Gem::Version
|
75
|
-
version: '0'
|
76
|
-
type: :runtime
|
77
|
-
prerelease: false
|
78
|
-
version_requirements: !ruby/object:Gem::Requirement
|
79
|
-
requirements:
|
80
|
-
- - ">="
|
81
|
-
- !ruby/object:Gem::Version
|
82
|
-
version: '0'
|
83
69
|
- !ruby/object:Gem::Dependency
|
84
70
|
name: bundler
|
85
71
|
requirement: !ruby/object:Gem::Requirement
|