twitter_with_auto_pagination 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5a766fdb1cdee5bb05a9a5a075d2249637284338
+  data.tar.gz: a493b330a56148deb5abf3408b3252c492d56be9
+SHA512:
+  metadata.gz: 8238fe4d1f500572e1993891ecd861f1cb6f90526732bf37defe460fd5e130b1605fded19fd951dc68901c62eead6d69325b71790b9cb552393203ceb402e355
+  data.tar.gz: 9fd76e42c73f82b200ce110eb9efeba3097150e29fd6bbb273d84da4a0d267698bb7d2bba2e6e6b65f5a78880b65b523febde6841a9d51a29a45bbd8dee76573

data/LICENSE.md

@@ -0,0 +1,7 @@
+Copyright (c) 2014 Shinohara Teruki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+ex-twitter
+==========
+
+[![Gem Version](https://badge.fury.io/rb/ex_twitter.png)](https://badge.fury.io/rb/ex_twitter)
+[![Build Status](https://travis-ci.org/ts-3156/ex-twitter.svg?branch=master)](https://travis-ci.org/ts-3156/ex-twitter)
+
+Add auto paginate feature to Twitter gem.
+
+## Installation
+
+### Gem
+
+```
+gem install ex_twitter
+```
+
+### Rails
+
+Add ex_twitter to your Gemfile, and bundle.
+
+## Features
+
+* Auto paginate feature
+
+## Configuration
+
+You can pass configuration options as a block to `ExTwitter.new` just like `Twitter::REST::Client.new`.
+
+```
+client = ExTwitter.new do |config|
+  config.consumer_key        = "YOUR_CONSUMER_KEY"
+  config.consumer_secret     = "YOUR_CONSUMER_SECRET"
+  config.access_token        = "YOUR_ACCESS_TOKEN"
+  config.access_token_secret = "YOUR_ACCESS_SECRET"
+end
+```
+
+You can pass advanced configuration options as a block to `ExTwitter.new`.
+
+```
+client = ExTwitter.new do |config|
+  config.auto_paginate = true
+  config.max_retries   = 1
+  config.max_paginates = 3
+end
+```
+
+## Usage Examples
+
+```
+client.user_timeline
+```

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task :test => :spec
+
+task default: [:spec]

data/lib/twitter_with_auto_pagination.rb

@@ -0,0 +1,4 @@
+require 'twitter_with_auto_pagination/client'
+
+module TwitterWithAutoPagination
+end

data/lib/twitter_with_auto_pagination/client.rb

@@ -0,0 +1,137 @@
+require 'active_support'
+require 'active_support/cache'
+require 'active_support/core_ext/string'
+
+require 'twitter_with_auto_pagination/log_subscriber'
+require 'twitter_with_auto_pagination/utils'
+require 'twitter_with_auto_pagination/existing_api'
+require 'twitter_with_auto_pagination/new_api'
+
+require 'twitter'
+require 'hashie'
+require 'parallel'
+
+module TwitterWithAutoPagination
+  class Client < Twitter::REST::Client
+    def initialize(options = {})
+      @cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache'))
+      @call_count = 0
+
+      @uid = options.has_key?(:uid) ? options.delete(:uid).to_i : nil
+      @screen_name = options.has_key?(:screen_name) ? options.delete(:screen_name).to_s : nil
+
+      @@logger = @logger =
+        if options[:logger]
+          options.delete(:logger)
+        else
+          Dir.mkdir('log') unless File.exists?('log')
+          Logger.new('log/twitter_with_auto_pagination.log')
+        end
+
+      super
+    end
+
+    def self.logger
+      @@logger
+    end
+
+    attr_accessor :call_count
+    attr_reader :cache, :authenticated_user, :logger
+
+    INDENT = 4
+
+    include TwitterWithAutoPagination::Utils
+
+    alias :old_verify_credentials :verify_credentials
+    alias :old_friendship? :friendship?
+    alias :old_user? :user?
+    alias :old_user :user
+    alias :old_friend_ids :friend_ids
+    alias :old_follower_ids :follower_ids
+    alias :old_friends :friends
+    alias :old_followers :followers
+    alias :old_users :users
+    alias :old_home_timeline :home_timeline
+    alias :old_user_timeline :user_timeline
+    alias :old_mentions_timeline :mentions_timeline
+    alias :old_favorites :favorites
+    alias :old_search :search
+
+    include TwitterWithAutoPagination::ExistingApi
+    include TwitterWithAutoPagination::NewApi
+
+    def usage_stats_wday_series_data(times)
+      wday_count = times.each_with_object((0..6).map { |n| [n, 0] }.to_h) do |time, memo|
+        memo[time.wday] += 1
+      end
+      wday_count.map { |k, v| [I18n.t('date.abbr_day_names')[k], v] }.map do |key, value|
+        {name: key, y: value, drilldown: key}
+      end
+    end
+
+    def usage_stats_wday_drilldown_series(times)
+      hour_count =
+        (0..6).each_with_object((0..6).map { |n| [n, nil] }.to_h) do |wday, wday_memo|
+          wday_memo[wday] =
+            times.select { |t| t.wday == wday }.map { |t| t.hour }.each_with_object((0..23).map { |n| [n, 0] }.to_h) do |hour, hour_memo|
+              hour_memo[hour] += 1
+            end
+        end
+      hour_count.map { |k, v| [I18n.t('date.abbr_day_names')[k], v] }.map do |key, value|
+        {name: key, id: key, data: value.to_a.map{|a| [a[0].to_s, a[1]] }}
+      end
+    end
+
+    def usage_stats_hour_series_data(times)
+      hour_count = times.each_with_object((0..23).map { |n| [n, 0] }.to_h) do |time, memo|
+        memo[time.hour] += 1
+      end
+      hour_count.map do |key, value|
+        {name: key.to_s, y: value, drilldown: key.to_s}
+      end
+    end
+
+    def usage_stats_hour_drilldown_series(times)
+      wday_count =
+        (0..23).each_with_object((0..23).map { |n| [n, nil] }.to_h) do |hour, hour_memo|
+          hour_memo[hour] =
+            times.select { |t| t.hour == hour }.map { |t| t.wday }.each_with_object((0..6).map { |n| [n, 0] }.to_h) do |wday, wday_memo|
+              wday_memo[wday] += 1
+            end
+        end
+      wday_count.map do |key, value|
+        {name: key.to_s, id: key.to_s, data: value.to_a.map{|a| [I18n.t('date.abbr_day_names')[a[0]], a[1]] }}
+      end
+    end
+
+    def twitter_addiction_series(times)
+      five_mins = 5.minutes
+      wday_expended_seconds =
+        (0..6).each_with_object((0..6).map { |n| [n, nil] }.to_h) do |wday, wday_memo|
+          target_times = times.select { |t| t.wday == wday }
+          wday_memo[wday] = target_times.empty? ? nil : target_times.each_cons(2).map {|a, b| (a - b) < five_mins ? a - b : five_mins }.sum
+        end
+      days = times.map{|t| t.to_date.to_s(:long) }.uniq.size
+      weeks = (days > 7) ? days / 7.0 : 1.0
+      wday_expended_seconds.map { |k, v| [I18n.t('date.abbr_day_names')[k], (v.nil? ? nil : v / weeks / 60)] }.map do |key, value|
+        {name: key, y: value}
+      end
+    end
+
+    def usage_stats(user, options = {})
+      n_days_ago = options.has_key?(:days) ? options[:days].days.ago : 100.years.ago
+      tweets = options.has_key?(:tweets) ? options.delete(:tweets) : user_timeline(user)
+      times =
+        # TODO Use user specific time zone
+        tweets.map { |t| ActiveSupport::TimeZone['Tokyo'].parse(t.created_at.to_s) }.
+          select { |t| t > n_days_ago }
+      [
+        usage_stats_wday_series_data(times),
+        usage_stats_wday_drilldown_series(times),
+        usage_stats_hour_series_data(times),
+        usage_stats_hour_drilldown_series(times),
+        twitter_addiction_series(times)
+      ]
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/existing_api.rb

@@ -0,0 +1,127 @@
+module TwitterWithAutoPagination
+  module ExistingApi
+    def verify_credentials(*args)
+      options = {skip_status: true}.merge(args.extract_options!)
+      fetch_cache_or_call_api(__method__, args) {
+        call_old_method("old_#{__method__}", *args, options)
+      }
+    end
+
+    def friendship?(*args)
+      options = args.extract_options!
+      fetch_cache_or_call_api(__method__, args) {
+        call_old_method("old_#{__method__}", *args, options)
+      }
+    end
+
+    def user?(*args)
+      options = args.extract_options!
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        call_old_method("old_#{__method__}", args[0], options)
+      }
+    end
+
+    def user(*args)
+      options = args.extract_options!
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        call_old_method("old_#{__method__}", args[0], options)
+      }
+    end
+
+    def friend_ids(*args)
+      options = {count: 5000, cursor: -1}.merge(args.extract_options!)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_cursor("old_#{__method__}", *args, options)
+      }
+    end
+
+    def follower_ids(*args)
+      options = {count: 5000, cursor: -1}.merge(args.extract_options!)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_cursor("old_#{__method__}", *args, options)
+      }
+    end
+
+    # specify reduce: false to use tweet for inactive_*
+    def friends(*args)
+      options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
+      options[:reduce] = false unless options.has_key?(:reduce)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_cursor("old_#{__method__}", *args, options)
+      }
+    end
+
+    # specify reduce: false to use tweet for inactive_*
+    def followers(*args)
+      options = {count: 200, include_user_entities: true, cursor: -1}.merge(args.extract_options!)
+      options[:reduce] = false unless options.has_key?(:reduce)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_cursor("old_#{__method__}", *args, options)
+      }
+    end
+
+    # use compact, not use sort and uniq
+    # specify reduce: false to use tweet for inactive_*
+    # TODO Perhaps `old_users` automatically merges result...
+    def users(*args)
+      options = args.extract_options!
+      options[:reduce] = false
+      users_per_workers = args.first.compact.each_slice(100).to_a
+      processed_users = []
+
+      Parallel.each_with_index(users_per_workers, in_threads: [users_per_workers.size, 10].min) do |users_per_worker, i|
+        _users = fetch_cache_or_call_api(__method__, users_per_worker, options) {
+          call_old_method("old_#{__method__}", users_per_worker, options)
+        }
+
+        processed_users << {i: i, users: _users}
+      end
+
+      processed_users.sort_by{|p| p[:i] }.map{|p| p[:users] }.flatten.compact
+    end
+
+    def home_timeline(*args)
+      options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
+      fetch_cache_or_call_api(__method__, user.id, options) {
+        collect_with_max_id("old_#{__method__}", options)
+      }
+    end
+
+    def user_timeline(*args)
+      options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_max_id("old_#{__method__}", *args, options)
+      }
+    end
+
+    def mentions_timeline(*args)
+      options = {count: 200, include_rts: true, call_limit: 1}.merge(args.extract_options!)
+      fetch_cache_or_call_api(__method__, user.id, options) {
+        collect_with_max_id("old_#{__method__}", options)
+      }
+    end
+
+    def favorites(*args)
+      options = {count: 100, call_count: 1}.merge(args.extract_options!)
+      args[0] = verify_credentials(skip_status: true).id if args.empty?
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_max_id("old_#{__method__}", *args, options)
+      }
+    end
+
+    def search(*args)
+      options = {count: 100, result_type: :recent, call_limit: 1}.merge(args.extract_options!)
+      options[:reduce] = false
+      fetch_cache_or_call_api(__method__, args[0], options) {
+        collect_with_max_id("old_#{__method__}", *args, options) { |response| response.attrs[:statuses] }
+      }
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/log_subscriber.rb

@@ -0,0 +1,82 @@
+require 'active_support'
+require 'active_support/core_ext'
+
+
+module TwitterWithAutoPagination
+  class LogSubscriber < ActiveSupport::LogSubscriber
+
+    def initialize
+      super
+      @odd = false
+    end
+
+    def cache_any(event)
+      return unless logger.debug?
+
+      payload = event.payload
+      name  = "#{payload.delete(:name)} (#{event.duration.round(1)}ms)"
+      name = colorize_payload_name(name, payload[:name], AS: true)
+      debug { "#{name} #{(payload.inspect)}" }
+    end
+
+    %w(read write fetch_hit generate delete exist?).each do |operation|
+      class_eval <<-METHOD, __FILE__, __LINE__ + 1
+          def cache_#{operation}(event)
+            event.payload[:name] = '#{operation}'
+            cache_any(event)
+          end
+      METHOD
+    end
+
+    def call(event)
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
+
+      name = colorize_payload_name(name, payload[:name])
+      # sql  = color(sql, sql_color(sql), true)
+
+      key = payload.delete(:key)
+      debug { "#{name} #{key} #{(payload.inspect)}" }
+    end
+
+    private
+
+    def colorize_payload_name(name, payload_name, options = {})
+      if options[:AS]
+        color(name, MAGENTA, true)
+      else
+        color(name, CYAN, true)
+      end
+    end
+
+    def sql_color(sql)
+      case sql
+        when /\A\s*rollback/mi
+          RED
+        when /select .*for update/mi, /\A\s*lock/mi
+          WHITE
+        when /\A\s*select/i
+          BLUE
+        when /\A\s*insert/i
+          GREEN
+        when /\A\s*update/i
+          YELLOW
+        when /\A\s*delete/i
+          RED
+        when /transaction\s*\Z/i
+          CYAN
+        else
+          MAGENTA
+      end
+    end
+
+    def logger
+      TwitterWithAutoPagination::Client.logger
+    end
+  end
+end
+
+TwitterWithAutoPagination::LogSubscriber.attach_to :twitter_with_auto_pagination
+TwitterWithAutoPagination::LogSubscriber.attach_to :active_support

data/lib/twitter_with_auto_pagination/new_api.rb

@@ -0,0 +1,331 @@
+module TwitterWithAutoPagination
+  module NewApi
+    def friends_parallelly(*args)
+      options = {super_operation: __method__}.merge(args.extract_options!)
+      _friend_ids = friend_ids(*(args + [options]))
+      users(_friend_ids.map { |id| id.to_i }, options)
+    end
+
+    def followers_parallelly(*args)
+      options = {super_operation: __method__}.merge(args.extract_options!)
+      _follower_ids = follower_ids(*(args + [options]))
+      users(_follower_ids.map { |id| id.to_i }, options)
+    end
+
+    def _fetch_parallelly(signatures) # [{method: :friends, args: ['ts_3156', ...], {...}]
+      result = Array.new(signatures.size)
+
+      Parallel.each_with_index(signatures, in_threads: result.size) do |signature, i|
+        result[i] = send(signature[:method], *signature[:args])
+      end
+
+      result
+    end
+
+    def friends_and_followers(*args)
+      _fetch_parallelly(
+        [
+          {method: :friends_parallelly, args: args},
+          {method: :followers_parallelly, args: args}])
+    end
+
+    def friends_followers_and_statuses(*args)
+      _fetch_parallelly(
+        [
+          {method: :friends_parallelly, args: args},
+          {method: :followers_parallelly, args: args},
+          {method: :user_timeline, args: args}])
+    end
+
+    def one_sided_following(me)
+      if uid_or_screen_name?(me)
+        # TODO use friends_and_followers
+        friends_parallelly(me).to_a - followers_parallelly(me).to_a
+      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
+        me.friends.to_a - me.followers.to_a
+      else
+        raise
+      end
+    end
+
+    def one_sided_followers(me)
+      if uid_or_screen_name?(me)
+        # TODO use friends_and_followers
+        followers_parallelly(me).to_a - friends_parallelly(me).to_a
+      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
+        me.followers.to_a - me.friends.to_a
+      else
+        raise
+      end
+    end
+
+    def mutual_friends(me)
+      if uid_or_screen_name?(me)
+        # TODO use friends_and_followers
+        friends_parallelly(me).to_a & followers_parallelly(me).to_a
+      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
+        me.friends.to_a & me.followers.to_a
+      else
+        raise
+      end
+    end
+
+    def common_friends(me, you)
+      if uid_or_screen_name?(me) && uid_or_screen_name?(you)
+        friends_parallelly(me).to_a & friends_parallelly(you).to_a
+      elsif me.respond_to?(:friends) && you.respond_to?(:friends)
+        me.friends.to_a & you.friends.to_a
+      else
+        raise
+      end
+    end
+
+    def common_followers(me, you)
+      if uid_or_screen_name?(me) && uid_or_screen_name?(you)
+        followers_parallelly(me).to_a & followers_parallelly(you).to_a
+      elsif me.respond_to?(:followers) && you.respond_to?(:followers)
+        me.followers.to_a & you.followers.to_a
+      else
+        raise
+      end
+    end
+
+    def removing(pre_me, cur_me)
+      if uid_or_screen_name?(pre_me) && uid_or_screen_name?(cur_me)
+        friends_parallelly(pre_me).to_a - friends_parallelly(cur_me).to_a
+      elsif pre_me.respond_to?(:friends) && cur_me.respond_to?(:friends)
+        pre_me.friends.to_a - cur_me.friends.to_a
+      else
+        raise
+      end
+    end
+
+    def removed(pre_me, cur_me)
+      if uid_or_screen_name?(pre_me) && uid_or_screen_name?(cur_me)
+        followers_parallelly(pre_me).to_a - followers_parallelly(cur_me).to_a
+      elsif pre_me.respond_to?(:followers) && cur_me.respond_to?(:followers)
+        pre_me.followers.to_a - cur_me.followers.to_a
+      else
+        raise
+      end
+    end
+
+    def _extract_screen_names(tweets)
+      tweets.map do |t|
+        $1 if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
+      end.compact
+    end
+
+    # users which specified user is replying
+    # in_reply_to_user_id and in_reply_to_status_id is not used because of distinguishing mentions from replies
+    def replying(*args)
+      options = args.extract_options!
+      tweets =
+        if args.empty?
+          user_timeline(options)
+        elsif uid_or_screen_name?(args[0])
+          user_timeline(args[0], options)
+        elsif args[0].kind_of?(Array) && args[0].all? { |t| t.respond_to?(:text) }
+          args[0]
+        else
+          raise
+        end
+
+      screen_names = _extract_screen_names(tweets)
+      result = users(screen_names, {super_operation: __method__}.merge(options))
+      if options.has_key?(:uniq) && !options[:uniq]
+        screen_names.map { |sn| result.find { |r| r.screen_name == sn } }.compact
+      else
+        result.uniq { |r| r.id }
+      end
+    rescue Twitter::Error::NotFound => e
+      e.message == 'No user matches for specified terms.' ? [] : (raise e)
+    rescue => e
+      logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    end
+
+    def _extract_uids(tweets)
+      tweets.map do |t|
+        t.user.id.to_i if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
+      end.compact
+    end
+
+    def _extract_users(tweets, uids)
+      uids.map { |uid| tweets.find { |t| t.user.id.to_i == uid.to_i } }.map { |t| t.user }.compact
+    end
+
+    # users which specified user is replied
+    # when user is login you had better to call mentions_timeline
+    def replied(*args)
+      options = args.extract_options!
+
+      result =
+        if args.empty? || (uid_or_screen_name?(args[0]) && authenticating_user?(args[0]))
+          mentions_timeline.map { |m| m.user }
+        else
+          searched_result = search('@' + user(args[0]).screen_name, options)
+          uids = _extract_uids(searched_result)
+          _extract_users(searched_result, uids)
+        end
+
+      if options.has_key?(:uniq) && !options[:uniq]
+        result
+      else
+        result.uniq { |r| r.id }
+      end
+    end
+
+    def _count_users_with_two_sided_threshold(users, options)
+      min = options.has_key?(:min) ? options[:min] : 0
+      max = options.has_key?(:max) ? options[:max] : 1000
+      users.each_with_object(Hash.new(0)) { |u, memo| memo[u.id] += 1 }.
+        select { |_k, v| min <= v && v <= max }.
+        sort_by { |_, v| -v }.to_h
+    end
+
+    def _extract_favorite_users(favs, options = {})
+      counted_value = _count_users_with_two_sided_threshold(favs.map { |t| t.user }, options)
+      counted_value.map do |uid, cnt|
+        fav = favs.find { |f| f.user.id.to_i == uid.to_i }
+        Array.new(cnt, fav.user)
+      end.flatten
+    end
+
+    def favoriting(*args)
+      options = args.extract_options!
+
+      favs =
+        if args.empty?
+          favorites(options)
+        elsif uid_or_screen_name?(args[0])
+          favorites(args[0], options)
+        elsif args[0].kind_of?(Array) && args[0].all? { |t| t.respond_to?(:text) }
+          args[0]
+        else
+          raise
+        end
+
+      result = _extract_favorite_users(favs, options)
+      if options.has_key?(:uniq) && !options[:uniq]
+        result
+      else
+        result.uniq { |r| r.id }
+      end
+    rescue => e
+      logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
+      raise e
+    end
+
+    def _extract_inactive_users(users, options = {})
+      authorized = options.delete(:authorized)
+      two_weeks_ago = 2.weeks.ago.to_i
+      users.select do |u|
+        if authorized
+          (Time.parse(u.status.created_at).to_i < two_weeks_ago) rescue false
+        else
+          false
+        end
+      end
+    end
+
+    def favorited_by(*args)
+    end
+
+    def close_friends(*args)
+      options = {uniq: false}.merge(args.extract_options!)
+      min_max = {
+        min: options.has_key?(:min) ? options.delete(:min) : 0,
+        max: options.has_key?(:max) ? options.delete(:max) : 1000
+      }
+
+      _replying, _replied, _favoriting =
+        if args.empty?
+          [replying(options), replied(options), favoriting(options)]
+        elsif uid_or_screen_name?(args[0])
+          [replying(args[0], options), replied(args[0], options), favoriting(args[0], options)]
+        elsif (m_names = %i(replying replied favoriting)).all? { |m_name| args[0].respond_to?(m_name) }
+          m_names.map { |mn| args[0].send(mn) }
+        else
+          raise
+        end
+
+      _users = _replying + _replied + _favoriting
+      return [] if _users.empty?
+
+      scores = _count_users_with_two_sided_threshold(_users, min_max)
+      replying_scores = _count_users_with_two_sided_threshold(_replying, min_max)
+      replied_scores = _count_users_with_two_sided_threshold(_replied, min_max)
+      favoriting_scores = _count_users_with_two_sided_threshold(_favoriting, min_max)
+
+      scores.keys.map { |uid| _users.find { |u| u.id.to_i == uid.to_i } }.
+        map do |u|
+        u[:score] = scores[u.id]
+        u[:replying_score] = replying_scores[u.id]
+        u[:replied_score] = replied_scores[u.id]
+        u[:favoriting_score] = favoriting_scores[u.id]
+        u
+      end
+    end
+
+    def inactive_friends(user = nil)
+      if user.blank?
+        _extract_inactive_users(friends_parallelly, authorized: true)
+      elsif uid_or_screen_name?(user)
+        authorized = authenticating_user?(user) || authorized_user?(user)
+        _extract_inactive_users(friends_parallelly(user), authorized: authorized)
+      elsif user.respond_to?(:friends)
+        authorized = authenticating_user?(user.uid.to_i) || authorized_user?(user.uid.to_i)
+        _extract_inactive_users(user.friends, authorized: authorized)
+      else
+        raise
+      end
+    end
+
+    def inactive_followers(user = nil)
+      if user.blank?
+        _extract_inactive_users(followers_parallelly, authorized: true)
+      elsif uid_or_screen_name?(user)
+        authorized = authenticating_user?(user) || authorized_user?(user)
+        _extract_inactive_users(followers_parallelly(user), authorized: authorized)
+      elsif user.respond_to?(:followers)
+        authorized = authenticating_user?(user.uid.to_i) || authorized_user?(user.uid.to_i)
+        _extract_inactive_users(user.followers, authorized: authorized)
+      else
+        raise
+      end
+    end
+
+    def clusters_belong_to(text)
+      return [] if text.blank?
+
+      exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
+      special_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_good_words_path']))
+
+      # クラスタ用の単語の出現回数を記録
+      cluster_word_counter =
+        special_words.map { |sw| [sw, text.scan(sw)] }
+          .delete_if { |item| item[1].empty? }
+          .each_with_object(Hash.new(1)) { |item, memo| memo[item[0]] = item[1].size }
+
+      # 同一文字種の繰り返しを見付ける。漢字の繰り返し、ひらがなの繰り返し、カタカナの繰り返し、など
+      text.scan(/[一-龠〆ヵヶ々]+|[ぁ-んー～]+|[ァ-ヴー～]+|[ａ-ｚＡ-Ｚ０-９]+|[、。！!？?]+/).
+
+        # 複数回繰り返される文字を除去
+        map { |w| w.remove /[？！?!。、ｗ]|(ー{2,})/ }.
+
+        # 文字数の少なすぎる単語、ひらがなだけの単語、除外単語を除去する
+        delete_if { |w| w.length <= 1 || (w.length <= 2 && w =~ /^[ぁ-んー～]+$/) || exclude_words.include?(w) }.
+
+        # 出現回数を記録
+        each { |w| cluster_word_counter[w] += 1 }
+
+      # 複数個以上見付かった単語のみを残し、出現頻度順にソート
+      cluster_word_counter.select { |_, v| v > 3 }.sort_by { |_, v| -v }.to_h
+    end
+
+    def clusters_assigned_to
+      raise NotImplementedError.new
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/utils.rb

@@ -0,0 +1,303 @@
+module TwitterWithAutoPagination
+  module Utils
+    # for backward compatibility
+    def uid
+      @uid || user.id.to_i
+    end
+
+    def __uid
+      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
+        `TwitterWithAutoPagination::Utils#__uid` is deprecated.
+      MESSAGE
+      uid
+    end
+
+    def __uid_i
+      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
+        `TwitterWithAutoPagination::Utils#__uid_i` is deprecated.
+      MESSAGE
+      uid
+    end
+
+    # for backward compatibility
+    def screen_name
+      @screen_name || user.screen_name
+    end
+
+    def __screen_name
+      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
+        `TwitterWithAutoPagination::Utils#__screen_name` is deprecated.
+      MESSAGE
+      screen_name
+    end
+
+    def uid_or_screen_name?(object)
+      object.kind_of?(String) || object.kind_of?(Integer)
+    end
+
+    def authenticating_user?(target)
+      user.id.to_i == user(target).id.to_i
+    end
+
+    def authorized_user?(target)
+      target_user = user(target)
+      !target_user.protected? || friendship?(user.id.to_i, target_user.id.to_i)
+    end
+
+    def instrument(operation, key, options = nil)
+      payload = {operation: operation, key: key}
+      payload.merge!(options) if options.is_a?(Hash)
+      ActiveSupport::Notifications.instrument('call.twitter_with_auto_pagination', payload) { yield(payload) }
+    end
+
+    def call_old_method(method_name, *args)
+      options = args.extract_options!
+      begin
+        self.call_count += 1
+        _options = {method_name: method_name, call_count: self.call_count, args: args}.merge(options)
+        instrument('api call', args[0], _options) { send(method_name, *args, options) }
+      rescue Twitter::Error::TooManyRequests => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} Retry after #{e.rate_limit.reset_in} seconds."
+        raise e
+      rescue Twitter::Error::ServiceUnavailable => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+        raise e
+      rescue Twitter::Error::InternalServerError => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+        raise e
+      rescue Twitter::Error::Forbidden => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+        raise e
+      rescue Twitter::Error::NotFound => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+        raise e
+      rescue => e
+        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+        raise e
+      end
+    end
+
+    # user_timeline, search
+    def collect_with_max_id(method_name, *args)
+      options = args.extract_options!
+      call_limit = options.delete(:call_limit) || 3
+      last_response = call_old_method(method_name, *args, options)
+      last_response = yield(last_response) if block_given?
+      return_data = last_response
+      call_count = 1
+
+      while last_response.any? && call_count < call_limit
+        options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
+        last_response = call_old_method(method_name, *args, options)
+        last_response = yield(last_response) if block_given?
+        return_data += last_response
+        call_count += 1
+      end
+
+      return_data.flatten
+    end
+
+    # friends, followers
+    def collect_with_cursor(method_name, *args)
+      options = args.extract_options!
+      last_response = call_old_method(method_name, *args, options).attrs
+      return_data = (last_response[:users] || last_response[:ids])
+
+      while (next_cursor = last_response[:next_cursor]) && next_cursor != 0
+        options[:cursor] = next_cursor
+        last_response = call_old_method(method_name, *args, options).attrs
+        return_data += (last_response[:users] || last_response[:ids])
+      end
+
+      return_data
+    end
+
+    require 'digest/md5'
+
+    def file_cache_key(method_name, user, options = {})
+      delim = ':'
+      identifier =
+        case
+          when method_name == :verify_credentials
+            "object-id#{delim}#{object_id}"
+          when method_name == :search
+            "str#{delim}#{user.to_s}"
+          when method_name == :mentions_timeline
+            "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
+          when method_name == :home_timeline
+            "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
+          when method_name.in?([:users, :replying]) && options[:super_operation].present?
+            case
+              when user.kind_of?(Array) && user.first.kind_of?(Integer)
+                "#{options[:super_operation]}-ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+              when user.kind_of?(Array) && user.first.kind_of?(String)
+                "#{options[:super_operation]}-sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+              else raise "#{method_name.inspect} #{user.inspect}"
+            end
+          when user.kind_of?(Integer)
+            "id#{delim}#{user.to_s}"
+          when user.kind_of?(Array) && user.first.kind_of?(Integer)
+            "ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+          when user.kind_of?(Array) && user.first.kind_of?(String)
+            "sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+          when user.kind_of?(String)
+            "sn#{delim}#{user}"
+          when user.kind_of?(Twitter::User)
+            "user#{delim}#{user.id.to_s}"
+          else raise "#{method_name.inspect} #{user.inspect}"
+        end
+
+      "#{method_name}#{delim}#{identifier}"
+    end
+
+    def namespaced_key(method_name, user, options = {})
+      file_cache_key(method_name, user, options)
+    end
+
+    PROFILE_SAVE_KEYS = %i(
+        id
+        name
+        screen_name
+        location
+        description
+        url
+        protected
+        followers_count
+        friends_count
+        listed_count
+        favourites_count
+        utc_offset
+        time_zone
+        geo_enabled
+        verified
+        statuses_count
+        lang
+        status
+        profile_image_url_https
+        profile_banner_url
+        profile_link_color
+        suspended
+        verified
+        entities
+        created_at
+      )
+
+    STATUS_SAVE_KEYS = %i(
+      created_at
+      id
+      text
+      source
+      truncated
+      coordinates
+      place
+      entities
+      user
+      contributors
+      is_quote_status
+      retweet_count
+      favorite_count
+      favorited
+      retweeted
+      possibly_sensitive
+      lang
+    )
+
+    # encode
+    def encode_json(obj, caller_name, options = {})
+      options[:reduce] = true unless options.has_key?(:reduce)
+      case caller_name
+        when :user_timeline, :home_timeline, :mentions_timeline, :favorites # Twitter::Tweet
+          JSON.pretty_generate(obj.map { |o| o.attrs })
+
+        when :search # Hash
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*STATUS_SAVE_KEYS) }
+            else
+              obj.map { |o| o.to_hash }
+            end
+          JSON.pretty_generate(data)
+
+        when :friends, :followers # Hash
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
+            else
+              obj.map { |o| o.to_hash }
+            end
+          JSON.pretty_generate(data)
+
+        when :friend_ids, :follower_ids # Integer
+          JSON.pretty_generate(obj)
+
+        when :verify_credentials # Twitter::User
+          JSON.pretty_generate(obj.to_hash.slice(*PROFILE_SAVE_KEYS))
+
+        when :user # Twitter::User
+          JSON.pretty_generate(obj.to_hash.slice(*PROFILE_SAVE_KEYS))
+
+        when :users, :friends_parallelly, :followers_parallelly # Twitter::User
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
+            else
+              obj.map { |o| o.to_hash }
+            end
+          JSON.pretty_generate(data)
+
+        when :user? # true or false
+          obj
+
+        when :friendship? # true or false
+          obj
+
+        else
+          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
+      end
+    end
+
+    # decode
+    def decode_json(json_str, caller_name, options = {})
+      obj = json_str.kind_of?(String) ? JSON.parse(json_str) : json_str
+      case
+        when obj.nil?
+          obj
+
+        when obj.kind_of?(Array) && obj.first.kind_of?(Hash)
+          obj.map { |o| Hashie::Mash.new(o) }
+
+        when obj.kind_of?(Array) && obj.first.kind_of?(Integer)
+          obj
+
+        when obj.kind_of?(Hash)
+          Hashie::Mash.new(obj)
+
+        when obj === true || obj === false
+          obj
+
+        when obj.kind_of?(Array) && obj.empty?
+          obj
+
+        else
+          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
+      end
+    end
+
+    def fetch_cache_or_call_api(method_name, user, options = {})
+      key = namespaced_key(method_name, user, options)
+      options.update(key: key)
+
+      data =
+        if options[:cache] == :read
+          instrument('Cache Read(Force)', key, caller: method_name) { cache.read(key) }
+        else
+          cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
+            _d = yield
+            instrument('serialize', key, caller: method_name) { encode_json(_d, method_name, options) }
+          end
+        end
+
+      instrument('deserialize', key, caller: method_name) { decode_json(data, method_name, options) }
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,9 @@
+require 'twitter_with_auto_pagination'
+require 'rspec'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+  config.order = 'random'
+end

data/spec/twitter_with_auto_pagination_spec.rb

@@ -0,0 +1,131 @@
+require 'helper'
+
+describe TwitterWithAutoPagination do
+  let(:config) {
+    {
+      consumer_key: 'CK',
+      consumer_secret: 'CS',
+      access_token: 'AT',
+      access_token_secret: 'ATS',
+    }
+  }
+  let(:client) { TwitterWithAutoPagination::Client.new(config) }
+
+  describe '#initialize' do
+    let(:default_call_count) { 0 }
+
+    it 'sets call_count to 0' do
+      expect(client.call_count).to eq(default_call_count)
+    end
+
+    context 'without params' do
+    end
+
+    context 'with params' do
+    end
+  end
+
+  describe '#logger' do
+    it 'has logger' do
+      expect(client.logger).to be_truthy
+    end
+  end
+
+  describe '#call_old_method' do
+  end
+
+  describe '#collect_with_max_id' do
+  end
+
+  describe '#collect_with_cursor' do
+  end
+
+  describe '#file_cache_key' do
+  end
+
+  describe '#namespaced_key' do
+  end
+
+  describe '#encode_json' do
+  end
+
+  describe '#decode_json' do
+  end
+
+  describe '#fetch_cache_or_call_api' do
+  end
+
+  describe '#user_timeline' do
+    it 'calls old_user_timeline' do
+      expect(client).to receive(:old_user_timeline)
+      client.user_timeline
+    end
+
+    it 'calls collect_with_max_id' do
+      expect(client).to receive(:collect_with_max_id)
+      client.user_timeline
+    end
+  end
+
+  describe '#user_photos' do
+    it 'calls user_timeline' do
+      expect(client).to receive(:user_timeline)
+      client.user_photos
+    end
+  end
+
+  describe '#friends' do
+    it 'calls old_friends' do
+      expect(client).to receive(:old_friends)
+      client.friends
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.friends
+    end
+  end
+
+  describe '#followers' do
+    it 'calls old_followers' do
+      expect(client).to receive(:old_followers)
+      client.followers
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.followers
+    end
+  end
+
+  describe '#friend_ids' do
+    it 'calls old_friend_ids' do
+      expect(client).to receive(:old_friend_ids)
+      client.friend_ids
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.friend_ids
+    end
+  end
+
+  describe '#follower_ids' do
+    it 'calls old_follower_ids' do
+      expect(client).to receive(:old_follower_ids)
+      client.follower_ids
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.follower_ids
+    end
+  end
+
+  describe '#users' do
+    it 'calls old_users' do
+      expect(client).to receive(:old_users)
+      client.users([1, 2, 3])
+    end
+  end
+end

data/twitter_with_auto_pagination.gemspec

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.add_dependency 'twitter'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'hashie'
+  spec.add_dependency 'parallel'
+
+  spec.add_development_dependency 'bundler'
+
+  spec.authors = ['Shinohara Teruki']
+  spec.description = %q(Add auto paginate feature to Twitter gem.)
+  spec.email = %w[ts_3156@yahoo.co.jp]
+  spec.files = %w[LICENSE.md README.md Rakefile twitter_with_auto_pagination.gemspec]
+  spec.files += Dir.glob('lib/**/*.rb')
+  spec.files += Dir.glob('spec/**/*')
+  spec.homepage = 'http://github.com/ts-3156/twitter_with_auto_pagination/'
+  spec.licenses = %w[MIT]
+  spec.name = 'twitter_with_auto_pagination'
+  spec.require_paths = %w[lib]
+  spec.summary = spec.description
+  spec.test_files = Dir.glob('spec/**/*')
+  spec.version = '0.4.0'
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: twitter_with_auto_pagination
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: ruby
+authors:
+- Shinohara Teruki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-07-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Add auto paginate feature to Twitter gem.
+email:
+- ts_3156@yahoo.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.md
+- README.md
+- Rakefile
+- lib/twitter_with_auto_pagination.rb
+- lib/twitter_with_auto_pagination/client.rb
+- lib/twitter_with_auto_pagination/existing_api.rb
+- lib/twitter_with_auto_pagination/log_subscriber.rb
+- lib/twitter_with_auto_pagination/new_api.rb
+- lib/twitter_with_auto_pagination/utils.rb
+- spec/helper.rb
+- spec/twitter_with_auto_pagination_spec.rb
+- twitter_with_auto_pagination.gemspec
+homepage: http://github.com/ts-3156/twitter_with_auto_pagination/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Add auto paginate feature to Twitter gem.
+test_files:
+- spec/helper.rb
+- spec/twitter_with_auto_pagination_spec.rb