twitter_with_auto_pagination 0.8.12 → 0.9.0

data/lib/twitter_with_auto_pagination/rest/friends_and_followers.rb

@@ -5,93 +5,56 @@ module TwitterWithAutoPagination
     module FriendsAndFollowers
       include TwitterWithAutoPagination::REST::Utils
 
-      def friendship?(*args)
-        options = args.extract_options!
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args) do
-            call_api(method(__method__).super_method, *args, options)
-          end
-        end
+      def friendship?(from, to, options = {})
+        twitter.send(__method__, from, to, options)
       end
 
-      def friend_ids(*args)
-        options = {count: 5000, cursor: -1}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_cursor(method(__method__).super_method, *args, options)
-          end
-        end
-      end
+      MAX_IDS_PER_REQUEST = 5000
 
-      def follower_ids(*args)
-        options = {count: 5000, cursor: -1}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_cursor(method(__method__).super_method, *args, options)
+      %i(friend_ids follower_ids).each do |name|
+        define_method(name) do |*args|
+          options = {count: MAX_IDS_PER_REQUEST, cursor: -1}.merge(args.extract_options!)
+
+          collect_with_cursor do |next_cursor|
+            options[:next_cursor] = next_cursor unless next_cursor.nil?
+            twitter.send(name, *args, options)
           end
         end
       end
 
-      # specify reduce: false to use tweet for inactive_*
       def friends(*args)
-        options = args.extract_options!
-        instrument(__method__, nil, options) do
-          if options.delete(:serial)
-            _friends_serially(*args, options)
-          else
-            _friends_parallelly(*args, options)
-          end
-        end
+        options = args.extract_options!.merge(super_operation: :friends)
+        ids = friend_ids(*args, options)
+        users_internal(ids, options)
       end
 
-      def _friends_serially(*args)
-        options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :friends}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(:friends, args[0], options) do
-            collect_with_cursor(method(:friends).super_method, *args, options).map { |u| u.to_hash }
-          end
-        end
+      def followers(*args)
+        options = args.extract_options!.merge(super_operation: :followers)
+        ids = follower_ids(*args, options)
+        users_internal(ids, options)
       end
 
-      def _friends_parallelly(*args)
-        options = args.extract_options!
-        instrument(__method__, nil, {super_operation: :friends}.merge(options)) do
-          opt = {super_operation: __method__}.merge(options)
-          users(friend_ids(*args, opt).map { |id| id.to_i }, opt)
-        end
-      end
+      def friend_ids_and_follower_ids(*args)
+        options = args.extract_options!.merge(super_operation: :friend_ids_and_follower_ids)
 
-      # specify reduce: false to use tweet for inactive_*
-      def followers(*args)
-        options = args.extract_options!
-        instrument(__method__, nil, options) do
-          if options.delete(:serial)
-            _followers_serially(*args, options)
-          else
-            _followers_parallelly(*args, options)
-          end
+        parallel(in_threads: 2) do |batch|
+          batch.friend_ids(*args, options)
+          batch.follower_ids(*args, options)
         end
       end
 
-      def _followers_serially(*args)
-        options = {count: 200, include_user_entities: true, cursor: -1, super_operation: :followers}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(:followers, args[0], options) do
-            collect_with_cursor(method(:followers).super_method, *args, options).map { |u| u.to_hash }
-          end
-        end
-      end
+      def friends_and_followers(*args)
+        options = args.extract_options!.merge(super_operation: :friends_and_followers)
 
-      def _followers_parallelly(*args)
-        options = args.extract_options!
-        instrument(__method__, nil, {super_operation: :followers}.merge(options)) do
-          opt = {super_operation: __method__}.merge(options)
-          users(follower_ids(*args, opt).map { |id| id.to_i }, opt)
-        end
+        following_ids, followed_ids = friend_ids_and_follower_ids(*args, options)
+        unique_ids = (following_ids + followed_ids).uniq
+        people = users_internal(unique_ids).index_by { |u| u[:id] }
+        [people.slice(*following_ids), people.slice(*followed_ids)]
+
+        # parallel(in_threads: 2) do |batch|
+        #   batch.friends(*args, options)
+        #   batch.followers(*args, options)
+        # end
       end
     end
   end

data/lib/twitter_with_auto_pagination/rest/lists.rb

@@ -5,22 +5,23 @@ module TwitterWithAutoPagination
     module Lists
       include TwitterWithAutoPagination::REST::Utils
 
+      # Returns the lists the specified user has been added to.
       def memberships(*args)
         options = {count: 1000, cursor: -1}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_cursor(method(__method__).super_method, *args, options)
-          end
+
+        collect_with_cursor do |next_cursor|
+          options[:next_cursor] = next_cursor unless next_cursor.nil?
+          twitter.send(:memberships, *args, options)
         end
       end
 
+      # Returns the members of the specified list.
       def list_members(*args)
         options = {count: 5000, skip_status: 1, cursor: -1}.merge(args.extract_options!)
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_cursor(method(__method__).super_method, *args, options)
-          end
+
+        collect_with_cursor do |next_cursor|
+          options[:next_cursor] = next_cursor unless next_cursor.nil?
+          twitter.send(:list_members, *args, options)
         end
       end
     end

data/lib/twitter_with_auto_pagination/rest/search.rb

@@ -5,12 +5,22 @@ module TwitterWithAutoPagination
     module Search
       include TwitterWithAutoPagination::REST::Utils
 
-      def search(*args)
-        options = {count: 100, result_type: :recent, call_limit: 1}.merge(args.extract_options!)
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_max_id(method(__method__).super_method, *args, options) { |response| response.attrs[:statuses] }.map { |s| s.to_hash }
-          end
+      MAX_TWEETS_PER_REQUEST = 100
+
+      %i(search).each do |name|
+        define_method(name) do |query, options = {}|
+          raise ArgumentError.new('specify a query') unless query.is_a?(String)
+
+          call_limit = calc_call_limit(options.delete(:count), MAX_TWEETS_PER_REQUEST)
+          options = {count: MAX_TWEETS_PER_REQUEST, result_type: :recent, call_count: 0, call_limit: call_limit}.merge(options)
+
+          collect_with_max_id do |max_id|
+            options[:max_id] = max_id unless max_id.nil?
+            options[:call_count] += 1
+            if options[:call_count] <= options[:call_limit]
+              twitter.send(name, query, options).attrs[:statuses].map { |s| Twitter::Tweet.new(s) }
+            end
+          end.map(&:attrs)
         end
       end
     end

data/lib/twitter_with_auto_pagination/rest/timelines.rb

@@ -5,34 +5,21 @@ module TwitterWithAutoPagination
     module Timelines
       include TwitterWithAutoPagination::REST::Utils
 
-      def home_timeline(*args)
-        mtd = __method__
-        options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
-        instrument(mtd, nil, options) do
-          fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
-            collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
-          end
-        end
-      end
+      MAX_TWEETS_PER_REQUEST = 200
 
-      def user_timeline(*args)
-        mtd = __method__
-        options = {count: 200, include_rts: true, call_limit: 3}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: mtd).id if args.empty?
-        instrument(mtd, nil, options) do
-          fetch_cache_or_call_api(mtd, args[0], options) do
-            collect_with_max_id(method(mtd).super_method, *args, options).map { |s| s.attrs }
-          end
-        end
-      end
+      %i(home_timeline user_timeline mentions_timeline).each do |name|
+        define_method(name) do |*args|
+          options = args.extract_options!.dup
+          call_limit = calc_call_limit(options.delete(:count), MAX_TWEETS_PER_REQUEST)
+          options = {count: MAX_TWEETS_PER_REQUEST, include_rts: true, call_count: 0, call_limit: call_limit}.merge(options)
 
-      def mentions_timeline(*args)
-        mtd = __method__
-        options = {count: 200, include_rts: true, call_limit: 1}.merge(args.extract_options!)
-        instrument(mtd, nil, options) do
-          fetch_cache_or_call_api(mtd, verify_credentials(super_operation: mtd).id, options) do
-            collect_with_max_id(method(mtd).super_method, options).map { |s| s.attrs }
-          end
+          collect_with_max_id do |max_id|
+            options[:max_id] = max_id unless max_id.nil?
+            options[:call_count] += 1
+            if options[:call_count] <= options[:call_limit]
+              twitter.send(name, *args, options)
+            end
+          end.map(&:attrs)
         end
       end
     end

data/lib/twitter_with_auto_pagination/rest/users.rb

@@ -6,56 +6,43 @@ module TwitterWithAutoPagination
     module Users
       include TwitterWithAutoPagination::REST::Utils
 
-      def verify_credentials(*args)
-        options = {skip_status: true}.merge(args.extract_options!)
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args) do
-            call_api(method(__method__).super_method, *args, options).to_hash
-          end
-        end
+      def verify_credentials(options = {})
+        twitter.send(__method__, {skip_status: true}.merge(options)).to_hash
       end
 
       def user?(*args)
-        options = args.extract_options!
-        return false if args.empty? || args[0].nil? || !args[0].to_s.match(/\A([a-zA-Z0-9_]{1,20}|[1-9][0-9]*)\z/)
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            call_api(method(__method__).super_method, *args, options)
-          end
-        end
+        twitter.send(__method__, *args)
       end
 
       def user(*args)
-        options = args.extract_options!
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            call_api(method(__method__).super_method, *args, options).to_hash
-          end
-        end
+        twitter.send(__method__, *args).to_hash
       end
 
-      # use compact, not use sort and uniq
-      # specify reduce: false to use tweet for inactive_*
-      # TODO Perhaps `old_users` automatically merges result...
-      def users(*args)
-        options = args.extract_options!
-        options[:reduce] = false
-        users_per_workers = args.first.compact.each_slice(100).to_a
-        processed_users = []
-        thread_size = [users_per_workers.size, 10].min
-
-        instrument(__method__, nil, options) do
-          Parallel.each_with_index(users_per_workers, in_threads: thread_size) do |users_per_worker, i|
-            _users = fetch_cache_or_call_api(__method__, users_per_worker, options) do
-              call_api(method(__method__).super_method, users_per_worker, options).map { |u| u.to_hash }
-            end
-
-            processed_users << {i: i, users: _users}
-          end
+      MAX_USERS_PER_REQUEST = 100
+
+      # client.users         -> cached
+      # users(internal call) -> cached
+      # super                -> not cached
+      def users(values, options = {})
+        if values.size <= MAX_USERS_PER_REQUEST
+          return twitter.send(__method__, *values, options).map(&:to_hash)
         end
 
-        processed_users.sort_by { |p| p[:i] }.map { |p| p[:users] }.flatten.compact
+        users_internal(values, options)
+      end
+
+      def blocked_ids(*args)
+        twitter.send(__method__, *args).attrs[:ids]
+      end
+
+      private
+
+      def users_internal(values, options = {})
+        options = options.merge(super_operation: :users)
+
+        parallel(in_threads: 10) do |batch|
+          values.each_slice(MAX_USERS_PER_REQUEST) { |targets| batch.users(targets, options) }
+        end.flatten
       end
     end
   end

data/lib/twitter_with_auto_pagination/rest/utils.rb

@@ -1,166 +1,46 @@
 require 'hashie'
 require 'digest/md5'
 
+require 'twitter_with_auto_pagination/collector'
+
 module TwitterWithAutoPagination
   module REST
     module Utils
-      def uid_or_screen_name?(object)
-        object.kind_of?(String) || object.kind_of?(Integer)
-      end
-
-      def authenticating_user?(target)
-        user.id.to_i == user(target).id.to_i
-      end
-
-      def authorized_user?(target)
-        target_user = user(target)
-        !target_user.protected? || friendship?(user.id.to_i, target_user.id.to_i)
-      end
-
-      def credentials_hash
-        Digest::MD5.hexdigest(access_token + access_token_secret + consumer_key + consumer_secret)
-      end
-
-      def instrument(operation, key, options = nil)
-        payload = {operation: operation}
-        payload.merge!(options) if options.is_a?(Hash)
-        ActiveSupport::Notifications.instrument('call.twitter_with_auto_pagination', payload) { yield(payload) }
-      end
-
-      def call_api(method, *args)
-        api_options = args.extract_options!
-        self.call_count += 1
-        options = {method: method.name, call_count: self.call_count, args: [*args, api_options]}
-        begin
-          instrument('request', nil, options) { method.call(*args, api_options) }
-        rescue Twitter::Error::TooManyRequests => e
-          logger.warn "#{__method__}: #{e.class} #{e.message} Retry after #{e.rate_limit.reset_in} seconds. #{options.inspect}"
-          raise e
-        rescue Twitter::Error::ServiceUnavailable, Twitter::Error::InternalServerError,
-          Twitter::Error::Forbidden, Twitter::Error::NotFound => e
-          logger.warn "#{__method__}: #{e.class} #{e.message} #{options.inspect}"
-          raise e
-        rescue => e
-          logger.warn "CATCH ME! #{__method__}: #{e.class} #{e.message} #{options.inspect}"
-          raise e
-        end
-      end
-
-      # user_timeline, search
-      def collect_with_max_id(method, *args)
-        options = args.extract_options!
-        call_limit = options.delete(:call_limit) || 3
-        return_data = []
-        call_num = 0
-
-        while call_num < call_limit
-          last_response = call_api(method, *args, options)
-          last_response = yield(last_response) if block_given?
-          call_num += 1
-          return_data += last_response
-          if last_response.nil? || last_response.empty?
-            break
-          else
-            options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
-          end
-        end
-
-        return_data
-      end
-
-      # friends, followers
-      def collect_with_cursor(method, *args)
-        options = args.extract_options!
-        call_limit = options.delete(:call_limit) || 30
-        return_data = []
-        call_num = 0
-
-        while call_num < call_limit
-          last_response = call_api(method, *args, options).attrs
-          call_num += 1
-          return_data += (last_response[:users] || last_response[:ids] || last_response[:lists])
-          options[:cursor] = last_response[:next_cursor]
-          if options[:cursor].nil? || options[:cursor] == 0
-            break
-          end
-        end
-
-        return_data
-      end
-
-      def normalize_key(method, user, options = {})
-        delim = ':'
-        identifier =
-          case
-            when method == :verify_credentials
-              "token-hash#{delim}#{credentials_hash}"
-            when method == :search
-              "str#{delim}#{user.to_s}"
-            when method == :list_members
-              "list_id#{delim}#{user.to_s}"
-            when method == :mentions_timeline
-              "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
-            when method == :home_timeline
-              "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
-            when method.in?([:users, :replying]) && options[:super_operation].present?
-              case
-                when user.kind_of?(Array) && user.first.kind_of?(Integer)
-                  "#{options[:super_operation]}-ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-                when user.kind_of?(Array) && user.first.kind_of?(String)
-                  "#{options[:super_operation]}-sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-                else
-                  raise "#{method.inspect} #{user.inspect}"
-              end
-            when user.kind_of?(Integer)
-              "id#{delim}#{user.to_s}"
-            when user.kind_of?(Array) && user.first.kind_of?(Integer)
-              "ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-            when user.kind_of?(Array) && user.first.kind_of?(String)
-              "sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-            when user.kind_of?(String)
-              "sn#{delim}#{user}"
-            when user.kind_of?(Twitter::User)
-              "user#{delim}#{user.id.to_s}"
-            else
-              raise "#{method.inspect} #{user.inspect}"
-          end
+     include TwitterWithAutoPagination::Collector
 
-        "#{method}#{delim}#{identifier}"
-      end
+     DEFAULT_CALL_LIMIT = 1
 
-      CODER = JSON
+     private
 
-      def encode(obj)
-        obj.in?([true, false]) ? obj : CODER.dump(obj)
-      end
+     def calc_call_limit(count, max_count)
+       return DEFAULT_CALL_LIMIT unless count
+       limit = count / max_count
+       limit += 1 if (count % max_count).nonzero?
+       limit
+     end
 
-      def decode(str)
-        obj = str.kind_of?(String) ? CODER.load(str) : str
-        to_mash(obj)
-      end
+     def uid_or_screen_name?(object)
+       object.kind_of?(String) || object.kind_of?(Integer)
+     end
 
-      def to_mash(obj)
-        case
-          when obj.kind_of?(Array)
-            obj.map { |o| to_mash(o) }
-          when obj.kind_of?(Hash)
-            Hashie::Mash.new(obj.map { |k, v| [k, to_mash(v)] }.to_h)
-          else
-            obj
-        end
-      end
+     def authenticating_user?(target)
+       user.id.to_i == user(target).id.to_i
+     end
 
-      def fetch_cache_or_call_api(method, user, options = {})
-        key = normalize_key(method, user, options)
+     def authorized_user?(target)
+       target_user = user(target)
+       !target_user.protected? || friendship?(user.id.to_i, target_user.id.to_i)
+     end
 
-        fetch_result =
-          cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
-            block_result = yield
-            instrument('serialize', nil, key: key, caller: method) { encode(block_result) }
-          end
+     def credentials_hash
+       Digest::MD5.hexdigest(access_token + access_token_secret + consumer_key + consumer_secret)
+     end
 
-        instrument('deserialize', nil, key: key, caller: method) { decode(fetch_result) }
-      end
+     def instrument(operation, name, options = nil)
+       payload = {operation: operation}
+       payload.merge!(options) if options.is_a?(Hash) && !options.empty?
+       ActiveSupport::Notifications.instrument("#{name.nil? ? operation : name}.twitter", payload) {yield(payload)}
+     end
     end
   end
 end

data/lib/twitter_with_auto_pagination/serializer.rb

@@ -0,0 +1,27 @@
+module TwitterWithAutoPagination
+  class Serializer
+    CODER = JSON
+
+    class << self
+      def encode(obj, options = {})
+        instrument(options) do
+          (!!obj == obj) ? obj : CODER.dump(obj)
+        end
+      end
+
+      def decode(str, options = {})
+        instrument(options) do
+          str.kind_of?(String) ? CODER.parse(str, symbolize_names: true) : str
+        end
+      end
+
+      private
+
+      def instrument(options, &block)
+        parent = caller[0][/`([^']*)'/, 1]
+        payload = {operation: parent, args: options[:args]}
+        ActiveSupport::Notifications.instrument("#{payload[:operation]}.twitter", payload) { yield(payload) }
+      end
+    end
+  end
+end

data/lib/twitter_with_auto_pagination.rb

@@ -2,43 +2,7 @@ require 'active_support'
 require 'active_support/core_ext'
 require 'twitter'
 
-require 'twitter_with_auto_pagination/log_subscriber'
+require 'twitter_with_auto_pagination/client'
 
 module TwitterWithAutoPagination
-end
-
-require 'twitter_with_auto_pagination/rest/api'
-
-module Twitter
-  module REST
-    class Client
-      prepend TwitterWithAutoPagination::REST::API
-
-      def initialize(options = {})
-        @cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache'))
-        @call_count = 0
-
-        logger =
-          if options.has_key?(:logger)
-            options.delete(:logger)
-          else
-            Dir.mkdir('log') unless File.exists?('log')
-            Logger.new('log/twitter_with_auto_pagination.log')
-          end
-        logger.level = options.has_key?(:log_level) ? options.delete(:log_level) : :debug
-        @@logger = @logger = logger
-
-        super
-      end
-
-      def self.logger
-        @@logger ||= nil
-      end
-
-      attr_accessor :call_count, :logger
-      attr_reader :cache
-
-      INDENT = 4
-    end
-  end
-end
+end