twitter_with_auto_pagination 0.8.12 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a5ffffb7a3903265ee83d5340491193424cf6b2
-  data.tar.gz: df27d9bc220d90ad97c28911ed67960233ba63dd
+  metadata.gz: ed91eaf1cf67bef358dda37cdd3886003fa80c0c
+  data.tar.gz: e91e410326935bb838a8c9c7d29f678d129b2417
 SHA512:
-  metadata.gz: 978c61c49f0fc373584cc3284483a3c775a0035499e2ba49b6271a5e78b8ea163ecc1a6438b682d540ee1e691f4461a78cbc813a2235869b619655777af74496
-  data.tar.gz: f6956150c2825aeae989067263e1eebf5d51df0dece18767239fedfa2b3970b899cfc51569f99f4e2f0fe671df7712081445636510633dd14d903a6c519db9e0
+  metadata.gz: 97ad05cdd642560bb57e5ef28f2d1b7fdbb492e9112dfdd0af8688fd7a3735f799a0d99957d97a28e7699d158724a14dccd57c3424b9b5964e68ead8990af020
+  data.tar.gz: f342f1131df4b7fe0410e5ad9b61879bb88a07077d08a4414e07828dea0fcb1a3ba6cec6ee9c638de382fb36b72dce4d36344e99d7a94562dfdf9d7c1eba3197

data/lib/twitter_with_auto_pagination/analysis/api.rb

@@ -0,0 +1,9 @@
+require 'twitter_with_auto_pagination/analysis/timelines'
+
+module TwitterWithAutoPagination
+  module Analysis
+    module API
+      include TwitterWithAutoPagination::Analysis::Timelines
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/{rest/uncategorized.rb → analysis/timelines.rb}

@@ -1,8 +1,8 @@
 require 'twitter_with_auto_pagination/rest/utils'
 
 module TwitterWithAutoPagination
-  module REST
-    module Uncategorized
+  module Analysis
+    module Timelines
       include TwitterWithAutoPagination::REST::Utils
 
       EVERY_DAY = (0..6)

data/lib/twitter_with_auto_pagination/cache.rb

@@ -0,0 +1,78 @@
+require 'forwardable'
+require 'digest/md5'
+
+require 'twitter_with_auto_pagination/serializer'
+
+module TwitterWithAutoPagination
+  class Cache
+    extend Forwardable
+
+    attr_reader :serializer, :client
+
+    def_delegators :@client, :clear, :cleanup
+
+    def initialize
+      @serializer = Serializer
+
+      path = File.join('tmp', 'twitter_cache')
+      Dir.mkdir(path) unless File.exists?(path)
+      @client = ActiveSupport::Cache::FileStore.new(path, expires_in: 1.hour, race_condition_ttl: 5.minutes)
+    end
+
+    def fetch(method, user, options = {}, &block)
+      key = normalize_key(method, user, options.except(:args))
+
+      block_result = nil
+      fetch_result =
+        @client.fetch(key) do
+          block_result = yield
+          serializer.encode(block_result, args: options[:args])
+        end
+
+      block_result ? block_result : serializer.decode(fetch_result, args: options[:args])
+    end
+
+    private
+
+    DELIM = ':'
+
+    def normalize_key(method, user, options = {})
+      identifier =
+        case
+          when method == :search                    then "query#{DELIM}#{user}"
+          when method == :friendship?               then "from#{DELIM}#{user[0]}#{DELIM}to#{DELIM}#{user[1]}"
+          when method == :list_members              then "list_id#{DELIM}#{user}"
+          when user.nil? && options[:hash].present? then "token-hash#{DELIM}#{options[:hash]}"
+          else user_identifier(user)
+        end
+
+      "#{method}#{DELIM}#{identifier}#{DELIM}#{options_identifier(options)}"
+    end
+
+    def user_identifier(user)
+      case
+        when user.kind_of?(Integer)                            then "id#{DELIM}#{user}"
+        when user.kind_of?(String)                             then "screen_name#{DELIM}#{user}"
+        when user.kind_of?(Array) && user.empty?               then 'The_#users_is_called_with_an_empty_array'
+        when user.kind_of?(Array) && user[0].kind_of?(Integer) then "ids#{DELIM}#{user.size}-#{hexdigest(user.join(','))}"
+        when user.kind_of?(Array) && user[0].kind_of?(String)  then "screen_names#{DELIM}#{user.size}-#{hexdigest(user.join(','))}"
+        else raise "#{__method__}: No matches #{user.inspect}"
+      end
+    end
+
+    def options_identifier(options)
+      options = options.except(:hash, :call_count, :call_limit, :super_operation)
+      str =
+        if options.empty?
+          'empty'
+        else
+          options.map { |k, v| "#{k}#{DELIM}#{v}" }.join(',')
+        end
+      "options#{DELIM}#{str}"
+    end
+
+    def hexdigest(str)
+      Digest::MD5.hexdigest(str)
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/caching_and_logging.rb

@@ -0,0 +1,80 @@
+module TwitterWithAutoPagination
+  module CachingAndLogging
+    %i(
+        verify_credentials
+        user?
+        user
+        users
+        blocked_ids
+        home_timeline
+        user_timeline
+        mentions_timeline
+        search
+        favorites
+        friendship?
+        friend_ids
+        follower_ids
+        memberships
+        list_members
+      ).each do |name|
+      define_method(name) do |*args|
+        options = args.extract_options!
+
+        Instrumenter.api_call(name, options) do
+          do_request =
+            proc { Instrumenter.perform_request(name, options) { options.empty? ? super(*args) : super(*args, options) } }
+
+          if Utils.cache_disabled?(options)
+            do_request.call
+          else
+            user = name == :friendship? ? args[0, 2] : args[0]
+            cache.fetch(name, user, options.merge(args: [name, options], hash: credentials_hash), &do_request)
+          end
+        end
+      end
+    end
+
+    # Cached in #users
+    %i(friends followers friends_and_followers).each do |name|
+      define_method(name) do |*args|
+        options = args.extract_options!
+        Instrumenter.api_call(name, options) { super(*args, options) }
+      end
+    end
+
+    module Instrumenter
+
+      module_function
+
+      def api_call(operation, options)
+        payload = {operation: operation}.merge(options)
+        ActiveSupport::Notifications.instrument('api_call.twitter', payload) { yield(payload) }
+      end
+
+      def perform_request(caller, options, &block)
+        payload = {operation: 'request', args: [caller, options]}
+        ActiveSupport::Notifications.instrument('request.twitter', payload) { yield(payload) }
+      end
+    end
+
+    module Utils
+
+      module_function
+
+      def cache_disabled?(options)
+        options.is_a?(Hash) && options.has_key?(:cache) && !options[:cache]
+      end
+
+      def to_mash(obj)
+        case
+          when obj.kind_of?(Array)
+            obj.map {|o| to_mash(o)}
+          when obj.kind_of?(Hash)
+            Hashie::Mash.new(obj.map {|k, v| [k, to_mash(v)]}.to_h)
+          else
+            obj
+        end
+      end
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/client.rb

@@ -0,0 +1,57 @@
+require 'forwardable'
+
+require 'twitter_with_auto_pagination/logger'
+require 'twitter_with_auto_pagination/cache'
+require 'twitter_with_auto_pagination/log_subscriber'
+require 'twitter_with_auto_pagination/analysis/api'
+require 'twitter_with_auto_pagination/rest/api'
+require 'twitter_with_auto_pagination/rate_limit'
+require 'twitter_with_auto_pagination/parallel'
+
+module TwitterWithAutoPagination
+  class Client
+    extend Forwardable
+
+    attr_reader :cache, :twitter
+
+    def_delegators :@twitter, :perform_get, :access_token, :access_token_secret, :consumer_key, :consumer_secret
+
+    include TwitterWithAutoPagination::RateLimit
+    include TwitterWithAutoPagination::Parallel
+    include TwitterWithAutoPagination::REST::API
+    include TwitterWithAutoPagination::Analysis::API
+
+    def initialize(*args)
+      options = args.extract_options!
+
+      @cache = TwitterWithAutoPagination::Cache.new
+      Logging.logger = logger = TwitterWithAutoPagination::Logger.new(options)
+
+      unless subscriber_attached?
+        @@subscriber_attached = true
+        if logger.debug?
+          # Super slow
+          TwitterWithAutoPagination::AllLogSubscriber.attach_to :twitter
+          TwitterWithAutoPagination::ASLogSubscriber.attach_to :active_support
+        elsif logger.info?
+          TwitterWithAutoPagination::ApiCallLogSubscriber.attach_to :twitter
+        end
+      end
+
+      @twitter = Twitter::REST::Client.new(options)
+    end
+
+    def self.logger
+      Logging.logger
+    end
+
+    def subscriber_attached?
+      @@subscriber_attached ||= false
+    end
+
+    # Deprecated
+    def call_count
+      -1
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/collector.rb

@@ -0,0 +1,18 @@
+module TwitterWithAutoPagination
+  module Collector
+
+    def collect_with_max_id(collection = [], max_id = nil, &block)
+      tweets = yield(max_id)
+      return collection if tweets.nil?
+      collection += tweets
+      tweets.empty? ? collection.flatten : collect_with_max_id(collection, tweets.last.id - 1, &block)
+    end
+
+    def collect_with_cursor(collection = [], cursor = nil, &block)
+      response = yield(cursor)
+      return collection if response.nil?
+      collection += (response.attrs[:ids] || response.attrs[:users] || response.attrs[:lists]) # TODO to_a
+      response.attrs[:next_cursor].zero? ? collection.flatten : collect_with_cursor(collection, response.attrs[:next_cursor], &block)
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/log_subscriber.rb

@@ -1,57 +1,90 @@
 module TwitterWithAutoPagination
-  class LogSubscriber < ActiveSupport::LogSubscriber
+  module Logging
+    def truncated_payload(payload)
+      return payload.inspect if !payload.has_key?(:args) || !payload[:args].is_a?(Array) || payload[:args].empty? || !payload[:args][0].is_a?(Array)
 
-    def initialize
-      super
+      args = payload[:args].dup
+      args[0] =
+        if args[0].size > 3
+          "[#{args[0].take(3).join(', ')} ... #{args[0].size}]"
+        else
+          args[0].inspect
+        end
+
+      {args: args}.merge(payload.except(:args)).inspect
     end
 
-    def call(event)
-      return unless logger.debug?
+    module_function
 
-      payload = event.payload
-      name = "TW::#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
-      name = color(name, CYAN, true) # WHITE, RED, GREEN, BLUE, CYAN, MAGENTA, YELLOW
-      debug { "#{name} #{(payload.inspect)}" }
+    def logger
+      @@logger
     end
 
-    private
+    def logger=(logger)
+      @@logger = logger
+    end
+  end
 
-    def logger
-      Twitter::REST::Client.logger
+  class ApiCallLogSubscriber < ActiveSupport::LogSubscriber
+    include Logging
+
+    def api_call(event)
+      payload = event.payload
+      name = "TW::#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
+      name = color(name, YELLOW, true) # WHITE, RED, GREEN, BLUE, CYAN, MAGENTA, YELLOW
+      info { "#{name}#{" #{truncated_payload(payload)}" unless payload.empty?}" }
     end
   end
 
-  class ASLogSubscriber < ActiveSupport::LogSubscriber
+  class AllLogSubscriber < ApiCallLogSubscriber
+    include Logging
 
-    def initialize
+    def api_call(*args)
       super
     end
 
+    def twitter_any(event)
+      payload = event.payload
+      payload.delete(:name)
+      name = "TW::#{payload.delete(:operation)} (#{event.duration.round(1)}ms)"
+      name = color(name, CYAN, true) # WHITE, RED, GREEN, BLUE, CYAN, MAGENTA, YELLOW
+      debug { "#{name}#{" #{truncated_payload(payload)}" unless payload.empty?}" }
+    end
+
+    %w(request encode decode to_mash).each do |operation|
+      class_eval <<-METHOD, __FILE__, __LINE__ + 1
+        def #{operation}(event)
+          event.payload[:name] = '#{operation}'
+          twitter_any(event)
+        end
+      METHOD
+    end
+  end
+
+  class ASLogSubscriber < ActiveSupport::LogSubscriber
     def cache_any(event)
       return unless logger.debug?
 
       payload = event.payload
-      name= "AS::#{payload.delete(:name)} (#{event.duration.round(1)}ms)"
+      name = "AS::#{payload[:name]} (#{event.duration.round(1)}ms)"
       name = color(name, MAGENTA, true)
-      debug { "#{name} #{(payload.inspect)}" }
+      debug { "#{name} #{(payload.except(:name, :expires_in, :race_condition_ttl).inspect)}" }
     end
 
-    %w(read write fetch_hit generate delete exist?).each do |operation|
+    # Ignore generate and fetch_hit
+    %w(read write delete exist?).each do |operation|
       class_eval <<-METHOD, __FILE__, __LINE__ + 1
-          def cache_#{operation}(event)
-            event.payload[:name] = '#{operation}'
-            cache_any(event)
-          end
+        def cache_#{operation}(event)
+          event.payload[:name] = '#{operation}'
+          cache_any(event)
+        end
       METHOD
     end
 
     private
 
     def logger
-      Twitter::REST::Client.logger
+      Logging.logger
     end
   end
-end
-
-TwitterWithAutoPagination::LogSubscriber.attach_to :twitter_with_auto_pagination
-TwitterWithAutoPagination::ASLogSubscriber.attach_to :active_support
+end

data/lib/twitter_with_auto_pagination/logger.rb

@@ -0,0 +1,10 @@
+module TwitterWithAutoPagination
+  class Logger < ::Logger
+
+    def initialize(options = {})
+      Dir.mkdir('log') unless File.exists?('log')
+      super('log/twitter.log')
+      self.level = options.has_key?(:log_level) ? options.delete(:log_level) : :debug
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/parallel.rb

@@ -0,0 +1,25 @@
+require 'parallel'
+
+module TwitterWithAutoPagination
+  module Parallel
+
+    def parallel(options = {}, &block)
+      batch = Arguments.new
+      yield(batch)
+
+      in_threads = options.fetch(:in_threads, batch.size)
+
+      ::Parallel.map_with_index(batch, in_threads: in_threads) do |args, i|
+        {i: i, result: send(*args)}
+      end.sort_by { |q| q[:i] }.map { |q| q[:result] }
+    end
+
+    class Arguments < Array
+      %i(users friend_ids follower_ids friends followers).each do |name|
+        define_method(name) do |*args|
+          send(:<< , [name, *args])
+        end
+      end
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/rate_limit.rb

@@ -0,0 +1,56 @@
+module TwitterWithAutoPagination
+  module RateLimit
+    def rate_limit
+      RateLimit.new(perform_get('/1.1/application/rate_limit_status.json')) rescue nil
+    end
+
+    class RateLimit
+      def initialize(status)
+        @status = status
+      end
+
+      def resources
+        @status[:resources]
+      end
+
+      def verify_credentials
+        extract_remaining_and_reset_in(resources[:account][:'/account/verify_credentials'])
+      end
+
+      def friend_ids
+        extract_remaining_and_reset_in(resources[:friends][:'/friends/ids'])
+      end
+
+      def follower_ids
+        extract_remaining_and_reset_in(resources[:followers][:'/followers/ids'])
+      end
+
+      def users
+        extract_remaining_and_reset_in(resources[:users][:'/users/lookup'])
+      end
+
+      def to_h
+        {
+          verify_credentials: verify_credentials,
+          friend_ids: friend_ids,
+          follower_ids: follower_ids,
+          users: users
+        }
+      end
+
+      def inspect
+        'verify_credentials ' + verify_credentials.inspect +
+          ' friend_ids ' + friend_ids.inspect +
+          ' follower_ids ' + follower_ids.inspect +
+          ' users ' + users.inspect
+      end
+
+      private
+
+      def extract_remaining_and_reset_in(limit)
+        {remaining: limit[:remaining], reset_in: (Time.at(limit[:reset]) - Time.now).round}
+      end
+    end
+  end
+end
+

data/lib/twitter_with_auto_pagination/rest/api.rb

@@ -4,14 +4,15 @@ require 'twitter_with_auto_pagination/rest/lists'
 require 'twitter_with_auto_pagination/rest/search'
 require 'twitter_with_auto_pagination/rest/timelines'
 require 'twitter_with_auto_pagination/rest/users'
-require 'twitter_with_auto_pagination/rest/uncategorized'
 
-require 'twitter_with_auto_pagination/rest/extension/clusters'
-require 'twitter_with_auto_pagination/rest/extension/favoriting'
-require 'twitter_with_auto_pagination/rest/extension/friends_and_followers'
-require 'twitter_with_auto_pagination/rest/extension/replying'
-require 'twitter_with_auto_pagination/rest/extension/unfollowing'
-require 'twitter_with_auto_pagination/rest/extension/users'
+# require 'twitter_with_auto_pagination/rest/extension/clusters'
+# require 'twitter_with_auto_pagination/rest/extension/favoriting'
+# require 'twitter_with_auto_pagination/rest/extension/friends_and_followers'
+# require 'twitter_with_auto_pagination/rest/extension/replying'
+# require 'twitter_with_auto_pagination/rest/extension/unfollowing'
+# require 'twitter_with_auto_pagination/rest/extension/users'
+
+require 'twitter_with_auto_pagination/caching_and_logging'
 
 module TwitterWithAutoPagination
   module REST
@@ -22,14 +23,15 @@ module TwitterWithAutoPagination
       include TwitterWithAutoPagination::REST::Search
       include TwitterWithAutoPagination::REST::Timelines
       include TwitterWithAutoPagination::REST::Users
-      include TwitterWithAutoPagination::REST::Uncategorized
 
-      include TwitterWithAutoPagination::REST::Extension::Clusters
-      include TwitterWithAutoPagination::REST::Extension::Favoriting
-      include TwitterWithAutoPagination::REST::Extension::FriendsAndFollowers
-      include TwitterWithAutoPagination::REST::Extension::Replying
-      include TwitterWithAutoPagination::REST::Extension::Unfollowing
-      include TwitterWithAutoPagination::REST::Extension::Users
+      # include TwitterWithAutoPagination::REST::Extension::Clusters
+      # include TwitterWithAutoPagination::REST::Extension::Favoriting
+      # include TwitterWithAutoPagination::REST::Extension::FriendsAndFollowers
+      # include TwitterWithAutoPagination::REST::Extension::Replying
+      # include TwitterWithAutoPagination::REST::Extension::Unfollowing
+      # include TwitterWithAutoPagination::REST::Extension::Users
+
+      include TwitterWithAutoPagination::CachingAndLogging
     end
   end
 end

data/lib/twitter_with_auto_pagination/rest/extension/friends_and_followers.rb

@@ -17,13 +17,6 @@ module TwitterWithAutoPagination
           result
         end
 
-        def friends_and_followers(*args)
-          _fetch_parallelly(
-            [
-              {method: :friends, args: args},
-              {method: :followers, args: args}])
-        end
-
         def friends_followers_and_statuses(*args)
           _fetch_parallelly(
             [

data/lib/twitter_with_auto_pagination/rest/favorites.rb

@@ -5,13 +5,19 @@ module TwitterWithAutoPagination
     module Favorites
       include TwitterWithAutoPagination::REST::Utils
 
-      def favorites(*args)
-        options = {count: 100, call_limit: 3}.merge(args.extract_options!)
-        args[0] = verify_credentials(super_operation: __method__).id if args.empty?
-        instrument(__method__, nil, options) do
-          fetch_cache_or_call_api(__method__, args[0], options) do
-            collect_with_max_id(method(__method__).super_method, *args, options).map { |s| s.attrs }
-          end
+      MAX_TWEETS_PER_REQUEST = 100
+
+      %i(favorites).each do |name|
+        define_method(name) do |*args|
+          options = args.extract_options!
+          call_limit = calc_call_limit(options.delete(:count), MAX_TWEETS_PER_REQUEST)
+          options = {count: MAX_TWEETS_PER_REQUEST, result_type: :recent, call_count: 0, call_limit: call_limit}.merge(options)
+
+          collect_with_max_id do |max_id|
+            options[:max_id] = max_id unless max_id.nil?
+            options[:call_count] += 1
+            twitter.send(name, *args, options) if options[:call_count] <= options[:call_limit]
+          end.map(&:attrs)
         end
       end
     end