twitter_friendly 0.1.0 → 0.2.0

data/lib/twitter_friendly/rest/api.rb

@@ -0,0 +1,34 @@
+require 'twitter_friendly/rest/utils'
+require 'twitter_friendly/rest/collector'
+require "twitter_friendly/rest/parallel"
+require "twitter_friendly/rest/base"
+require 'twitter_friendly/rest/friends_and_followers'
+require 'twitter_friendly/rest/users'
+require 'twitter_friendly/rest/timelines'
+require 'twitter_friendly/rest/search'
+require 'twitter_friendly/rest/favorites'
+require 'twitter_friendly/rest/lists'
+require 'twitter_friendly/rest/tweets'
+
+require 'twitter_friendly/caching'
+
+module TwitterFriendly
+  module REST
+    module API
+      include TwitterFriendly::REST::Utils
+      include TwitterFriendly::REST::Collector
+      include TwitterFriendly::REST::Parallel
+      include TwitterFriendly::REST::Base
+      include TwitterFriendly::REST::FriendsAndFollowers
+      include TwitterFriendly::REST::Users
+      include TwitterFriendly::REST::Timelines
+      include TwitterFriendly::REST::Search
+      include TwitterFriendly::REST::Favorites
+      include TwitterFriendly::REST::Lists
+      include TwitterFriendly::REST::Tweets
+
+      include TwitterFriendly::Caching
+      include TwitterFriendly::REST::Collector::Caching
+    end
+  end
+end

data/lib/twitter_friendly/rest/base.rb

@@ -0,0 +1,32 @@
+module TwitterFriendly
+  module REST
+    module Base
+      def fetch_tweets_with_max_id(name, args, max_count)
+        options = args.extract_options!
+        total_count = options.delete(:count) || max_count
+        call_count = total_count / max_count + (total_count % max_count == 0 ? 0 : 1)
+        options[:count] = max_count
+
+        collect_with_max_id(args[0], [], nil, {super_operation: name}.merge(options)) do |max_id|
+          options[:max_id] = max_id unless max_id.nil?
+          if (call_count -= 1) >= 0
+            if name == :search
+              @twitter.send(name, *args, options).attrs[:statuses]
+            else
+              @twitter.send(name, *args, options).map(&:attrs)
+            end
+          end
+        end
+      end
+
+      def fetch_resources_with_cursor(name, args)
+        options = args.extract_options!
+
+        collect_with_cursor(args[0], [], -1, {super_operation: name}.merge(options)) do |next_cursor|
+          options[:cursor] = next_cursor unless next_cursor.nil?
+          @twitter.send(name, *args, options)
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/collector.rb

@@ -0,0 +1,78 @@
+module TwitterFriendly
+  module REST
+    module Collector
+      def collect_with_max_id(user, collection, max_id, options, &block)
+        fetch_options = options.dup
+        fetch_options[:max_id] = max_id
+        fetch_options.merge!(args: [__method__, fetch_options], hash: credentials_hash)
+
+        # TODO Handle {cache: false} option
+        tweets =
+            @cache.fetch(__method__, user, fetch_options) do
+              Instrumenter.perform_request(args: [__method__, max_id: max_id, super_operation: options[:super_operation]]) do
+                yield(max_id)
+              end
+            end
+        return collection if tweets.nil?
+
+        options[:recursive] = true
+
+        collection.concat tweets
+        tweets.empty? ? collection.flatten : collect_with_max_id(user, collection, tweets.last[:id] - 1, options, &block)
+      end
+
+      def collect_with_cursor(user, collection, cursor, options, &block)
+        fetch_options = options.dup
+        fetch_options[:cursor] = cursor
+        fetch_options.merge!(args: [__method__, fetch_options], hash: credentials_hash)
+
+        # TODO Handle {cache: false} option
+        response =
+            @cache.fetch(__method__, user, fetch_options) do
+              Instrumenter.perform_request(args: [__method__, cursor: cursor, super_operation: options[:super_operation]]) do
+                yield(cursor).attrs
+              end
+            end
+        return collection if response.nil?
+
+        options[:recursive] = true
+
+        # Notice: If you call response.to_a, it automatically fetch all results and the results are not cached.
+        collection.concat (response[:ids] || response[:users] || response[:lists])
+        response[:next_cursor].zero? ? collection.flatten : collect_with_cursor(user, collection, response[:next_cursor], options, &block)
+      end
+
+      module Instrumenter
+
+        module_function
+
+        # 他のメソッドと違い再帰的に呼ばれるため、全体をキャッシュすると、すべてを再帰的にキャッシュしてしまう。
+        # それを防ぐために、特別にここでキャッシュの処理を登録している。
+
+        def perform_request(options, &block)
+          payload = {operation: 'collect', args: options[:args]}
+          ::ActiveSupport::Notifications.instrument('collect.twitter_friendly', payload) { yield(payload) }
+        end
+      end
+
+      module Caching
+        %i(
+          collect_with_max_id
+          collect_with_cursor
+        ).each do |name|
+          define_method(name) do |*args, &block|
+            options = args.extract_options!
+            do_request = Proc.new { options.empty? ? super(*args, &block) : super(*args, options, &block) }
+
+            if options[:recursive]
+              do_request.call
+            else
+              TwitterFriendly::Caching::Instrumenter.start_processing(name, options)
+              TwitterFriendly::Caching::Instrumenter.complete_processing(name, options, &do_request)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/favorites.rb

@@ -0,0 +1,15 @@
+module TwitterFriendly
+  module REST
+    module Favorites
+
+      MAX_TWEETS_PER_REQUEST = 100
+
+      %i(favorites).each do |name|
+        define_method(name) do |*args|
+          args << {result_type: :recent}.merge(args.extract_options!)
+          fetch_tweets_with_max_id(name, args, MAX_TWEETS_PER_REQUEST)
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/friends_and_followers.rb

@@ -0,0 +1,64 @@
+module TwitterFriendly
+  module REST
+    module FriendsAndFollowers
+      def friendship?(from, to, options = {})
+        @twitter.send(__method__, from, to, options)
+      end
+
+      MAX_IDS_PER_REQUEST = 5000
+
+      %i(friend_ids follower_ids).each do |name|
+        define_method(name) do |*args|
+          options = {count: MAX_IDS_PER_REQUEST}.merge(args.extract_options!)
+          if options[:super_operation]
+            options[:super_super_operation] = options[:super_operation]
+            options[:super_operation] = name
+          end
+          args << options
+          fetch_resources_with_cursor(name, args)
+        end
+      end
+
+      def friends(*args)
+        options = args.extract_options!.merge(super_operation: :friends)
+        ids = friend_ids(*args, options)
+        users(ids, options)
+      end
+
+      def followers(*args)
+        options = args.extract_options!.merge(super_operation: :followers)
+        ids = follower_ids(*args, options)
+        users(ids, options)
+      end
+
+      def friend_ids_and_follower_ids(*args)
+        options = {super_operation: :friend_ids_and_follower_ids, parallel: true}.merge(args.extract_options!)
+
+        if options[:parallel]
+          require 'parallel'
+
+          parallel(in_threads: 2) do |batch|
+            batch.friend_ids(*args, options)
+            batch.follower_ids(*args, options)
+          end
+        else
+          [friend_ids(*args, options), follower_ids(*args, options)]
+        end
+      end
+
+      def friends_and_followers(*args)
+        options = args.extract_options!.merge(super_operation: :friends_and_followers)
+
+        following_ids, followed_ids = friend_ids_and_follower_ids(*args, options)
+        unique_ids = (following_ids + followed_ids).uniq
+        people = _users(unique_ids).index_by { |u| u[:id] }
+        [people.slice(*following_ids).values, people.slice(*followed_ids).values]
+
+        # parallel(in_threads: 2) do |batch|
+        #   batch.friends(*args, options)
+        #   batch.followers(*args, options)
+        # end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/lists.rb

@@ -0,0 +1,22 @@
+module TwitterFriendly
+  module REST
+    module Lists
+
+      MAX_LISTS_PER_REQUEST = 1000
+
+      # Returns the lists the specified user has been added to.
+      def memberships(*args)
+        args << {count: MAX_LISTS_PER_REQUEST}.merge(args.extract_options!)
+        fetch_resources_with_cursor(__method__, args)
+      end
+
+      MAX_MEMBERS_PER_REQUEST = 5000
+
+      # Returns the members of the specified list.
+      def list_members(*args)
+        args << {count: MAX_MEMBERS_PER_REQUEST, skip_status: 1}.merge(args.extract_options!)
+        fetch_resources_with_cursor(__method__, args)
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/parallel.rb

@@ -0,0 +1,30 @@
+module TwitterFriendly
+  module REST
+    module Parallel
+      def parallel(options = {}, &block)
+        batch = Arguments.new
+        yield(batch)
+
+        in_threads = options.fetch(:in_threads, batch.size)
+
+        ::Parallel.map_with_index(batch, in_threads: in_threads) do |args, i|
+          {i: i, result: send(*args)} # Cached here
+        end.sort_by { |q| q[:i] }.map { |q| q[:result] }
+      end
+
+      class Arguments < Array
+        %i(
+        users
+        friend_ids
+        follower_ids
+        friends
+        followers
+      ).each do |name|
+          define_method(name) do |*args|
+            send(:<< , [name, *args])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/search.rb

@@ -0,0 +1,16 @@
+module TwitterFriendly
+  module REST
+    module Search
+
+      MAX_TWEETS_PER_REQUEST = 100
+
+      %i(search).each do |name|
+        define_method(name) do |query, options = {}|
+          raise ArgumentError.new('You must specify a search query.') unless query.is_a?(String)
+          args = [query, {result_type: :recent}.merge(options)]
+          fetch_tweets_with_max_id(name, args, MAX_TWEETS_PER_REQUEST)
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/timelines.rb

@@ -0,0 +1,15 @@
+module TwitterFriendly
+  module REST
+    module Timelines
+
+      MAX_TWEETS_PER_REQUEST = 200
+
+      %i(home_timeline user_timeline mentions_timeline).each do |name|
+        define_method(name) do |*args|
+          args << {include_rts: true}.merge(args.extract_options!)
+          fetch_tweets_with_max_id(name, args, MAX_TWEETS_PER_REQUEST)
+        end
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/tweets.rb

@@ -0,0 +1,13 @@
+module TwitterFriendly
+  module REST
+    module Tweets
+
+      MAX_IDS_PER_REQUEST = 100
+
+      def retweeters_ids(*args)
+        args << {count: MAX_IDS_PER_REQUEST}.merge(args.extract_options!)
+        fetch_resources_with_cursor(__method__, args)
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/users.rb

@@ -0,0 +1,49 @@
+module TwitterFriendly
+  module REST
+    module Users
+      def verify_credentials(options = {})
+        @twitter.send(__method__, {skip_status: true}.merge(options))&.to_hash
+      end
+
+      def user?(*args)
+        @twitter.send(__method__, *args)
+      end
+
+      def user(*args)
+        @twitter.send(__method__, *args)&.to_hash
+      end
+
+      MAX_USERS_PER_REQUEST = 100
+
+      def users(values, options = {})
+        if values.size <= MAX_USERS_PER_REQUEST
+          @twitter.send(__method__, values, options)&.compact&.map(&:to_hash)
+        else
+          _users(values, options)
+        end
+      end
+
+      def blocked_ids(*args)
+        @twitter.send(__method__, *args)&.attrs&.fetch(:ids)
+      end
+
+      private
+
+      def _users(values, options = {})
+        options = {super_operation: :users, parallel: true}.merge(options)
+
+        if options[:parallel]
+          require 'parallel'
+
+          parallel(in_threads: 10) do |batch|
+            values.each_slice(MAX_USERS_PER_REQUEST) { |targets| batch.users(targets, options) }
+          end.flatten
+        else
+          values.each_slice(MAX_USERS_PER_REQUEST).map do |targets|
+            @twitter.send(:users, targets, options)
+          end
+        end&.flatten&.compact&.map(&:to_hash)
+      end
+    end
+  end
+end

data/lib/twitter_friendly/rest/utils.rb

@@ -0,0 +1,11 @@
+require 'digest/md5'
+
+module TwitterFriendly
+  module REST
+    module Utils
+      def credentials_hash
+        Digest::MD5.hexdigest(access_token + access_token_secret + consumer_key + consumer_secret)
+      end
+    end
+  end
+end

data/lib/twitter_friendly/serializer.rb

@@ -0,0 +1,79 @@
+require 'json'
+require 'oj'
+
+module TwitterFriendly
+  class Serializer
+    class << self
+      def encode(obj, options = {})
+        Instrumenter.perform_encode(options) do
+          (!!obj == obj) ? obj : coder.encode(obj)
+        end
+      end
+
+      def decode(str, options = {})
+        Instrumenter.perform_decode(options) do
+          str.kind_of?(String) ? coder.decode(str) : str
+        end
+      end
+
+      def coder
+        @@coder ||= Coder.instance
+      end
+
+      def coder=(coder)
+        @@coder = Coder.instance(coder)
+      end
+    end
+
+    private
+
+    module Instrumenter
+
+      module_function
+
+      def perform_encode(options, &block)
+        payload = {operation: 'encode', args: options[:args]}
+        ::ActiveSupport::Notifications.instrument('encode.twitter_friendly', payload) { yield(payload) }
+      end
+
+      def perform_decode(options, &block)
+        payload = {operation: 'decode', args: options[:args]}
+        ::ActiveSupport::Notifications.instrument('decode.twitter_friendly', payload) { yield(payload) }
+      end
+    end
+
+    class Coder
+      def initialize(coder)
+        @coder = coder
+      end
+
+      def encode(obj)
+        @coder.dump(obj)
+      end
+
+      def self.instance(coder = nil)
+        if coder.nil? && defined?(Oj)
+          OjCoder.new(Oj)
+        else
+          JsonCoder.new(coder)
+        end
+      end
+    end
+
+    class JsonCoder < Coder
+      def decode(str)
+        @coder.parse(str, symbolize_names: true)
+      end
+    end
+
+    class OjCoder < Coder
+      def encode(obj)
+        @coder.dump(obj, mode: :compat)
+      end
+
+      def decode(str)
+        @coder.load(str, symbol_keys: true)
+      end
+    end
+  end
+end