twitter_with_auto_pagination 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5ac0a96a8ef591cb272f5760baf4e5b931638bd
-  data.tar.gz: e1172d7a72e4792347f761f34dd9d516e13f5a23
+  metadata.gz: 4c889d340f03d3c1ef475e99fb2ca4860a809312
+  data.tar.gz: 216b8dd6601a1417778a6e6fa70af3b8b5c97493
 SHA512:
-  metadata.gz: d5b734f49a0d8c625a113a52373399ca91d45d77582a5ad1f4966e320225ffc161d2f59ee3d615c0d46ca318666a536ce62ac96cdddd52e8da1ab118c8954c17
-  data.tar.gz: 5bfe0d7c34fff554f3253b8c2144065f7a83def1c6a403a43134146e8f215b14e97c1c33b1ad36867f521ae7f005cc2d5c03f896346e879f55d2a224654226b7
+  metadata.gz: 53a6aae09286af007d3e5dd857b42263f3173ee4da5ce9b1b8392a16a6da82db6b4f265612d92a3b0b98406d4ef85ad59122eee90aed0284a091a9f335d78509
+  data.tar.gz: 3ad068e546f28b445f83654e7603f053a96e4d3c192aec2365d6ccdafff2ffd87414caa7a72edef75b5b3371bef69589f97771614902871ef257f867ae6a4868

data/README.md

@@ -1,8 +1,8 @@
 twitter-with-auto-pagination
 ============================
 
-[![Gem Version](https://badge.fury.io/rb/ex_twitter.png)](https://badge.fury.io/rb/twitter_with_auto_pagination)
-[![Build Status](https://travis-ci.org/ts-3156/ex-twitter.svg?branch=master)](https://travis-ci.org/ts-3156/twitter-with-auto-pagination)
+[![Gem Version](https://badge.fury.io/rb/twitter_with_auto_pagination.png)](https://badge.fury.io/rb/twitter_with_auto_pagination)
+[![Build Status](https://travis-ci.org/ts-3156/twitter-with-auto-pagination.svg?branch=master)](https://travis-ci.org/ts-3156/twitter-with-auto-pagination)
 
 Add auto pagination, auto caching and parallelly fetching features to Twitter gem.
 
@@ -26,16 +26,14 @@ Add `twitter_with_auto_pagination` to your Gemfile, and bundle.
 
 ## Configuration
 
-You can pass configuration options as a block to `TwitterWithAutoPagination::Client.new` just like `Twitter::REST::Client.new`.
+You can pass configuration options as a block to `Twitter::REST::Client.new` just like Twitter gem.
 
 ```
-client = TwitterWithAutoPagination::Client.new do |config|
+client = Twitter::REST::Client.new do |config|
   config.consumer_key        = "YOUR_CONSUMER_KEY"
   config.consumer_secret     = "YOUR_CONSUMER_SECRET"
   config.access_token        = "YOUR_ACCESS_TOKEN"
   config.access_token_secret = "YOUR_ACCESS_SECRET"
-  config.log_level           = :debug             # optional
-  config.logger              = Logger.new(STDOUT) # optional
 end
 ```
 
@@ -135,25 +133,17 @@ client.close_friends
 ```
 
 ```
-client.users_which_you_removed(pre_me, cur_me)
+client.removing(past_me, cur_me)
 ```
 
 ```
-client.users_who_removed_you(pre_me, cur_me)
+client.removed(past_me, cur_me)
 ```
 
 ```
-client.users_which_you_replied_to
+client.replying
 ```
 
 ```
-client.users_who_replied_to_you
-```
-
-```
-client.users_which_you_faved
-```
-
-```
-client.users_who_faved_you
+client.replied
 ```

data/lib/twitter_with_auto_pagination.rb

@@ -1,4 +1,45 @@
-require 'twitter_with_auto_pagination/client'
+require 'twitter'
+
+require 'twitter_with_auto_pagination/log_subscriber'
 
 module TwitterWithAutoPagination
+end
+
+require 'twitter_with_auto_pagination/rest/api'
+
+module Twitter
+  module REST
+    class Client
+      prepend TwitterWithAutoPagination::REST::API
+
+      def initialize(options = {})
+        @cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache'))
+        @call_count = 0
+
+        @uid = options.has_key?(:uid) ? options.delete(:uid).to_i : nil
+        @screen_name = options.has_key?(:screen_name) ? options.delete(:screen_name).to_s : nil
+
+        logger =
+          if options.has_key?(:logger)
+            options.delete(:logger)
+          else
+            Dir.mkdir('log') unless File.exists?('log')
+            Logger.new('log/twitter_with_auto_pagination.log')
+          end
+        logger.level = options.has_key?(:log_level) ? options.delete(:log_level) : :debug
+        @@logger = @logger = logger
+
+        super
+      end
+
+      def self.logger
+        @@logger
+      end
+
+      attr_accessor :call_count, :logger
+      attr_reader :cache
+
+      INDENT = 4
+    end
+  end
 end

data/lib/twitter_with_auto_pagination/log_subscriber.rb

@@ -38,7 +38,7 @@ module TwitterWithAutoPagination
       # sql  = color(sql, sql_color(sql), true)
 
       key = payload.delete(:key)
-      debug { "#{name} #{key} #{(payload.inspect)}" }
+      debug { "#{name}#{key.nil? ? '' : " #{key}"} #{(payload.inspect)}" }
     end
 
     private
@@ -73,7 +73,7 @@ module TwitterWithAutoPagination
     end
 
     def logger
-      TwitterWithAutoPagination::Client.logger
+      Twitter::REST::Client.logger
     end
   end
 end

data/lib/twitter_with_auto_pagination/rest/api.rb

@@ -0,0 +1,31 @@
+require 'twitter_with_auto_pagination/rest/favorites'
+require 'twitter_with_auto_pagination/rest/friends_and_followers'
+require 'twitter_with_auto_pagination/rest/search'
+require 'twitter_with_auto_pagination/rest/timelines'
+require 'twitter_with_auto_pagination/rest/users'
+require 'twitter_with_auto_pagination/rest/uncategorized'
+
+require 'twitter_with_auto_pagination/rest/extension/clusters'
+require 'twitter_with_auto_pagination/rest/extension/favoriting'
+require 'twitter_with_auto_pagination/rest/extension/friends_and_followers'
+require 'twitter_with_auto_pagination/rest/extension/replying'
+require 'twitter_with_auto_pagination/rest/extension/unfollowing'
+
+module TwitterWithAutoPagination
+  module REST
+    module API
+      include TwitterWithAutoPagination::REST::Favorites
+      include TwitterWithAutoPagination::REST::FriendsAndFollowers
+      include TwitterWithAutoPagination::REST::Search
+      include TwitterWithAutoPagination::REST::Timelines
+      include TwitterWithAutoPagination::REST::Users
+      include TwitterWithAutoPagination::REST::Uncategorized
+
+      include TwitterWithAutoPagination::REST::Extension::Clusters
+      include TwitterWithAutoPagination::REST::Extension::Favoriting
+      include TwitterWithAutoPagination::REST::Extension::FriendsAndFollowers
+      include TwitterWithAutoPagination::REST::Extension::Replying
+      include TwitterWithAutoPagination::REST::Extension::Unfollowing
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/rest/extension/clusters.rb

@@ -0,0 +1,43 @@
+require 'twitter_with_auto_pagination/rest/utils'
+
+module TwitterWithAutoPagination
+  module REST
+    module Extension
+      module Clusters
+        include TwitterWithAutoPagination::REST::Utils
+
+        def clusters_belong_to(text)
+          return [] if text.blank?
+
+          exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
+          special_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_good_words_path']))
+
+          # クラスタ用の単語の出現回数を記録
+          cluster_word_counter =
+            special_words.map { |sw| [sw, text.scan(sw)] }
+              .delete_if { |item| item[1].empty? }
+              .each_with_object(Hash.new(1)) { |item, memo| memo[item[0]] = item[1].size }
+
+          # 同一文字種の繰り返しを見付ける。漢字の繰り返し、ひらがなの繰り返し、カタカナの繰り返し、など
+          text.scan(/[一-龠〆ヵヶ々]+|[ぁ-んー～]+|[ァ-ヴー～]+|[ａ-ｚＡ-Ｚ０-９]+|[、。！!？?]+/).
+
+            # 複数回繰り返される文字を除去
+            map { |w| w.remove /[？！?!。、ｗ]|(ー{2,})/ }.
+
+            # 文字数の少なすぎる単語、ひらがなだけの単語、除外単語を除去する
+            delete_if { |w| w.length <= 1 || (w.length <= 2 && w =~ /^[ぁ-んー～]+$/) || exclude_words.include?(w) }.
+
+            # 出現回数を記録
+            each { |w| cluster_word_counter[w] += 1 }
+
+          # 複数個以上見付かった単語のみを残し、出現頻度順にソート
+          cluster_word_counter.select { |_, v| v > 3 }.sort_by { |_, v| -v }.to_h
+        end
+
+        def clusters_assigned_to
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/rest/extension/favoriting.rb

@@ -0,0 +1,106 @@
+require 'twitter_with_auto_pagination/rest/utils'
+
+module TwitterWithAutoPagination
+  module REST
+    module Extension
+      module Favoriting
+        include TwitterWithAutoPagination::REST::Utils
+
+        def _count_users_with_two_sided_threshold(users, options)
+          min = options.has_key?(:min) ? options[:min] : 0
+          max = options.has_key?(:max) ? options[:max] : 1000
+          users.each_with_object(Hash.new(0)) { |u, memo| memo[u.id] += 1 }.
+            select { |_k, v| min <= v && v <= max }.
+            sort_by { |_, v| -v }.to_h
+        end
+
+        def _extract_favorite_users(favs, options = {})
+          counted_value = _count_users_with_two_sided_threshold(favs.map { |t| t.user }, options)
+          counted_value.map do |uid, cnt|
+            fav = favs.find { |f| f.user.id.to_i == uid.to_i }
+            Array.new(cnt, fav.user)
+          end.flatten
+        end
+
+        def _retrieve_favs(*args)
+          options = args.extract_options!
+          if args.empty?
+            favorites(options)
+          elsif uid_or_screen_name?(args[0])
+            favorites(args[0], options)
+          elsif args[0].kind_of?(Array) && args[0].all? { |t| t.respond_to?(:text) }
+            args[0]
+          else
+            raise ArgumentError
+          end
+        end
+
+        def users_which_you_faved(*args)
+          options = args.extract_options!
+          instrument(__method__, nil, options) do
+            favs = _retrieve_favs(*args, options)
+            result = _extract_favorite_users(favs, options)
+            if options.has_key?(:uniq) && !options[:uniq]
+              result
+            else
+              result.uniq { |r| r.id }
+            end
+          end
+        rescue => e
+          logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
+          raise e
+        end
+
+        alias favoriting users_which_you_faved
+
+        def users_who_faved_you(*args)
+          raise NotImplementedError
+        end
+
+        alias favorited users_who_faved_you
+
+        def _retrieve_replying_replied_and_favoriting(*args)
+          names = %i(replying replied favoriting)
+          options = args.extract_options!
+          if args.empty?
+            _fetch_parallelly(names.map { |n| {method: n, args: [options]} })
+          elsif uid_or_screen_name?(args[0])
+            _fetch_parallelly(names.map { |n| {method: n, args: [args[0], options]} })
+          elsif names.all? { |n| args[0].respond_to?(n) }
+            names.map { |n| args[0].send(n) }
+          else
+            raise ArgumentError
+          end
+        end
+
+        def close_friends(*args)
+          options = {uniq: false}.merge(args.extract_options!)
+          min_max = {
+            min: options.has_key?(:min) ? options.delete(:min) : 0,
+            max: options.has_key?(:max) ? options.delete(:max) : 1000
+          }
+
+          instrument(__method__, nil, options) do
+            replying, replied, favoriting = _retrieve_replying_replied_and_favoriting(*args, options)
+
+            users = replying + replied + favoriting
+            return [] if users.empty?
+
+            score = _count_users_with_two_sided_threshold(users, min_max)
+            replying_score = _count_users_with_two_sided_threshold(replying, min_max)
+            replied_score = _count_users_with_two_sided_threshold(replied, min_max)
+            favoriting_score = _count_users_with_two_sided_threshold(favoriting, min_max)
+
+            score.keys.map { |uid| users.find { |u| u.id.to_i == uid.to_i } }.map do |u|
+              u[:score] = score[u.id]
+              u[:replying_score] = replying_score[u.id]
+              u[:replied_score] = replied_score[u.id]
+              u[:favoriting_score] = favoriting_score[u.id]
+              u
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/twitter_with_auto_pagination/rest/extension/friends_and_followers.rb

@@ -0,0 +1,131 @@
+require 'twitter_with_auto_pagination/rest/utils'
+require 'parallel'
+
+module TwitterWithAutoPagination
+  module REST
+    module Extension
+      module FriendsAndFollowers
+        include TwitterWithAutoPagination::REST::Utils
+
+        def _fetch_parallelly(signatures) # [{method: :friends, args: ['ts_3156', ...], {...}]
+          result = Array.new(signatures.size)
+
+          Parallel.each_with_index(signatures, in_threads: result.size) do |signature, i|
+            result[i] = send(signature[:method], *signature[:args])
+          end
+
+          result
+        end
+
+        def friends_and_followers(*args)
+          _fetch_parallelly(
+            [
+              {method: :friends, args: args},
+              {method: :followers, args: args}])
+        end
+
+        def friends_followers_and_statuses(*args)
+          _fetch_parallelly(
+            [
+              {method: :friends, args: args},
+              {method: :followers, args: args},
+              {method: :user_timeline, args: args}])
+        end
+
+        def _retrieve_friends_and_followers(*args)
+          obj = args[0]
+          if obj.nil?
+            friends_and_followers
+          elsif uid_or_screen_name?(obj)
+            friends_and_followers(obj)
+          elsif obj.respond_to?(:friends) && obj.respond_to?(:followers)
+            [obj.friends, obj.followers]
+          else
+            raise ArgumentError, args.inspect
+          end
+        end
+
+        def one_sided_friends(me = nil)
+          instrument(__method__, nil) do
+            _friends, _followers = _retrieve_friends_and_followers(me)
+            _friends.to_a - _followers.to_a
+          end
+        end
+
+        def one_sided_followers(me = nil)
+          instrument(__method__, nil) do
+            _friends, _followers = _retrieve_friends_and_followers(me)
+            _followers.to_a - _friends.to_a
+          end
+        end
+
+        def mutual_friends(me = nil)
+          instrument(__method__, nil) do
+            _friends, _followers = _retrieve_friends_and_followers(me)
+            _friends.to_a & _followers.to_a
+          end
+        end
+
+        def _retrieve_friends(*args)
+          if args.size == 1
+            args[0].nil? ? friends : friends(args[0])
+          elsif args.all? { |obj| uid_or_screen_name?(obj) }
+            _fetch_parallelly(args.map { |obj| {method: :friends, args: [obj]} })
+          elsif args.all? { |obj| obj.respond_to?(:friends) }
+            args.map { |obj| obj.friends }
+          else
+            raise ArgumentError, args.inspect
+          end
+        end
+
+        def common_friends(me, you)
+          instrument(__method__, nil) do
+            my_friends, your_friends = _retrieve_friends(me, you)
+            my_friends.to_a & your_friends.to_a
+          end
+        end
+
+        def _retrieve_followers(*args)
+          if args.size == 1
+            args[0].nil? ? followers : followers(args[0])
+          elsif args.all? { |obj| uid_or_screen_name?(obj) }
+            _fetch_parallelly(args.map { |obj| {method: :followers, args: [obj]} })
+          elsif args.all? { |obj| obj.respond_to?(:followers) }
+            args.map { |obj| obj.followers }
+          else
+            raise ArgumentError, args.inspect
+          end
+        end
+
+        def common_followers(me, you)
+          instrument(__method__, nil) do
+            my_followers, your_followers = _retrieve_followers(me, you)
+            my_followers.to_a & your_followers.to_a
+          end
+        end
+
+        def _extract_inactive_users(users)
+          two_weeks_ago = 2.weeks.ago.to_i
+          users.select do |u|
+            (Time.parse(u.status.created_at).to_i < two_weeks_ago) rescue false
+          end
+        end
+
+        def inactive_friends(user = nil)
+          instrument(__method__, nil) do
+            _friends = _retrieve_friends(user)
+            _extract_inactive_users(_friends)
+          end
+        end
+
+        def inactive_followers(user = nil)
+          instrument(__method__, nil) do
+            _followers = _retrieve_followers(user)
+            _extract_inactive_users(_followers)
+          end
+        end
+      end
+    end
+  end
+end
+