twitter_with_auto_pagination 0.6.2 → 0.7.0

data/lib/twitter_with_auto_pagination/new_api.rb

@@ -1,337 +0,0 @@
-module TwitterWithAutoPagination
-  module NewApi
-    def friends_parallelly(*args)
-      options = {super_operation: __method__}.merge(args.extract_options!)
-      _friend_ids = friend_ids(*(args + [options]))
-      users(_friend_ids.map { |id| id.to_i }, options)
-    end
-
-    def followers_parallelly(*args)
-      options = {super_operation: __method__}.merge(args.extract_options!)
-      _follower_ids = follower_ids(*(args + [options]))
-      users(_follower_ids.map { |id| id.to_i }, options)
-    end
-
-    def _fetch_parallelly(signatures) # [{method: :friends, args: ['ts_3156', ...], {...}]
-      result = Array.new(signatures.size)
-
-      Parallel.each_with_index(signatures, in_threads: result.size) do |signature, i|
-        result[i] = send(signature[:method], *signature[:args])
-      end
-
-      result
-    end
-
-    def friends_and_followers(*args)
-      _fetch_parallelly(
-        [
-          {method: :friends_parallelly, args: args},
-          {method: :followers_parallelly, args: args}])
-    end
-
-    def friends_followers_and_statuses(*args)
-      _fetch_parallelly(
-        [
-          {method: :friends_parallelly, args: args},
-          {method: :followers_parallelly, args: args},
-          {method: :user_timeline, args: args}])
-    end
-
-    def one_sided_friends(me = nil)
-      if me.nil?
-        friends_parallelly.to_a - followers_parallelly.to_a
-      elsif uid_or_screen_name?(me)
-        # TODO use friends_and_followers
-        friends_parallelly(me).to_a - followers_parallelly(me).to_a
-      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
-        me.friends.to_a - me.followers.to_a
-      else
-        raise
-      end
-    end
-
-    def one_sided_followers(me = nil)
-      if me.nil?
-        followers_parallelly.to_a - friends_parallelly.to_a
-      elsif uid_or_screen_name?(me)
-        # TODO use friends_and_followers
-        followers_parallelly(me).to_a - friends_parallelly(me).to_a
-      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
-        me.followers.to_a - me.friends.to_a
-      else
-        raise
-      end
-    end
-
-    def mutual_friends(me = nil)
-      if me.nil?
-        friends_parallelly.to_a & followers_parallelly.to_a
-      elsif uid_or_screen_name?(me)
-        # TODO use friends_and_followers
-        friends_parallelly(me).to_a & followers_parallelly(me).to_a
-      elsif me.respond_to?(:friends) && me.respond_to?(:followers)
-        me.friends.to_a & me.followers.to_a
-      else
-        raise
-      end
-    end
-
-    def common_friends(me, you)
-      if uid_or_screen_name?(me) && uid_or_screen_name?(you)
-        friends_parallelly(me).to_a & friends_parallelly(you).to_a
-      elsif me.respond_to?(:friends) && you.respond_to?(:friends)
-        me.friends.to_a & you.friends.to_a
-      else
-        raise
-      end
-    end
-
-    def common_followers(me, you)
-      if uid_or_screen_name?(me) && uid_or_screen_name?(you)
-        followers_parallelly(me).to_a & followers_parallelly(you).to_a
-      elsif me.respond_to?(:followers) && you.respond_to?(:followers)
-        me.followers.to_a & you.followers.to_a
-      else
-        raise
-      end
-    end
-
-    def users_which_you_removed(pre_me, cur_me)
-      if uid_or_screen_name?(pre_me) && uid_or_screen_name?(cur_me)
-        friends_parallelly(pre_me).to_a - friends_parallelly(cur_me).to_a
-      elsif pre_me.respond_to?(:friends) && cur_me.respond_to?(:friends)
-        pre_me.friends.to_a - cur_me.friends.to_a
-      else
-        raise
-      end
-    end
-
-    def users_who_removed_you(pre_me, cur_me)
-      if uid_or_screen_name?(pre_me) && uid_or_screen_name?(cur_me)
-        followers_parallelly(pre_me).to_a - followers_parallelly(cur_me).to_a
-      elsif pre_me.respond_to?(:followers) && cur_me.respond_to?(:followers)
-        pre_me.followers.to_a - cur_me.followers.to_a
-      else
-        raise
-      end
-    end
-
-    def _extract_screen_names(tweets)
-      tweets.map do |t|
-        $1 if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
-      end.compact
-    end
-
-    # users which specified user is replying
-    # in_reply_to_user_id and in_reply_to_status_id is not used because of distinguishing mentions from replies
-    def users_which_you_replied_to(*args)
-      options = args.extract_options!
-      tweets =
-        if args.empty?
-          user_timeline(options)
-        elsif uid_or_screen_name?(args[0])
-          user_timeline(args[0], options)
-        elsif args[0].kind_of?(Array) && args[0].all? { |t| t.respond_to?(:text) }
-          args[0]
-        else
-          raise
-        end
-
-      screen_names = _extract_screen_names(tweets)
-      result = users(screen_names, {super_operation: __method__}.merge(options))
-      if options.has_key?(:uniq) && !options[:uniq]
-        screen_names.map { |sn| result.find { |r| r.screen_name == sn } }.compact
-      else
-        result.uniq { |r| r.id }
-      end
-    rescue Twitter::Error::NotFound => e
-      e.message == 'No user matches for specified terms.' ? [] : (raise e)
-    rescue => e
-      logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
-      raise e
-    end
-
-    def _extract_uids(tweets)
-      tweets.map do |t|
-        t.user.id.to_i if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
-      end.compact
-    end
-
-    def _extract_users(tweets, uids)
-      uids.map { |uid| tweets.find { |t| t.user.id.to_i == uid.to_i } }.map { |t| t.user }.compact
-    end
-
-    # users which specified user is replied
-    # when user is login you had better to call mentions_timeline
-    def users_who_replied_to_you(*args)
-      options = args.extract_options!
-
-      result =
-        if args.empty? || (uid_or_screen_name?(args[0]) && authenticating_user?(args[0]))
-          mentions_timeline.map { |m| m.user }
-        else
-          searched_result = search('@' + user(args[0]).screen_name, options)
-          uids = _extract_uids(searched_result)
-          _extract_users(searched_result, uids)
-        end
-
-      if options.has_key?(:uniq) && !options[:uniq]
-        result
-      else
-        result.uniq { |r| r.id }
-      end
-    end
-
-    def _count_users_with_two_sided_threshold(users, options)
-      min = options.has_key?(:min) ? options[:min] : 0
-      max = options.has_key?(:max) ? options[:max] : 1000
-      users.each_with_object(Hash.new(0)) { |u, memo| memo[u.id] += 1 }.
-        select { |_k, v| min <= v && v <= max }.
-        sort_by { |_, v| -v }.to_h
-    end
-
-    def _extract_favorite_users(favs, options = {})
-      counted_value = _count_users_with_two_sided_threshold(favs.map { |t| t.user }, options)
-      counted_value.map do |uid, cnt|
-        fav = favs.find { |f| f.user.id.to_i == uid.to_i }
-        Array.new(cnt, fav.user)
-      end.flatten
-    end
-
-    def users_which_you_faved(*args)
-      options = args.extract_options!
-
-      favs =
-        if args.empty?
-          favorites(options)
-        elsif uid_or_screen_name?(args[0])
-          favorites(args[0], options)
-        elsif args[0].kind_of?(Array) && args[0].all? { |t| t.respond_to?(:text) }
-          args[0]
-        else
-          raise
-        end
-
-      result = _extract_favorite_users(favs, options)
-      if options.has_key?(:uniq) && !options[:uniq]
-        result
-      else
-        result.uniq { |r| r.id }
-      end
-    rescue => e
-      logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
-      raise e
-    end
-
-    def users_who_faved_you(*args)
-    end
-
-    def _extract_inactive_users(users, options = {})
-      authorized = options.delete(:authorized)
-      two_weeks_ago = 2.weeks.ago.to_i
-      users.select do |u|
-        if authorized
-          (Time.parse(u.status.created_at).to_i < two_weeks_ago) rescue false
-        else
-          false
-        end
-      end
-    end
-
-    def close_friends(*args)
-      options = {uniq: false}.merge(args.extract_options!)
-      min_max = {
-        min: options.has_key?(:min) ? options.delete(:min) : 0,
-        max: options.has_key?(:max) ? options.delete(:max) : 1000
-      }
-
-      _users_which_you_replied_to, _users_who_replied_to_you, _users_which_you_faved =
-        if args.empty?
-          [users_which_you_replied_to(options), users_who_replied_to_you(options), users_which_you_faved(options)]
-        elsif uid_or_screen_name?(args[0])
-          [users_which_you_replied_to(args[0], options), users_who_replied_to_you(args[0], options), users_which_you_faved(args[0], options)]
-        elsif (m_names = %i(users_which_you_replied_to users_who_replied_to_you users_which_you_faved)).all? { |m_name| args[0].respond_to?(m_name) }
-          m_names.map { |mn| args[0].send(mn) }
-        else
-          raise
-        end
-
-      _users = _users_which_you_replied_to + _users_who_replied_to_you + _users_which_you_faved
-      return [] if _users.empty?
-
-      scores = _count_users_with_two_sided_threshold(_users, min_max)
-      users_which_you_replied_to_scores = _count_users_with_two_sided_threshold(_users_which_you_replied_to, min_max)
-      users_who_replied_to_you_scores = _count_users_with_two_sided_threshold(_users_who_replied_to_you, min_max)
-      users_which_you_faved_scores = _count_users_with_two_sided_threshold(_users_which_you_faved, min_max)
-
-      scores.keys.map { |uid| _users.find { |u| u.id.to_i == uid.to_i } }.
-        map do |u|
-        u[:score] = scores[u.id]
-        u[:users_which_you_replied_to_score] = users_which_you_replied_to_scores[u.id]
-        u[:users_who_replied_to_you_score] = users_who_replied_to_you_scores[u.id]
-        u[:users_which_you_faved_score] = users_which_you_faved_scores[u.id]
-        u
-      end
-    end
-
-    def inactive_friends(user = nil)
-      if user.blank?
-        _extract_inactive_users(friends_parallelly, authorized: true)
-      elsif uid_or_screen_name?(user)
-        authorized = authenticating_user?(user) || authorized_user?(user)
-        _extract_inactive_users(friends_parallelly(user), authorized: authorized)
-      elsif user.respond_to?(:friends)
-        authorized = authenticating_user?(user.uid.to_i) || authorized_user?(user.uid.to_i)
-        _extract_inactive_users(user.friends, authorized: authorized)
-      else
-        raise
-      end
-    end
-
-    def inactive_followers(user = nil)
-      if user.blank?
-        _extract_inactive_users(followers_parallelly, authorized: true)
-      elsif uid_or_screen_name?(user)
-        authorized = authenticating_user?(user) || authorized_user?(user)
-        _extract_inactive_users(followers_parallelly(user), authorized: authorized)
-      elsif user.respond_to?(:followers)
-        authorized = authenticating_user?(user.uid.to_i) || authorized_user?(user.uid.to_i)
-        _extract_inactive_users(user.followers, authorized: authorized)
-      else
-        raise
-      end
-    end
-
-    def clusters_belong_to(text)
-      return [] if text.blank?
-
-      exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
-      special_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_good_words_path']))
-
-      # クラスタ用の単語の出現回数を記録
-      cluster_word_counter =
-        special_words.map { |sw| [sw, text.scan(sw)] }
-          .delete_if { |item| item[1].empty? }
-          .each_with_object(Hash.new(1)) { |item, memo| memo[item[0]] = item[1].size }
-
-      # 同一文字種の繰り返しを見付ける。漢字の繰り返し、ひらがなの繰り返し、カタカナの繰り返し、など
-      text.scan(/[一-龠〆ヵヶ々]+|[ぁ-んー～]+|[ァ-ヴー～]+|[ａ-ｚＡ-Ｚ０-９]+|[、。！!？?]+/).
-
-        # 複数回繰り返される文字を除去
-        map { |w| w.remove /[？！?!。、ｗ]|(ー{2,})/ }.
-
-        # 文字数の少なすぎる単語、ひらがなだけの単語、除外単語を除去する
-        delete_if { |w| w.length <= 1 || (w.length <= 2 && w =~ /^[ぁ-んー～]+$/) || exclude_words.include?(w) }.
-
-        # 出現回数を記録
-        each { |w| cluster_word_counter[w] += 1 }
-
-      # 複数個以上見付かった単語のみを残し、出現頻度順にソート
-      cluster_word_counter.select { |_, v| v > 3 }.sort_by { |_, v| -v }.to_h
-    end
-
-    def clusters_assigned_to
-      raise NotImplementedError.new
-    end
-  end
-end

data/lib/twitter_with_auto_pagination/utils.rb

@@ -1,303 +0,0 @@
-module TwitterWithAutoPagination
-  module Utils
-    # for backward compatibility
-    def uid
-      @uid || user.id.to_i
-    end
-
-    def __uid
-      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
-        `TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
-      MESSAGE
-      uid
-    end
-
-    def __uid_i
-      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
-        `TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
-      MESSAGE
-      uid
-    end
-
-    # for backward compatibility
-    def screen_name
-      @screen_name || user.screen_name
-    end
-
-    def __screen_name
-      ActiveSupport::Deprecation.warn(<<-MESSAGE.strip_heredoc)
-        `TwitterWithAutoPagination::Utils##{__method__}` is deprecated.
-      MESSAGE
-      screen_name
-    end
-
-    def uid_or_screen_name?(object)
-      object.kind_of?(String) || object.kind_of?(Integer)
-    end
-
-    def authenticating_user?(target)
-      user.id.to_i == user(target).id.to_i
-    end
-
-    def authorized_user?(target)
-      target_user = user(target)
-      !target_user.protected? || friendship?(user.id.to_i, target_user.id.to_i)
-    end
-
-    def instrument(operation, key, options = nil)
-      payload = {operation: operation, key: key}
-      payload.merge!(options) if options.is_a?(Hash)
-      ActiveSupport::Notifications.instrument('call.twitter_with_auto_pagination', payload) { yield(payload) }
-    end
-
-    def call_old_method(method_name, *args)
-      options = args.extract_options!
-      begin
-        self.call_count += 1
-        _options = {method_name: method_name, call_count: self.call_count, args: args}.merge(options)
-        instrument('api call', args[0], _options) { send(method_name, *args, options) }
-      rescue Twitter::Error::TooManyRequests => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} Retry after #{e.rate_limit.reset_in} seconds."
-        raise e
-      rescue Twitter::Error::ServiceUnavailable => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
-        raise e
-      rescue Twitter::Error::InternalServerError => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
-        raise e
-      rescue Twitter::Error::Forbidden => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
-        raise e
-      rescue Twitter::Error::NotFound => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
-        raise e
-      rescue => e
-        logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
-        raise e
-      end
-    end
-
-    # user_timeline, search
-    def collect_with_max_id(method_name, *args)
-      options = args.extract_options!
-      call_limit = options.delete(:call_limit) || 3
-      last_response = call_old_method(method_name, *args, options)
-      last_response = yield(last_response) if block_given?
-      return_data = last_response
-      call_count = 1
-
-      while last_response.any? && call_count < call_limit
-        options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
-        last_response = call_old_method(method_name, *args, options)
-        last_response = yield(last_response) if block_given?
-        return_data += last_response
-        call_count += 1
-      end
-
-      return_data.flatten
-    end
-
-    # friends, followers
-    def collect_with_cursor(method_name, *args)
-      options = args.extract_options!
-      last_response = call_old_method(method_name, *args, options).attrs
-      return_data = (last_response[:users] || last_response[:ids])
-
-      while (next_cursor = last_response[:next_cursor]) && next_cursor != 0
-        options[:cursor] = next_cursor
-        last_response = call_old_method(method_name, *args, options).attrs
-        return_data += (last_response[:users] || last_response[:ids])
-      end
-
-      return_data
-    end
-
-    require 'digest/md5'
-
-    def file_cache_key(method_name, user, options = {})
-      delim = ':'
-      identifier =
-        case
-          when method_name == :verify_credentials
-            "object-id#{delim}#{object_id}"
-          when method_name == :search
-            "str#{delim}#{user.to_s}"
-          when method_name == :mentions_timeline
-            "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
-          when method_name == :home_timeline
-            "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
-          when method_name.in?([:users, :replying]) && options[:super_operation].present?
-            case
-              when user.kind_of?(Array) && user.first.kind_of?(Integer)
-                "#{options[:super_operation]}-ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-              when user.kind_of?(Array) && user.first.kind_of?(String)
-                "#{options[:super_operation]}-sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-              else raise "#{method_name.inspect} #{user.inspect}"
-            end
-          when user.kind_of?(Integer)
-            "id#{delim}#{user.to_s}"
-          when user.kind_of?(Array) && user.first.kind_of?(Integer)
-            "ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-          when user.kind_of?(Array) && user.first.kind_of?(String)
-            "sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
-          when user.kind_of?(String)
-            "sn#{delim}#{user}"
-          when user.kind_of?(Twitter::User)
-            "user#{delim}#{user.id.to_s}"
-          else raise "#{method_name.inspect} #{user.inspect}"
-        end
-
-      "#{method_name}#{delim}#{identifier}"
-    end
-
-    def namespaced_key(method_name, user, options = {})
-      file_cache_key(method_name, user, options)
-    end
-
-    PROFILE_SAVE_KEYS = %i(
-        id
-        name
-        screen_name
-        location
-        description
-        url
-        protected
-        followers_count
-        friends_count
-        listed_count
-        favourites_count
-        utc_offset
-        time_zone
-        geo_enabled
-        verified
-        statuses_count
-        lang
-        status
-        profile_image_url_https
-        profile_banner_url
-        profile_link_color
-        suspended
-        verified
-        entities
-        created_at
-      )
-
-    STATUS_SAVE_KEYS = %i(
-      created_at
-      id
-      text
-      source
-      truncated
-      coordinates
-      place
-      entities
-      user
-      contributors
-      is_quote_status
-      retweet_count
-      favorite_count
-      favorited
-      retweeted
-      possibly_sensitive
-      lang
-    )
-
-    # encode
-    def encode_json(obj, caller_name, options = {})
-      options[:reduce] = true unless options.has_key?(:reduce)
-      case caller_name
-        when :user_timeline, :home_timeline, :mentions_timeline, :favorites # Twitter::Tweet
-          JSON.pretty_generate(obj.map { |o| o.attrs })
-
-        when :search # Hash
-          data =
-            if options[:reduce]
-              obj.map { |o| o.to_hash.slice(*STATUS_SAVE_KEYS) }
-            else
-              obj.map { |o| o.to_hash }
-            end
-          JSON.pretty_generate(data)
-
-        when :friends, :followers # Hash
-          data =
-            if options[:reduce]
-              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
-            else
-              obj.map { |o| o.to_hash }
-            end
-          JSON.pretty_generate(data)
-
-        when :friend_ids, :follower_ids # Integer
-          JSON.pretty_generate(obj)
-
-        when :verify_credentials # Twitter::User
-          JSON.pretty_generate(obj.to_hash.slice(*PROFILE_SAVE_KEYS))
-
-        when :user # Twitter::User
-          JSON.pretty_generate(obj.to_hash.slice(*PROFILE_SAVE_KEYS))
-
-        when :users, :friends_parallelly, :followers_parallelly # Twitter::User
-          data =
-            if options[:reduce]
-              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
-            else
-              obj.map { |o| o.to_hash }
-            end
-          JSON.pretty_generate(data)
-
-        when :user? # true or false
-          obj
-
-        when :friendship? # true or false
-          obj
-
-        else
-          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
-      end
-    end
-
-    # decode
-    def decode_json(json_str, caller_name, options = {})
-      obj = json_str.kind_of?(String) ? JSON.parse(json_str) : json_str
-      case
-        when obj.nil?
-          obj
-
-        when obj.kind_of?(Array) && obj.first.kind_of?(Hash)
-          obj.map { |o| Hashie::Mash.new(o) }
-
-        when obj.kind_of?(Array) && obj.first.kind_of?(Integer)
-          obj
-
-        when obj.kind_of?(Hash)
-          Hashie::Mash.new(obj)
-
-        when obj === true || obj === false
-          obj
-
-        when obj.kind_of?(Array) && obj.empty?
-          obj
-
-        else
-          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
-      end
-    end
-
-    def fetch_cache_or_call_api(method_name, user, options = {})
-      key = namespaced_key(method_name, user, options)
-      options.update(key: key)
-
-      data =
-        if options[:cache] == :read
-          instrument('Cache Read(Force)', key, caller: method_name) { cache.read(key) }
-        else
-          cache.fetch(key, expires_in: 1.hour, race_condition_ttl: 5.minutes) do
-            _d = yield
-            instrument('serialize', key, caller: method_name) { encode_json(_d, method_name, options) }
-          end
-        end
-
-      instrument('deserialize', key, caller: method_name) { decode_json(data, method_name, options) }
-    end
-  end
-end