ex_twitter 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68eb499d7d42d9fa3351be383a820068509588b1
-  data.tar.gz: 9a5977112d10a15feb4e4b5544ae164cd1f69d36
+  metadata.gz: 1e529d197b3ac12452f7d3864c2e00bb7e78822a
+  data.tar.gz: 2363dd11f96d395e19f68a4db3aa814240d54031
 SHA512:
-  metadata.gz: 7dc179bd00f61fe17739bdfc60a99124193afb8372121869ceba71bfd6f8854cabb77491a38af577af4be6e6801b58e4af290b657dc384d837e481fe20d22ac1
-  data.tar.gz: 361e9707b96c3d40c2ecc6a1d45f17d29901dce7a17af85acb2df5c773a051d7f16d4e463571ab11a5c1a94a6497a3e235424113704d6e85c6c8e196e9589cad
+  metadata.gz: 11012a0796bf7d3ab4ba4838803c03125425220b20d1ba777397f5f588141efb6fcd6bb8c1b4678d55a6b4ac3719172dc92cecbb52379776be315808abc15c8e
+  data.tar.gz: 67a189e3be3eeccfd934bebc9c37bcc039eb39340fe2b710b5a2eaad375087c5441ccfe15ebed2e9fed8daaff74ea6d893380101bf80d0d91444d948469c1415

data/ex_twitter.gemspec

@@ -4,6 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |spec|
   spec.add_dependency 'twitter'
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'hashie'
   spec.add_dependency 'parallel'
 
   spec.add_development_dependency 'bundler'
@@ -20,5 +21,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
   spec.summary = spec.description
   spec.test_files = Dir.glob('spec/**/*')
-  spec.version = '0.0.4'
+  spec.version = '0.1.0'
 end

data/lib/ex_twitter.rb

@@ -1,267 +1,806 @@
 require 'active_support'
-require 'active_support/core_ext'
+require 'active_support/cache'
+require 'active_support/core_ext/string'
+
+require 'ex_twitter_subscriber'
+
 require 'twitter'
-require 'yaml'
-require 'active_support'
+require 'hashie'
+require 'memoist'
 require 'parallel'
-require 'logger'
 
-# extended twitter
-class ExTwitter < Twitter::REST::Client
-  attr_accessor :cache, :cache_expires_in, :max_retries, :wait, :auto_paginate, :max_paginates, :logger
 
-  MAX_RETRIES = 1
-  WAIT = false
-  # CACHE_EXPIRES_IN = 300
-  AUTO_PAGINATE = true
-  MAX_PAGINATES = 3
+
+class ExTwitter < Twitter::REST::Client
+  extend Memoist
 
   def initialize(options = {})
-    self.logger = Logger.new(STDOUT)
-    self.logger.level = Logger::DEBUG
+    api_cache_prefix = options.has_key?(:api_cache_prefix) ? options.delete(:api_cache_prefix) : '%Y%m%d%H'
+    @cache = ActiveSupport::Cache::FileStore.new(File.join('tmp', 'api_cache', Time.now.strftime(api_cache_prefix)))
+    @uid = options[:uid]
+    @screen_name = options[:screen_name]
+    @authenticated_user = Hashie::Mash.new({uid: options[:uid].to_i, screen_name: options[:screen_name]})
+    @call_count = 0
+    ExTwitterSubscriber.attach_to :ex_twitter
+    super
+  end
 
-    self.auto_paginate = AUTO_PAGINATE
-    self.max_retries = MAX_RETRIES
-    self.wait = WAIT
-    self.max_paginates = MAX_PAGINATES
+  attr_accessor :call_count
+  attr_reader :cache, :authenticated_user
 
-    # self.cache_expires_in = (config[:cache_expires_in] || CACHE_EXPIRES_IN)
-    # self.cache = ActiveSupport::Cache::FileStore.new(File.join(Dir::pwd, 'ex_twitter_cache'),
-    #   {expires_in: self.cache_expires_in, race_condition_ttl: self.cache_expires_in})
+  INDENT = 4
 
-    super
+  # for backward compatibility
+  def uid
+    @uid
+  end
+
+  def __uid
+    @uid
   end
 
-  # def read(key)
-  #   self.cache.read(key)
-  # rescue => e
-  #   logger.warn "in read #{key} #{e.inspect}"
-  #   nil
-  # end
-  #
-  # def write(key, value)
-  #   self.cache.write(key, value)
-  # rescue => e
-  #   logger.warn "in write #{key} #{value} #{e.inspect}"
-  #   false
-  # end
-
-  # githubでは、レスポンスが次のエンドポイントを知っているため、ブロックをデータの結合のみに使っている。引数ではAPI名を渡している。
-  # 一方、twitterでは、レスポンスが次のエンドポイントを知らないため、ブロック内にAPI名を持ち、再帰的にブロックを呼び出している。
-  # また、再帰を使うため、引数をコレクションの引き渡しに使ってしまっている。(この問題については通常のループを使えば解決する)
-  #
-  # APIは通常繰り返しの呼び出しになるため、API名は常に同じものを保持しておき、ブロックはデータの結合に使った方が無駄がなく自由度が高い。
-  # また、再帰ではなく通常のループを使った方が分かりやすい。
-  #
-  # このgithub方式を実現するためには、メソッド名(例：user_timeline)を渡し、ループさせるか、user_timelineが内部で使っている
-  # objects_from_response_with_userをループさせればよい。
-  #
-  # この2種類の方法を比較した場合、より外側のレイヤーを使った方が、使うライブラリの内部情報に依存しなくなるため、好ましい。
-  #
-  # twitterで再帰方式がとられている理由は、おそらく、一般ユーザー向けのサンプルでメソッド名を渡すようなリフレクションを避けるため、
-  # なのかもしれない。
-
-  # max_idを使って自動ページングを行う
-  def collect_with_max_id(method_name, *args, &block)
+  def __uid_i
+    @uid.to_i
+  end
+
+  # for backward compatibility
+  def screen_name
+    @screen_name
+  end
+
+  def __screen_name
+    @screen_name
+  end
+
+  def logger
+    Dir.mkdir('log') unless File.exists?('log')
+    @logger ||= Logger.new('log/ex_twitter.log')
+  end
+
+  def instrument(operation)
+    ActiveSupport::Notifications.instrument('call.ex_twitter', name: operation) do
+      yield
+    end
+  end
+
+  def call_old_method(method_name, *args)
     options = args.extract_options!
-    logger.info "#{method_name}, #{args.inspect} #{options.inspect}"
+    begin
+      start_t = Time.now
+      result = send(method_name, *args, options); self.call_count += 1
+      end_t = Time.now
+      logger.debug "#{method_name} #{args.inspect} #{options.inspect} (#{end_t - start_t}s)".indent(INDENT)
+      result
+    rescue Twitter::Error::TooManyRequests => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} Retry after #{e.rate_limit.reset_in} seconds."
+      raise e
+    rescue Twitter::Error::ServiceUnavailable => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    rescue Twitter::Error::InternalServerError => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    rescue Twitter::Error::Forbidden => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    rescue Twitter::Error::NotFound => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    rescue => e
+      logger.warn "#{__method__}: call=#{method_name} #{args.inspect} #{e.class} #{e.message}"
+      raise e
+    end
+  end
 
-    max_paginates = options.delete(:max_paginates) || MAX_PAGINATES
-    data = []
-    last_response = send(method_name, *args, options)
-    if block_given?
-      last_response = yield(data, last_response)
-    else
-      data.concat(last_response) if last_response.is_a?(Array)
+  # user_timeline, search
+  def collect_with_max_id(method_name, *args)
+    options = args.extract_options!
+    options[:call_count] = 3 unless options.has_key?(:call_count)
+    last_response = call_old_method(method_name, *args, options)
+    last_response = yield(last_response) if block_given?
+    return_data = last_response
+    call_count = 1
+
+    while last_response.any? && call_count < options[:call_count]
+      options[:max_id] = last_response.last.kind_of?(Hash) ? last_response.last[:id] : last_response.last.id
+      last_response = call_old_method(method_name, *args, options)
+      last_response = yield(last_response) if block_given?
+      return_data += last_response
+      call_count += 1
     end
 
-    if auto_paginate
-      num_retries = 0
-      (max_paginates - 1).times do
-        break if last_response.blank?
-
-        options[:max_id] = last_response.last[:id] - 1
-
-        begin
-          last_response = send(method_name, *args, options)
-          logger.info "#{method_name}, #{args.inspect} #{options.inspect}"
-        rescue Twitter::Error::TooManyRequests => e
-          if num_retries <= MAX_RETRIES
-            if WAIT
-              sleep e.rate_limit.reset_in
-              num_retries += 1
-              retry
+    return_data.flatten
+  end
+
+  # friends, followers
+  def collect_with_cursor(method_name, *args)
+    options = args.extract_options!
+    last_response = call_old_method(method_name, *args, options).attrs
+    return_data = (last_response[:users] || last_response[:ids])
+
+    while (next_cursor = last_response[:next_cursor]) && next_cursor != 0
+      options[:cursor] = next_cursor
+      last_response = call_old_method(method_name, *args, options).attrs
+      return_data += (last_response[:users] || last_response[:ids])
+    end
+
+    return_data
+  end
+
+  require 'digest/md5'
+
+  # currently ignore options
+  def file_cache_key(method_name, user)
+    delim = ':'
+    identifier =
+      case
+        when method_name == :search
+          "str#{delim}#{user.to_s}"
+        when method_name == :mentions_timeline
+          "myself#{delim}#{user.to_s}"
+        when method_name == :home_timeline
+          "myself#{delim}#{user.to_s}"
+        when user.kind_of?(Integer)
+          "id#{delim}#{user.to_s}"
+        when user.kind_of?(Array) && user.first.kind_of?(Integer)
+          "ids#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+        when user.kind_of?(Array) && user.first.kind_of?(String)
+          "sns#{delim}#{Digest::MD5.hexdigest(user.join(','))}"
+        when user.kind_of?(String)
+          "sn#{delim}#{user}"
+        when user.kind_of?(Twitter::User)
+          "user#{delim}#{user.id.to_s}"
+        else raise "#{method_name.inspect} #{user.inspect}"
+      end
+
+    "#{method_name}#{delim}#{identifier}"
+  end
+
+  def namespaced_key(method_name, user)
+    file_cache_key(method_name, user)
+  end
+
+  PROFILE_SAVE_KEYS = %i(
+      id
+      name
+      screen_name
+      location
+      description
+      url
+      protected
+      followers_count
+      friends_count
+      listed_count
+      favourites_count
+      utc_offset
+      time_zone
+      geo_enabled
+      verified
+      statuses_count
+      lang
+      status
+      profile_image_url_https
+      profile_banner_url
+      profile_link_color
+      suspended
+      verified
+      entities
+      created_at
+    )
+
+  STATUS_SAVE_KEYS = %i(
+    created_at
+    id
+    text
+    source
+    truncated
+    coordinates
+    place
+    entities
+    user
+    contributors
+    is_quote_status
+    retweet_count
+    favorite_count
+    favorited
+    retweeted
+    possibly_sensitive
+    lang
+  )
+
+  # encode
+  def encode_json(obj, caller_name, options = {})
+    options[:reduce] = true unless options.has_key?(:reduce)
+    start_t = Time.now
+    result =
+      case caller_name
+        when :user_timeline, :home_timeline, :mentions_timeline, :favorites # Twitter::Tweet
+          JSON.pretty_generate(obj.map { |o| o.attrs })
+
+        when :search # Hash
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*STATUS_SAVE_KEYS) }
             else
-              logger.warn "retry #{e.rate_limit.reset_in} seconds later, #{e.inspect}"
+              obj.map { |o| o.to_hash }
             end
-          else
-            logger.warn "fail. num_retries > MAX_RETRIES(=#{MAX_RETRIES}), #{e.inspect}"
-          end
-        rescue => e
-          if num_retries <= MAX_RETRIES
-            logger.warn "retry till num_retries > MAX_RETRIES(=#{MAX_RETRIES}), #{e.inspect}"
-            num_retries += 1
-            retry
-          else
-            logger.warn "fail. num_retries > MAX_RETRIES(=#{MAX_RETRIES}), something error #{e.inspect}"
-          end
-        end
+          JSON.pretty_generate(data)
+
+        when :friends, :followers # Hash
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
+            else
+              obj.map { |o| o.to_hash }
+            end
+          JSON.pretty_generate(data)
+
+        when :friend_ids, :follower_ids # Integer
+          JSON.pretty_generate(obj)
+
+        when :user # Twitter::User
+          JSON.pretty_generate(obj.to_hash.slice(*PROFILE_SAVE_KEYS))
+
+        when :users, :friends_advanced, :followers_advanced # Twitter::User
+          data =
+            if options[:reduce]
+              obj.map { |o| o.to_hash.slice(*PROFILE_SAVE_KEYS) }
+            else
+              obj.map { |o| o.to_hash }
+            end
+          JSON.pretty_generate(data)
+
+        when :user? # true or false
+          obj
+
+        when :friendship? # true or false
+          obj
 
-        if block_given?
-          last_response = yield(data, last_response)
         else
-          data.concat(last_response) if last_response.is_a?(Array)
-        end
+          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
       end
-    end
+    end_t = Time.now
+    logger.debug "#{__method__}: caller=#{caller_name} key=#{options[:key]} (#{end_t - start_t}s)".indent(INDENT)
+    result
+  end
+
+  # decode
+  def decode_json(json_str, caller_name, options = {})
+    start_t = Time.now
+    obj = json_str.kind_of?(String) ? JSON.parse(json_str) : json_str
+    result =
+      case
+        when obj.nil?
+          obj
 
-    data
+        when obj.kind_of?(Array) && obj.first.kind_of?(Hash)
+          obj.map { |o| Hashie::Mash.new(o) }
+
+        when obj.kind_of?(Array) && obj.first.kind_of?(Integer)
+          obj
+
+        when obj.kind_of?(Hash)
+          Hashie::Mash.new(obj)
+
+        when obj === true || obj === false
+          obj
+
+        when obj.kind_of?(Array) && obj.empty?
+          obj
+
+        else
+          raise "#{__method__}: caller=#{caller_name} key=#{options[:key]} obj=#{obj.inspect}"
+      end
+    end_t = Time.now
+    logger.debug "#{__method__}: caller=#{caller_name} key=#{options[:key]} (#{end_t - start_t}s)".indent(INDENT)
+    result
   end
 
-  # cursorを使って自動ページングを行う
-  def collect_with_cursor(method_name, *args, &block)
-    options = args.extract_options!
-    logger.info "#{method_name}, #{args.inspect} #{options.inspect}"
-
-    max_paginates = options.delete(:max_paginates) || MAX_PAGINATES
-    last_response = send(method_name, *args, options).attrs
-    data = last_response[:users] || last_response[:ids]
-
-    if auto_paginate
-      num_retries = 0
-      (max_paginates - 1).times do
-        next_cursor = last_response[:next_cursor]
-        break if !next_cursor || next_cursor == 0
-
-        options[:cursor] = next_cursor
-
-        begin
-          last_response = send(method_name, *args, options).attrs
-          logger.info "#{method_name}, #{args.inspect} #{options.inspect}"
-        rescue Twitter::Error::TooManyRequests => e
-          if num_retries <= MAX_RETRIES
-            if WAIT
-              sleep e.rate_limit.reset_in
-              num_retries += 1
-              retry
-            else
-              logger.warn "retry #{e.rate_limit.reset_in} seconds later, #{e.inspect}"
-            end
-          else
-            logger.warn "fail. num_retries > MAX_RETRIES(=#{MAX_RETRIES}), #{e.inspect}"
-          end
-        rescue => e
-          if num_retries <= MAX_RETRIES
-            logger.warn "retry till num_retries > MAX_RETRIES(=#{MAX_RETRIES}), #{e.inspect}"
-            num_retries += 1
-            retry
-          else
-            logger.warn "fail. num_retries > MAX_RETRIES(=#{MAX_RETRIES}), something error #{e.inspect}"
-          end
-        end
+  def fetch_cache_or_call_api(method_name, user, options = {})
+    start_t = Time.now
+    key = namespaced_key(method_name, user)
+    options.update(key: key)
+
+    logger.debug "#{__method__}: caller=#{method_name} key=#{key}"
 
+    data =
+      if options[:cache] == :read
+        hit = 'force read'
+        cache.read(key)
+      else
         if block_given?
-          yield(data, last_response)
+          hit = 'fetch(hit)'
+          cache.fetch(key, expires_in: 1.hour) do
+            hit = 'fetch(not hit)'
+            instrument(:encode_json) { encode_json(yield, method_name, options) }
+          end
         else
-          items = last_response[:users] || last_response[:ids]
-          data.concat(items) if items.is_a?(Array)
+          if cache.exist?(key)
+            hit = 'read(hit)'
+            cache.read(key)
+          else
+            hit = 'read(not hit)'
+            nil
+          end
         end
       end
-    end
 
-    data
+    result = decode_json(data, method_name, options)
+    logger.debug "#{__method__}: caller=#{method_name} key=#{key} #{hit} (#{Time.now - start_t}s)"
+    result
   end
 
-  alias :old_user_timeline :user_timeline
-  def user_timeline(*args)
-    options = {count: 200, include_rts: true}.merge(args.extract_options!)
-    collect_with_max_id(:old_user_timeline, *args, options)
+  alias :old_friendship? :friendship?
+  def friendship?(*args)
+    options = args.extract_options!
+    fetch_cache_or_call_api(:friendship?, args) {
+      call_old_method(:old_friendship?, *args, options)
+    }
+  end
+
+  alias :old_user? :user?
+  def user?(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:user?, args[0], options) {
+      call_old_method(:old_user?, args[0], options)
+    }
+  end
+
+  alias :old_user :user
+  def user(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:user, args[0], options) {
+      call_old_method(:old_user, args[0], options)
+    }
+  rescue => e
+    logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+  # memoize :user
+
+  alias :old_friend_ids :friend_ids
+  def friend_ids(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:friend_ids, args[0], options) {
+      options = {count: 5000, cursor: -1}.merge(options)
+      collect_with_cursor(:old_friend_ids, *args, options)
+    }
   end
 
-  def user_photos(*args)
-    tweets = user_timeline(*args)
-    tweets.select{|t| t.media? }.map{|t| t.media }.flatten
+  alias :old_follower_ids :follower_ids
+  def follower_ids(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:follower_ids, args[0], options) {
+      options = {count: 5000, cursor: -1}.merge(options)
+      collect_with_cursor(:old_follower_ids, *args, options)
+    }
   end
 
+  # specify reduce: false to use tweet for inactive_*
   alias :old_friends :friends
   def friends(*args)
-    options = {count: 200, include_user_entities: true}.merge(args.extract_options!)
-    collect_with_cursor(:old_friends, *args, options)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    options[:reduce] = false unless options.has_key?(:reduce)
+    fetch_cache_or_call_api(:friends, args[0], options) {
+      options = {count: 200, include_user_entities: true, cursor: -1}.merge(options)
+      collect_with_cursor(:old_friends, *args, options)
+    }
   end
+  # memoize :friends
 
+  def friends_advanced(*args)
+    options = args.extract_options!
+    _friend_ids = friend_ids(*(args + [options]))
+    users(_friend_ids.map { |id| id.to_i }, options)
+  end
+
+  # specify reduce: false to use tweet for inactive_*
   alias :old_followers :followers
   def followers(*args)
-    options = {count: 200, include_user_entities: true}.merge(args.extract_options!)
-    collect_with_cursor(:old_followers, *args, options)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    options[:reduce] = false unless options.has_key?(:reduce)
+    fetch_cache_or_call_api(:followers, args[0], options) {
+      options = {count: 200, include_user_entities: true, cursor: -1}.merge(options)
+      collect_with_cursor(:old_followers, *args, options)
+    }
   end
+  # memoize :followers
 
-  alias :old_friend_ids :friend_ids
-  def friend_ids(*args)
-    options = {count: 5000}.merge(args.extract_options!)
-    collect_with_cursor(:old_friend_ids, *args, options)
+  def followers_advanced(*args)
+    options = args.extract_options!
+    _follower_ids = follower_ids(*(args + [options]))
+    users(_follower_ids.map { |id| id.to_i }, options)
   end
 
-  alias :old_follower_ids :follower_ids
-  def follower_ids(*args)
-    options = {count: 5000}.merge(args.extract_options!)
-    collect_with_cursor(:old_follower_ids, *args, options)
+  def fetch_parallelly(signatures) # [{method: :friends, args: ['ts_3156', ...], {...}]
+    logger.debug "#{__method__} #{signatures.inspect}"
+    result = Array.new(signatures.size)
+
+    Parallel.each_with_index(signatures, in_threads: result.size) do |signature, i|
+      result[i] = send(signature[:method], *signature[:args])
+    end
+
+    result
+  end
+
+  def friends_and_followers(*args)
+    fetch_parallelly(
+      [
+        {method: 'friends_advanced', args: args},
+        {method: 'followers_advanced', args: args}])
+  end
+
+  def friends_followers_and_statuses(*args)
+    fetch_parallelly(
+      [
+        {method: 'friends_advanced', args: args},
+        {method: 'followers_advanced', args: args},
+        {method: 'user_timeline', args: args}])
+  end
+
+  def one_sided_following(me)
+    me.friends.to_a - me.followers.to_a
+  end
+
+  def one_sided_followers(me)
+    me.followers.to_a - me.friends.to_a
+  end
+
+  def mutual_friends(me)
+    me.friends.to_a & me.followers.to_a
   end
 
+  def common_friends(me, you)
+    me.friends.to_a & you.friends.to_a
+  end
+
+  def common_followers(me, you)
+    me.followers.to_a & you.followers.to_a
+  end
+
+  def removing(pre_me, cur_me)
+    pre_me.friends.to_a - cur_me.friends.to_a
+  end
+
+  def removed(pre_me, cur_me)
+    pre_me.followers.to_a - cur_me.followers.to_a
+  end
+
+  # use compact, not use sort and uniq
+  # specify reduce: false to use tweet for inactive_*
   alias :old_users :users
-  def users(ids, options = {})
-    ids_per_worker = ids.each_slice(100).to_a
+  def users(*args)
+    options = args.extract_options!
+    options[:reduce] = false
+    users_per_workers = args.first.compact.each_slice(100).to_a
     processed_users = []
 
-    Parallel.each_with_index(ids_per_worker, in_threads: ids_per_worker.size) do |ids, i|
-      _users = {i: i, users: old_users(ids, options)}
-      processed_users << _users
+    Parallel.each_with_index(users_per_workers, in_threads: [users_per_workers.size, 10].min) do |users_per_worker, i|
+      _users = fetch_cache_or_call_api(:users, users_per_worker, options) {
+        call_old_method(:old_users, users_per_worker, options)
+      }
+
+      result = {i: i, users: _users}
+      processed_users << result
     end
 
-    processed_users.sort_by{|p| p[:i] }.map{|p| p[:users] }.flatten
-  end
-
-  # Returns tweets that match a specified query.
-  #
-  # @see https://dev.twitter.com/docs/api/1.1/get/search/tweets
-  # @see https://dev.twitter.com/docs/using-search
-  # @note Please note that Twitter's search service and, by extension, the Search API is not meant to be an exhaustive source of Tweets. Not all Tweets will be indexed or made available via the search interface.
-  # @rate_limited Yes
-  # @authentication Requires user context
-  # @raise [Twitter::Error::Unauthorized] Error raised when supplied user credentials are not valid.
-  # @param q [String] A search term. (from|to):hello min_retweets:3 OR min_faves:3 OR min_replies:3, #nhk @hello (in|ex)clude:retweets https://twitter.com/search-advanced
-  # @param options [Hash] A customizable set of options.
-  # @option options [String] :geocode Returns tweets by users located within a given radius of the given latitude/longitude. The location is preferentially taking from the Geotagging API, but will fall back to their Twitter profile. The parameter value is specified by "latitude,longitude,radius", where radius units must be specified as either "mi" (miles) or "km" (kilometers). Note that you cannot use the near operator via the API to geocode arbitrary locations; however you can use this geocode parameter to search near geocodes directly.
-  # @option options [String] :lang Restricts tweets to the given language, given by an ISO 639-1 code.
-  # @option options [String] :locale Specify the language of the query you are sending (only ja is currently effective). This is intended for language-specific clients and the default should work in the majority of cases.
-  # @option options [String] :result_type Specifies what type of search results you would prefer to receive. Options are "mixed", "recent", and "popular". The current default is "mixed."
-  # @option options [Integer] :count The number of tweets to return per page, up to a maximum of 100.
-  # @option options [String] :until Optional. Returns tweets generated before the given date. Date should be formatted as YYYY-MM-DD.
-  # @option options [Integer] :since_id Returns results with an ID greater than (that is, more recent than) the specified ID. There are limits to the number of Tweets which can be accessed through the API. If the limit of Tweets has occured since the since_id, the since_id will be forced to the oldest ID available.
-  # @option options [Integer] :max_id Returns results with an ID less than (that is, older than) or equal to the specified ID.
-  # @return [Twitter::SearchResults] Return tweets that match a specified query with search metadata
+    processed_users.sort_by{|p| p[:i] }.map{|p| p[:users] }.flatten.compact
+  rescue => e
+    logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
+  def called_by_authenticated_user?(user)
+    authenticated_user = self.old_user; self.call_count += 1
+    if user.kind_of?(String)
+      authenticated_user.screen_name == user
+    elsif user.kind_of?(Integer)
+      authenticated_user.id.to_i == user
+    else
+      raise user.inspect
+    end
+  rescue => e
+    logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
+  # can't get tweets if you are not authenticated by specified user
+  alias :old_home_timeline :home_timeline
+  def home_timeline(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    raise 'this method must be called by authenticated user' unless called_by_authenticated_user?(args[0])
+    options = args.extract_options!
+    fetch_cache_or_call_api(:home_timeline, args[0], options) {
+      options = {count: 200, include_rts: true, call_count: 3}.merge(options)
+      collect_with_max_id(:old_home_timeline, options)
+    }
+  end
+
+  # can't get tweets if you are not authenticated by specified user
+  alias :old_user_timeline :user_timeline
+  def user_timeline(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:user_timeline, args[0], options) {
+      options = {count: 200, include_rts: true, call_count: 3}.merge(options)
+      collect_with_max_id(:old_user_timeline, *args, options)
+    }
+  end
+
+  # can't get tweets if you are not authenticated by specified user
+  alias :old_mentions_timeline :mentions_timeline
+  def mentions_timeline(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    raise 'this method must be called by authenticated user' unless called_by_authenticated_user?(args[0])
+    options = args.extract_options!
+    fetch_cache_or_call_api(:mentions_timeline, args[0], options) {
+      options = {count: 200, include_rts: true, call_count: 1}.merge(options)
+      collect_with_max_id(:old_mentions_timeline, options)
+    }
+  rescue => e
+    logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
+  def select_screen_names_replied(tweets, options = {})
+    result = tweets.map do |t|
+      $1 if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
+    end.compact
+    (options.has_key?(:uniq) && !options[:uniq]) ? result : result.uniq
+  end
+
+  # users which specified user is replying
+  # in_reply_to_user_id and in_reply_to_status_id is not used because of distinguishing mentions from replies
+  def replying(user, options = {})
+    tweets = options.has_key?(:tweets) ? options.delete(:tweets) : user_timeline(user, options)
+    screen_names = select_screen_names_replied(tweets, options)
+    users(screen_names, options)
+  rescue Twitter::Error::NotFound => e
+    e.message == 'No user matches for specified terms.' ? [] : (raise e)
+  rescue => e
+    logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
   alias :old_search :search
   def search(*args)
-    options = {count: 100, result_type: 'recent'}.merge(args.extract_options!)
-    collect_with_max_id(:old_search, *args, options) do |data, last_response|
-      statuses = last_response.attrs[:statuses]
-      data.concat(statuses)
-      statuses
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    options[:reduce] = false
+    fetch_cache_or_call_api(:search, args[0], options) {
+      options = {count: 100, result_type: :recent, call_count: 1}.merge(options)
+      collect_with_max_id(:old_search, *args, options) { |response| response.attrs[:statuses] }
+    }
+  rescue => e
+    logger.warn "#{__method__} #{args.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
+  def select_uids_replying_to(tweets, options)
+    result = tweets.map do |t|
+      t.user.id.to_i if t.text =~ /^(?:\.)?@(\w+)( |\W)/ # include statuses starts with .
+    end.compact
+    (options.has_key?(:uniq) && !options[:uniq]) ? result : result.uniq
+  end
+
+  def select_replied_from_search(tweets, options = {})
+    return [] if tweets.empty?
+    uids = select_uids_replying_to(tweets, options)
+    uids.map { |u| tweets.find { |t| t.user.id.to_i == u.to_i } }.map { |t| t.user }
+  end
+
+  # users which specified user is replied
+  # when user is login you had better to call mentions_timeline
+  def replied(_user, options = {})
+    user = self.user(_user, options)
+    if user.id.to_i == __uid_i
+      mentions_timeline(__uid_i, options).uniq { |m| m.user.id }.map { |m| m.user }
+    else
+      select_replied_from_search(search('@' + user.screen_name, options))
     end
+  rescue => e
+    logger.warn "#{__method__} #{_user.inspect} #{e.class} #{e.message}"
+    raise e
   end
 
-  # mentions_timeline is to fetch the timeline of Tweets mentioning the authenticated user
-  # get_mentions is to fetch the Tweets mentioning the screen_name's user
-  def get_mentions(screen_name)
-    search("to:#{screen_name}")
+  def select_inactive_users(users, options = {})
+    options[:authorized] = false unless options.has_key?(:authorized)
+    two_weeks_ago = 2.weeks.ago.to_i
+    users.select do |u|
+      if options[:authorized] || !u.protected
+        (Time.parse(u.status.created_at).to_i < two_weeks_ago) rescue false
+      else
+        false
+      end
+    end
+  end
+
+  def inactive_friends(user)
+    select_inactive_users(friends_advanced(user))
   end
 
-  def search_japanese_tweets(str)
-    search(str, {lang: 'ja'})
+  def inactive_followers(user)
+    select_inactive_users(followers_advanced(user))
   end
 
-  def search_tweets_except_rt(str)
-    search("#{str} exclude:retweets")
+  def clusters_belong_to(text)
+    return [] if text.blank?
+
+    exclude_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_bad_words_path']))
+    special_words = JSON.parse(File.read(Rails.configuration.x.constants['cluster_good_words_path']))
+
+    # クラスタ用の単語の出現回数を記録
+    cluster_word_counter =
+      special_words.map { |sw| [sw, text.scan(sw)] }
+        .delete_if { |item| item[1].empty? }
+        .each_with_object(Hash.new(1)) { |item, memo| memo[item[0]] = item[1].size }
+
+    # 同一文字種の繰り返しを見付ける。漢字の繰り返し、ひらがなの繰り返し、カタカナの繰り返し、など
+    text.scan(/[一-龠〆ヵヶ々]+|[ぁ-んー～]+|[ァ-ヴー～]+|[ａ-ｚＡ-Ｚ０-９]+|[、。！!？?]+/).
+
+      # 複数回繰り返される文字を除去
+      map { |w| w.remove /[？！?!。、ｗ]|(ー{2,})/ }.
+
+      # 文字数の少なすぎる単語、ひらがなだけの単語、除外単語を除去する
+      delete_if { |w| w.length <= 1 || (w.length <= 2 && w =~ /^[ぁ-んー～]+$/) || exclude_words.include?(w) }.
+
+      # 出現回数を記録
+      each { |w| cluster_word_counter[w] += 1 }
+
+    # 複数個以上見付かった単語のみを残し、出現頻度順にソート
+    cluster_word_counter.select { |_, v| v > 3 }.sort_by { |_, v| -v }.to_h
   end
-end
 
+  def clusters_assigned_to
 
+  end
+
+  def usage_stats_wday_series_data(times)
+    wday_count = times.each_with_object((0..6).map { |n| [n, 0] }.to_h) do |time, memo|
+      memo[time.wday] += 1
+    end
+    wday_count.map { |k, v| [I18n.t('date.abbr_day_names')[k], v] }.map do |key, value|
+      {name: key, y: value, drilldown: key}
+    end
+  end
+
+  def usage_stats_wday_drilldown_series(times)
+    hour_count =
+      (0..6).each_with_object((0..6).map { |n| [n, nil] }.to_h) do |wday, wday_memo|
+        wday_memo[wday] =
+          times.select { |t| t.wday == wday }.map { |t| t.hour }.each_with_object((0..23).map { |n| [n, 0] }.to_h) do |hour, hour_memo|
+            hour_memo[hour] += 1
+          end
+      end
+    hour_count.map { |k, v| [I18n.t('date.abbr_day_names')[k], v] }.map do |key, value|
+      {name: key, id: key, data: value.to_a.map{|a| [a[0].to_s, a[1]] }}
+    end
+  end
+
+  def usage_stats_hour_series_data(times)
+    hour_count = times.each_with_object((0..23).map { |n| [n, 0] }.to_h) do |time, memo|
+      memo[time.hour] += 1
+    end
+    hour_count.map do |key, value|
+      {name: key.to_s, y: value, drilldown: key.to_s}
+    end
+  end
+
+  def usage_stats_hour_drilldown_series(times)
+    wday_count =
+      (0..23).each_with_object((0..23).map { |n| [n, nil] }.to_h) do |hour, hour_memo|
+        hour_memo[hour] =
+          times.select { |t| t.hour == hour }.map { |t| t.wday }.each_with_object((0..6).map { |n| [n, 0] }.to_h) do |wday, wday_memo|
+            wday_memo[wday] += 1
+          end
+      end
+    wday_count.map do |key, value|
+      {name: key.to_s, id: key.to_s, data: value.to_a.map{|a| [I18n.t('date.abbr_day_names')[a[0]], a[1]] }}
+    end
+  end
+
+  def twitter_addiction_series(times)
+    five_mins = 5.minutes
+    wday_expended_seconds =
+      (0..6).each_with_object((0..6).map { |n| [n, nil] }.to_h) do |wday, wday_memo|
+        target_times = times.select { |t| t.wday == wday }
+        wday_memo[wday] = target_times.empty? ? nil : target_times.each_cons(2).map {|a, b| (a - b) < five_mins ? a - b : five_mins }.sum
+      end
+    days = times.map{|t| t.to_date.to_s(:long) }.uniq.size
+    weeks = (days > 7) ? days / 7.0 : 1.0
+    wday_expended_seconds.map { |k, v| [I18n.t('date.abbr_day_names')[k], (v.nil? ? nil : v / weeks / 60)] }.map do |key, value|
+      {name: key, y: value}
+    end
+  end
+
+  def usage_stats(user, options = {})
+    n_days_ago = options.has_key?(:days) ? options[:days].days.ago : 100.years.ago
+    tweets = options.has_key?(:tweets) ? options.delete(:tweets) : user_timeline(user)
+    times =
+      # TODO Use user specific time zone
+      tweets.map { |t| ActiveSupport::TimeZone['Tokyo'].parse(t.created_at.to_s) }.
+        select { |t| t > n_days_ago }
+    [
+      usage_stats_wday_series_data(times),
+      usage_stats_wday_drilldown_series(times),
+      usage_stats_hour_series_data(times),
+      usage_stats_hour_drilldown_series(times),
+      twitter_addiction_series(times)
+    ]
+  end
+
+
+  alias :old_favorites :favorites
+  def favorites(*args)
+    raise 'this method needs at least one param to use cache' if args.empty?
+    options = args.extract_options!
+    fetch_cache_or_call_api(:favorites, args[0], options) {
+      options = {count: 100, call_count: 1}.merge(options)
+      collect_with_max_id(:old_favorites, *args, options)
+    }
+  end
+
+  def calc_scores_from_users(users, options)
+    min = options.has_key?(:min) ? options[:min] : 0
+    max = options.has_key?(:max) ? options[:max] : 1000
+    users.each_with_object(Hash.new(0)) { |u, memo| memo[u.id] += 1 }.
+      select { |_k, v| min <= v && v <= max }.
+      sort_by { |_, v| -v }.to_h
+  end
+
+  def calc_scores_from_tweets(tweets, options = {})
+    calc_scores_from_users(tweets.map { |t| t.user }, options)
+  end
+
+  def select_favoriting_from_favs(favs, options = {})
+    return [] if favs.empty?
+    uids = calc_scores_from_tweets(favs)
+    result = uids.map { |uid, score| f = favs.
+      find { |f| f.user.id.to_i == uid.to_i }; Array.new(score, f) }.flatten.map { |f| f.user }
+    (options.has_key?(:uniq) && !options[:uniq]) ? result : result.uniq { |u| u.id }
+  end
+
+  def favoriting(user, options= {})
+    favs = options.has_key?(:favorites) ? options.delete(:favorites) : favorites(user, options)
+    select_favoriting_from_favs(favs, options)
+  rescue => e
+    logger.warn "#{__method__} #{user.inspect} #{e.class} #{e.message}"
+    raise e
+  end
+
+  def favorited_by(user)
+  end
+
+  def close_friends(_uid, options = {})
+    min = options.has_key?(:min) ? options[:min] : 0
+    max = options.has_key?(:max) ? options[:max] : 1000
+    uid_i = _uid.to_i
+    _replying = options.has_key?(:replying) ? options.delete(:replying) : replying(uid_i, options)
+    _replied = options.has_key?(:replied) ? options.delete(:replied) : replied(uid_i, options)
+    _favoriting = options.has_key?(:favoriting) ? options.delete(:favoriting) : favoriting(uid_i, options)
+
+    min_max = {min: min, max: max}
+    _users = _replying + _replied + _favoriting
+    return [] if _users.empty?
+
+    scores = calc_scores_from_users(_users, min_max)
+    replying_scores = calc_scores_from_users(_replying, min_max)
+    replied_scores = calc_scores_from_users(_replied, min_max)
+    favoriting_scores = calc_scores_from_users(_favoriting, min_max)
+
+    scores.keys.map { |uid| _users.find { |u| u.id.to_i == uid.to_i } }.
+      map do |u|
+      u[:score] = scores[u.id]
+      u[:replying_score] = replying_scores[u.id]
+      u[:replied_score] = replied_scores[u.id]
+      u[:favoriting_score] = favoriting_scores[u.id]
+      u
+    end
+  end
+end

data/lib/ex_twitter_subscriber.rb

@@ -0,0 +1,8 @@
+require 'active_support'
+require 'active_support/subscriber'
+
+class ExTwitterSubscriber < ActiveSupport::Subscriber
+  def call(event)
+    puts "#{event.payload[:name]} #{event.duration} ms"
+  end
+end

data/spec/ex_twitter_spec.rb

@@ -6,38 +6,32 @@ describe ExTwitter do
       consumer_key: 'CK',
       consumer_secret: 'CS',
       access_token: 'AT',
-      access_token_secret: 'ATS'
+      access_token_secret: 'ATS',
     }
   }
   let(:client) { ExTwitter.new(config) }
 
   describe '#initialize' do
-    let(:default_value) { 3 }
-    let(:value) { 100 }
+    let(:default_call_count) { 0 }
 
-    context 'without params' do
-      it "doesn't set max_paginates" do
-        expect(ExTwitter.new.max_paginates).to eq(default_value)
-      end
+    it 'sets call_count to 0' do
+      expect(client.call_count).to eq(default_call_count)
     end
 
-    context 'with params' do
-      it 'set max_paginates' do
-        expect(ExTwitter.new(max_paginates: value).max_paginates).to eq(value)
-      end
+    context 'without params' do
     end
 
-    context 'with block' do
-      it 'set max_paginates' do
-        expect(ExTwitter.new {|config| config.max_paginates = value }.max_paginates).to eq(value)
-      end
+    context 'with params' do
     end
   end
 
-  describe '#read' do
+  describe '#logger' do
+    it 'has logger' do
+      expect(client.logger).to be_truthy
+    end
   end
 
-  describe '#write' do
+  describe '#call_old_method' do
   end
 
   describe '#collect_with_max_id' do
@@ -46,45 +40,92 @@ describe ExTwitter do
   describe '#collect_with_cursor' do
   end
 
-  # describe '#user_timeline' do
-  #   it 'call collect_with_max_id' do
-  #     expect(client).to receive(:collect_with_max_id)
-  #     client.user_timeline
-  #   end
-  # end
-  #
-  # describe '#friends' do
-  #   it 'call collect_with_cursor' do
-  #     expect(client).to receive(:collect_with_cursor)
-  #     client.friends
-  #   end
-  # end
-  #
-  # describe '#followers' do
-  #   it 'call collect_with_cursor' do
-  #     expect(client).to receive(:collect_with_cursor)
-  #     client.followers
-  #   end
-  # end
-  #
-  # describe '#friend_ids' do
-  #   it 'call collect_with_cursor' do
-  #     expect(client).to receive(:collect_with_cursor)
-  #     client.friend_ids
-  #   end
-  # end
-  #
-  # describe '#follower_ids' do
-  #   it 'call collect_with_cursor' do
-  #     expect(client).to receive(:collect_with_cursor)
-  #     client.follower_ids
-  #   end
-  # end
-  #
-  # describe '#users' do
-  #   it 'call old_users' do
-  #     expect(client).to receive(:old_users)
-  #     client.users([1, 2, 3])
-  #   end
-  # end
+  describe '#file_cache_key' do
+  end
+
+  describe '#namespaced_key' do
+  end
+
+  describe '#encode_json' do
+  end
+
+  describe '#decode_json' do
+  end
+
+  describe '#fetch_cache_or_call_api' do
+  end
+
+  describe '#user_timeline' do
+    it 'calls old_user_timeline' do
+      expect(client).to receive(:old_user_timeline)
+      client.user_timeline
+    end
+
+    it 'calls collect_with_max_id' do
+      expect(client).to receive(:collect_with_max_id)
+      client.user_timeline
+    end
+  end
+
+  describe '#user_photos' do
+    it 'calls user_timeline' do
+      expect(client).to receive(:user_timeline)
+      client.user_photos
+    end
+  end
+
+  describe '#friends' do
+    it 'calls old_friends' do
+      expect(client).to receive(:old_friends)
+      client.friends
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.friends
+    end
+  end
+
+  describe '#followers' do
+    it 'calls old_followers' do
+      expect(client).to receive(:old_followers)
+      client.followers
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.followers
+    end
+  end
+
+  describe '#friend_ids' do
+    it 'calls old_friend_ids' do
+      expect(client).to receive(:old_friend_ids)
+      client.friend_ids
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.friend_ids
+    end
+  end
+
+  describe '#follower_ids' do
+    it 'calls old_follower_ids' do
+      expect(client).to receive(:old_follower_ids)
+      client.follower_ids
+    end
+
+    it 'calls collect_with_cursor' do
+      expect(client).to receive(:collect_with_cursor)
+      client.follower_ids
+    end
+  end
+
+  describe '#users' do
+    it 'calls old_users' do
+      expect(client).to receive(:old_users)
+      client.users([1, 2, 3])
+    end
+  end
 end

data/spec/helper.rb

@@ -5,4 +5,5 @@ RSpec.configure do |config|
   config.expect_with :rspec do |c|
     c.syntax = :expect
   end
+  config.order = 'random'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ex_twitter
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Shinohara Teruki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-29 00:00:00.000000000 Z
+date: 2016-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -79,6 +93,7 @@ files:
 - ex_twitter.gemspec
 - lib/ex_stream.rb
 - lib/ex_twitter.rb
+- lib/ex_twitter_subscriber.rb
 - spec/ex_twitter_spec.rb
 - spec/helper.rb
 homepage: http://github.com/ts-3156/ex-twitter/
@@ -101,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Add auto paginate feature to Twitter gem.