twitter_with_auto_pagination 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35aa309dc97a81e62812edb59c1fe61c62da69b9
-  data.tar.gz: 9c3a07ab41c467685e144edadda32c3c399b1b47
+  metadata.gz: 21617993d79d35c184e0a5eb3bd68cedf7f9fe36
+  data.tar.gz: 017b1a7d412e514c697f4296292fc7075bbff88d
 SHA512:
-  metadata.gz: 55f1e531dc9dbaa174cff7f1a13dbec4a8e6ea85c920b58af0525048efcbbbee3746444790fb3120ea7e24fab647db4e2180a0a218811c0f0cf5421f59ca70b3
-  data.tar.gz: 52859557ffae6a65e7fc4c56900aab1d0dc234a4641a1ed22db744e6a2ea70a1e137de8fcddebb8239fe5a3cdaadde68d267387f5939ce5e111f35d38c6175aa
+  metadata.gz: c7667ea3c41a50167aa7765ca7bce2a6d6a321ea71f417a44f57fb213b258d6425190148fe944d371a41b66165e1e76814694c99a4aee1d290425b1c53ae2d93
+  data.tar.gz: 49ac0e5a907bf133ed1a8ab304873c5efae18c525b4a1b56e7291178a459bfdf9f6b3989e5d8acecbc3c31316488f85280a89ca83c8bb14ec9d3763c070240f6

data/lib/twitter_with_auto_pagination/rest/extension/clusters.rb

@@ -43,8 +43,123 @@ module TwitterWithAutoPagination
           frequency.select { |_, v| 2 < v }.sort_by { |_, v| -v }.slice(0, limit).to_h
         end
 
-        def clusters_assigned_to
-          raise NotImplementedError
+        alias tweet_clusters clusters_belong_to
+
+        def list_clusters(user, each_member: 300, total_member: 1000, rate: 0.3, limit: 10, debug: false)
+          lists = memberships(user).sort_by { |li| li.member_count }
+          puts "lists: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          return {} if lists.empty?
+
+          while lists.size > 200
+            percentile25 = ((lists.length * 0.25).ceil) - 1
+            percentile75 = ((lists.length * 0.75).ceil) - 1
+            lists = lists[percentile25..percentile75]
+            puts "lists sliced by 25-75 percentile: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          end
+
+          list_special_words = %w()
+          list_exclude_words = %w(list people met)
+
+          words = lists.map { |li| li.full_name.split('/')[1].split('-') }.flatten.delete_if { |w| w.size < 2 || list_exclude_words.include?(w) }.
+            each_with_object(Hash.new(0)) { |w, memo| memo[w] += 1 }.select { |_, v| (10 < lists.size ? 1 : 0) < v }.sort_by { |k, v| [-v, -k.size] }
+
+          puts "words: #{words.slice(0, 10)}" if debug
+          return {} if words.empty?
+
+          word = words[0][0]
+          puts "word: #{word}" if debug
+
+          # TODO: listsの数が小さすぎる場合はwordを増やす
+          lists = lists.select { |li| li.full_name.split('/')[1].include?(word) }
+          puts "lists include specified word: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          return {} if lists.empty?
+
+          3.times do
+            _lists = lists.select { |li| (10 < lists.size ? 10 : 0) < li.member_count && li.member_count < each_member }
+            if _lists.size > 2 || _lists.size == lists.size
+              lists = _lists
+              break
+            else
+              each_member *= 1.25
+            end
+          end
+          puts "lists limited by each member #{each_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          return {} if lists.empty?
+
+          3.times do
+            _lists = lists.select.with_index { |_, i| lists[0..i].map { |li| li.member_count }.sum < total_member }
+            if _lists.any?
+              lists = _lists
+              break
+            else
+              total_member *= 1.25
+            end
+          end
+          puts "lists limited by total members #{total_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          return {} if lists.empty?
+
+          members = lists.map do |li|
+            begin
+              list_members(li.id)
+            rescue => e
+              puts "#{e.class}: #{e.message} #{li.id} #{li.full_name} #{li.mode}" if debug
+              nil
+            end
+          end.compact.flatten
+          puts "candidate members: #{members.size}" if debug
+          return {} if members.empty?
+
+          3.times do
+            _members = members.each_with_object(Hash.new(0)) { |member, memo| memo[member] += 1 }.select { |_, v| lists.size * rate < v }.keys
+            if _members.size > 100
+              members = _members
+              break
+            else
+              rate += 0.1
+            end
+          end
+          puts "members included multi lists #{rate}: #{members.size}" if debug
+
+          require 'mecab'
+
+          profile_special_words = %w()
+          profile_exclude_words = %w(in at of my no er the and for inc Inc com gmail 好き こと 最近 情報 さん ちゃん くん 発言 関係 もの 活動 見解 所属 組織 連絡 大好き サイト ブログ つぶやき こちら アカ アカウント イベント フォロー)
+
+          descriptions = members.map { |m| m.description.remove(URI.regexp) }
+
+          candidates, remains = descriptions.partition { |desc| desc.scan('/').size > 2 }
+          slash_freq = count_by_word_with_delim(candidates, delim: '/')
+          puts "words splitted by /: #{slash_freq.to_a.slice(0, 10)}" if debug
+
+          candidates, remains = remains.partition { |desc| desc.scan('|').size > 2 }
+          pipe_freq = count_by_word_with_delim(candidates, delim: '|')
+          puts "words splitted by |: #{pipe_freq.to_a.slice(0, 10)}" if debug
+
+          noun_freq = count_by_word_with_tagger(remains, exclude_words: profile_exclude_words)
+          puts "words with nouns added: #{noun_freq.to_a.slice(0, 10)}" if debug
+
+          slash_freq.merge(pipe_freq) { |_, old, neww| old + neww }.merge(noun_freq) { |_, old, neww| old + neww }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit)
+        end
+
+        private
+
+        def count_by_word_with_delim(texts, delim:)
+          texts.map { |t| t.split(delim) }.flatten.
+            map(&:strip).
+            delete_if { |w| w.empty? || w.size < 2 || 5 < w.size }.
+            each_with_object(Hash.new(0)) { |w, memo| memo[w] += 1 }.
+            sort_by { |k, v| [-v, -k.size] }.to_h
+        end
+
+        def count_by_word_with_tagger(texts, tagger: nil, exclude_words: [])
+          tagger = MeCab::Tagger.new("-d #{`mecab-config --dicdir`.chomp}/mecab-ipadic-neologd/") if tagger.nil?
+          nouns = tagger.parse(texts.join(' ')).split("\n").
+            select { |line| line.include?('名詞') }.
+            map { |line| line.split("\t")[0] }.
+            delete_if { |w| w.empty? || w.size < 2 || 5 < w.size || exclude_words.include?(w) }
+
+          nouns.each_with_object(Hash.new(0)) { |noun, memo| memo[noun] += 1 }.
+            sort_by { |k, v| [-v, -k.size] }.to_h
         end
       end
     end

data/lib/twitter_with_auto_pagination/rest/lists.rb

@@ -14,6 +14,15 @@ module TwitterWithAutoPagination
           end
         end
       end
+
+      def list_members(*args)
+        options = {count: 5000, skip_status: 1, cursor: -1}.merge(args.extract_options!)
+        instrument(__method__, nil, options) do
+          fetch_cache_or_call_api(__method__, args[0], options) do
+            collect_with_cursor(method(__method__).super_method, *args, options)
+          end
+        end
+      end
     end
   end
 end

data/lib/twitter_with_auto_pagination/rest/utils.rb

@@ -108,7 +108,7 @@ module TwitterWithAutoPagination
         while (next_cursor = last_response[:next_cursor]) && next_cursor != 0
           options[:cursor] = next_cursor
           last_response = call_api(method_obj, *args, options).attrs
-          return_data += (last_response[:users] || last_response[:ids])
+          return_data += (last_response[:users] || last_response[:ids] || last_response[:lists])
         end
 
         return_data
@@ -122,6 +122,8 @@ module TwitterWithAutoPagination
               "hash-str#{delim}#{credentials_hash}"
             when method_name == :search
               "str#{delim}#{user.to_s}"
+            when method_name == :list_members
+              "list_id#{delim}#{user.to_s}"
             when method_name == :mentions_timeline
               "#{user.kind_of?(Integer) ? 'id' : 'sn'}#{delim}#{user.to_s}"
             when method_name == :home_timeline

data/twitter_with_auto_pagination.gemspec

@@ -6,6 +6,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
   spec.add_dependency 'hashie'
   spec.add_dependency 'parallel'
+  spec.add_dependency 'mecab'
 
   spec.add_development_dependency 'bundler'
 
@@ -22,5 +23,5 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.3'
   spec.summary = spec.description
   spec.test_files = Dir.glob('spec/**/*')
-  spec.version = '0.8.2'
+  spec.version = '0.8.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_with_auto_pagination
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Shinohara Teruki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-01 00:00:00.000000000 Z
+date: 2016-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: mecab
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement