twitter_with_auto_pagination 0.8.3 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21617993d79d35c184e0a5eb3bd68cedf7f9fe36
-  data.tar.gz: 017b1a7d412e514c697f4296292fc7075bbff88d
+  metadata.gz: 7c6e0231b2860c7fca4ae5d2b5b8c8a7ab397c8b
+  data.tar.gz: 9f8698b8e86923e4855d75be50440ac803a96d3e
 SHA512:
-  metadata.gz: c7667ea3c41a50167aa7765ca7bce2a6d6a321ea71f417a44f57fb213b258d6425190148fe944d371a41b66165e1e76814694c99a4aee1d290425b1c53ae2d93
-  data.tar.gz: 49ac0e5a907bf133ed1a8ab304873c5efae18c525b4a1b56e7291178a459bfdf9f6b3989e5d8acecbc3c31316488f85280a89ca83c8bb14ec9d3763c070240f6
+  metadata.gz: bf30ac3e1930e548e99941977260e5bf9ac70d48b51e5a978638cd4b7b71a998cda96f7dc1b9ea12e2587a4a70992bcb5df99cddd96086f84b1824bfdb07edaf
+  data.tar.gz: a2c5ca9ec3c096d8a892f616139211ee76eafc4e7a27a5c1e93a2259e38387a5fdf5e16d5f1d39e02e4ac199e64c8c14cb6861b0685f4ba5eb74e769396cacfe

data/lib/twitter_with_auto_pagination/rest/extension/clusters.rb

@@ -45,8 +45,13 @@ module TwitterWithAutoPagination
 
         alias tweet_clusters clusters_belong_to
 
-        def list_clusters(user, each_member: 300, total_member: 1000, rate: 0.3, limit: 10, debug: false)
-          lists = memberships(user).sort_by { |li| li.member_count }
+        def list_clusters(user, shrink: false, each_member: 300, total_member: 1000, rate: 0.3, limit: 10, debug: false)
+          begin
+            lists = memberships(user).sort_by { |li| li.member_count }
+          rescue => e
+            puts "#{e.class}: #{e.message} #{user.inspect}" if debug
+            lists = []
+          end
           puts "lists: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
           return {} if lists.empty?
 
@@ -55,47 +60,48 @@ module TwitterWithAutoPagination
             percentile75 = ((lists.length * 0.75).ceil) - 1
             lists = lists[percentile25..percentile75]
             puts "lists sliced by 25-75 percentile: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
-          end
+          end if shrink
 
           list_special_words = %w()
-          list_exclude_words = %w(list people met)
+          list_exclude_words1 = %r(list[0-9]*|people-ive-faved|twizard-magic-list|my-favstar-fm-list)
+          list_exclude_words2 = %w(it list people met)
 
-          words = lists.map { |li| li.full_name.split('/')[1].split('-') }.flatten.delete_if { |w| w.size < 2 || list_exclude_words.include?(w) }.
-            each_with_object(Hash.new(0)) { |w, memo| memo[w] += 1 }.select { |_, v| (10 < lists.size ? 1 : 0) < v }.sort_by { |k, v| [-v, -k.size] }
+          words = lists.map { |li| li.full_name.split('/')[1] }.
+            select { |n| !n.match(list_exclude_words1) }.
+            map { |n| n.split('-') }.flatten.
+            delete_if { |w| w.size < 2 || list_exclude_words2.include?(w) }.
+            each_with_object(Hash.new(0)) { |w, memo| memo[w] += 1 }.
+            sort_by { |k, v| [-v, -k.size] }
 
           puts "words: #{words.slice(0, 10)}" if debug
           return {} if words.empty?
 
-          word = words[0][0]
-          puts "word: #{word}" if debug
-
-          # TODO: listsの数が小さすぎる場合はwordを増やす
-          lists = lists.select { |li| li.full_name.split('/')[1].include?(word) }
-          puts "lists include specified word: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          _words = []
+          lists =
+            filter(lists, min: 2) do |li, i|
+              _words = words[0..i].map(&:first)
+              name = li.full_name.split('/')[1]
+              _words.any? { |w| name.include?(w) }
+            end
+          puts "lists include #{_words.inspect}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
           return {} if lists.empty?
 
-          3.times do
-            _lists = lists.select { |li| (10 < lists.size ? 10 : 0) < li.member_count && li.member_count < each_member }
-            if _lists.size > 2 || _lists.size == lists.size
-              lists = _lists
-              break
-            else
-              each_member *= 1.25
+          _each_member = 0
+          lists =
+            filter(lists, min: 2) do |li, i|
+              _each_member = each_member * (1.0 + 0.25 * i)
+              (10 < lists.size ? 10 : 0) < li.member_count && li.member_count < _each_member
             end
-          end
-          puts "lists limited by each member #{each_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          puts "lists limited by each member #{_each_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
           return {} if lists.empty?
 
-          3.times do
-            _lists = lists.select.with_index { |_, i| lists[0..i].map { |li| li.member_count }.sum < total_member }
-            if _lists.any?
-              lists = _lists
-              break
-            else
-              total_member *= 1.25
+          _total_member = 0
+          lists =
+            filter(lists, min: 1) do |_, i|
+              _total_member = total_member * (1.0 + 0.25 * i)
+              lists[0..i].map { |li| li.member_count }.sum < _total_member
             end
-          end
-          puts "lists limited by total members #{total_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
+          puts "lists limited by total members #{_total_member}: #{lists.size} (#{lists.map { |li| li.member_count }.join(', ')})" if debug
           return {} if lists.empty?
 
           members = lists.map do |li|
@@ -110,7 +116,8 @@ module TwitterWithAutoPagination
           return {} if members.empty?
 
           3.times do
-            _members = members.each_with_object(Hash.new(0)) { |member, memo| memo[member] += 1 }.select { |_, v| lists.size * rate < v }.keys
+            _members = members.each_with_object(Hash.new(0)) { |member, memo| memo[member] += 1 }.
+              select { |_, v| lists.size * rate < v }.keys
             if _members.size > 100
               members = _members
               break
@@ -120,7 +127,6 @@ module TwitterWithAutoPagination
           end
           puts "members included multi lists #{rate}: #{members.size}" if debug
 
-          require 'mecab'
 
           profile_special_words = %w()
           profile_exclude_words = %w(in at of my no er the and for inc Inc com gmail 好き こと 最近 情報 さん ちゃん くん 発言 関係 もの 活動 見解 所属 組織 連絡 大好き サイト ブログ つぶやき こちら アカ アカウント イベント フォロー)
@@ -128,37 +134,49 @@ module TwitterWithAutoPagination
           descriptions = members.map { |m| m.description.remove(URI.regexp) }
 
           candidates, remains = descriptions.partition { |desc| desc.scan('/').size > 2 }
-          slash_freq = count_by_word_with_delim(candidates, delim: '/')
+          slash_freq = count_by_word(candidates, delim: '/')
           puts "words splitted by /: #{slash_freq.to_a.slice(0, 10)}" if debug
 
           candidates, remains = remains.partition { |desc| desc.scan('|').size > 2 }
-          pipe_freq = count_by_word_with_delim(candidates, delim: '|')
+          pipe_freq = count_by_word(candidates, delim: '|')
           puts "words splitted by |: #{pipe_freq.to_a.slice(0, 10)}" if debug
 
-          noun_freq = count_by_word_with_tagger(remains, exclude_words: profile_exclude_words)
-          puts "words with nouns added: #{noun_freq.to_a.slice(0, 10)}" if debug
+          require 'mecab'
+          tagger = MeCab::Tagger.new("-d #{`mecab-config --dicdir`.chomp}/mecab-ipadic-neologd/")
+
+          noun_freq = count_by_word(remains, tagger: tagger, exclude_words: profile_exclude_words)
+          puts "words tagged as noun: #{noun_freq.to_a.slice(0, 10)}" if debug
 
           slash_freq.merge(pipe_freq) { |_, old, neww| old + neww }.merge(noun_freq) { |_, old, neww| old + neww }.sort_by { |k, v| [-v, -k.size] }.slice(0, limit)
         end
 
         private
 
-        def count_by_word_with_delim(texts, delim:)
-          texts.map { |t| t.split(delim) }.flatten.
-            map(&:strip).
-            delete_if { |w| w.empty? || w.size < 2 || 5 < w.size }.
-            each_with_object(Hash.new(0)) { |w, memo| memo[w] += 1 }.
-            sort_by { |k, v| [-v, -k.size] }.to_h
+        def filter(lists, min:)
+          min = [min, lists.size].min
+          _lists = []
+          3.times do |i|
+            _lists = lists.select { |li| yield(li, i) }
+            break if _lists.size >= min
+          end
+          _lists
         end
 
-        def count_by_word_with_tagger(texts, tagger: nil, exclude_words: [])
-          tagger = MeCab::Tagger.new("-d #{`mecab-config --dicdir`.chomp}/mecab-ipadic-neologd/") if tagger.nil?
-          nouns = tagger.parse(texts.join(' ')).split("\n").
-            select { |line| line.include?('名詞') }.
-            map { |line| line.split("\t")[0] }.
-            delete_if { |w| w.empty? || w.size < 2 || 5 < w.size || exclude_words.include?(w) }
+        def count_by_word(texts, delim: nil, tagger: nil, exclude_words: [])
+          texts = texts.dup
+
+          if delim
+            texts = texts.map { |t| t.split(delim) }.flatten.map(&:strip)
+          end
+
+          if tagger
+            texts = tagger.parse(texts.join(' ')).split("\n").
+              select { |line| line.include?('名詞') }.
+              map { |line| line.split("\t")[0] }
+          end
 
-          nouns.each_with_object(Hash.new(0)) { |noun, memo| memo[noun] += 1 }.
+          texts.delete_if { |w| w.empty? || w.size < 2 || 5 < w.size || exclude_words.include?(w) }.
+            each_with_object(Hash.new(0)) { |word, memo| memo[word] += 1 }.
             sort_by { |k, v| [-v, -k.size] }.to_h
         end
       end

data/twitter_with_auto_pagination.gemspec

@@ -23,5 +23,5 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.3'
   spec.summary = spec.description
   spec.test_files = Dir.glob('spec/**/*')
-  spec.version = '0.8.3'
+  spec.version = '0.8.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: twitter_with_auto_pagination
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
 platform: ruby
 authors:
 - Shinohara Teruki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-04 00:00:00.000000000 Z
+date: 2016-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: twitter