highscore 1.1.0 → 1.2.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODQ1NGUyMjdjNGUzNjdlYTA4MjFlYjViMmY0NzM1MjBhMTU2ODlhOQ==
+  data.tar.gz: !binary |-
+    MTVmNzMyOTVhZjFkNjBhMTFjODc5NWM2MzdlN2E1NTljYzNjY2ExMw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZGVlOWUwNTgxM2MwMTlkMGMzN2MzYjViYjdhZDkxMjkxYmZiZTNkYTEzOThi
+    Yzg2Y2E4MjdmOGE1NWJhNDMzYjBkZGM4MDMyNzA4ODEwMmFiMTVmYWQyMzk2
+    ODIwOTIwMjRkMzgyN2E1Y2QzNTMxYjFmNzllM2NlOTE2ZjA4ZjM=
+  data.tar.gz: !binary |-
+    NDZlMzFiYjVjYmNiMDcyOGVlZGI1YWZkYmY1YWZhNWI5ZWRmNzgyNWU1NjZi
+    NzJlZDEzZjNmYWJhOGJhMWM3NTE1NDJiNDhlYWI1MTVkNmI1MjNhNzY0OTg0
+    MDhjZGY5MDkxY2FjZWU4NTU1NzcwYzNiNzA4MGVjZDlhMDVmMGI=

data/History.txt

@@ -1,3 +1,9 @@
+== 1.2.0 / 2013-12-06
+
+* configurable minimum word length
+* bonus words get rated higher than normal words (configurable just like blacklists)
+* (Thanks to Tim-B for the new features)
+
 == 1.1.0 / 2013-04-
 
 * added support for custom word ignore handlers using lambda functions

data/README.md

@@ -15,6 +15,7 @@ Easily find and rank keywords in long texts.
 * merge together Keywords from multiple sources
 * contains a CLI tool that operates on STDIN/OUT and is configurable via parameters
 * can use `bloomfilter-rb` gem for better performance (optional)
+* words on the bonus list will receive a higher score
 
 ## Installation
 
@@ -38,6 +39,8 @@ text.configure do
   set :upper_case, 3
   set :long_words, 2
   set :long_words_threshold, 15
+  set :short_words_threshold, 3      # => default: 2
+  set :bonus_multiplier, 2           # => default: 3
   set :vowels, 1                     # => default: 0 = not considered
   set :consonants, 5                 # => default: 0 = not considered
   set :ignore_case, true             # => default: false
@@ -106,6 +109,14 @@ whitelist = Highscore::Whitelist.load %w{these are valid keywords}
 content = Highscore::Content.new "invalid words", whitelist
 ```
 
+### Using bonus words
+
+```ruby
+# construct and inject it just like a blacklist
+bonuslist = Highscore::Bonuslist.load %w{bonus words}
+content = Highscore::Content.new "A string with bonus words in it", bonuslist
+```
+
 ## I18n
 
 ```ruby

data/bin/highscore

@@ -26,11 +26,21 @@ optparse = OptionParser.new do |opts|
     options[:wordlist] = Highscore::Whitelist.load_file(filepath)
   end
 
+  # bonus word file
+  opts.on('--bonuslist FILEPATH', 'specify a bonus word file') do |filepath|
+    options[:bonus_list] = Highscore::Bonuslist.load_file(filepath)
+  end
+
   # general multiplier
   opts.on('-m', '--multiplier MULTIPLIER', 'set the global rank multiplier') do |multiplier|
     options[:emphasis][:multiplier] = multiplier.to_f
   end
 
+  # bonus multiplier
+  opts.on('--bonusmultiplier MULTIPLIER', 'set the bonus word multiplier') do |multiplier|
+    options[:emphasis][:bonus_multiplier] = multiplier.to_f
+  end
+
   opts.on('--no-ignore-short', 'don\'t ignore short words (<= 2 chars)') do
     options[:emphasis][:ignore_short_words] = false
   end
@@ -68,6 +78,12 @@ optparse = OptionParser.new do |opts|
     options[:emphasis][:long_words_threshold] = u.to_i
   end
 
+  # short words threshold
+  opts.on('--shortwordsthreshold THRESHOLD', 'threshold for short words (default is 2 chars)') do |u|
+    options[:emphasis][:short_words_threshold] = u.to_i
+  end
+
+
   opts.on('-n', '--top N', 'show only the top N keywords') do |u|
     options[:top] = u.to_i
   end

data/lib/highscore.rb

@@ -14,6 +14,7 @@ module Highscore
       string
       whitelist
       wordlist
+      bonuslist
     )
     
     modules.each do |m|

data/lib/highscore/bonuslist.rb

@@ -0,0 +1,12 @@
+$:.unshift(File.dirname(__FILE__))
+require "wordlist"
+
+module Highscore
+  # Bonus words
+  #
+  class Bonuslist < Wordlist
+    def filter(keywords)
+      keywords
+    end
+  end
+end

data/lib/highscore/content.rb

@@ -10,13 +10,15 @@ module Highscore
     # @param wordlist Highscore::Wordlist
     def initialize(content, wordlist = nil)
       @content = content
-      @whitelist = @blacklist = nil
+      @whitelist = @blacklist = bonuslist = nil
       @language_wordlists = {}
 
       if wordlist.nil?
         @blacklist = Highscore::Blacklist.load_default_file
       elsif wordlist.kind_of? Highscore::Blacklist
         @blacklist = wordlist
+      elsif wordlist.kind_of? Highscore::Bonuslist
+        bonuslist = wordlist
       else
         @whitelist = wordlist
       end
@@ -25,6 +27,9 @@ module Highscore
         :multiplier => 1.0,
         :upper_case => 3.0,
         :long_words => 2.0,
+        :short_words_threshold => 2,
+        :bonus_multiplier => 3.0,
+        :bonus_list => bonuslist,
         :long_words_threshold => 15,
         :vowels => 0,
         :consonants => 0,
@@ -34,6 +39,7 @@ module Highscore
         :word_pattern => /\p{Word}+/u,
         :stemming => false
       }
+
     end
 
     # configure ranking
@@ -120,7 +126,7 @@ module Highscore
     #
     # @return TrueClass FalseClass
     def ignore? word
-      ignore = word.short?
+      ignore = word.short?(@emphasis[:short_words_threshold])
 
       # exception: allow short words
       ignore = (not allow_short_words?) if ignore
@@ -172,10 +178,19 @@ module Highscore
         weight *= @emphasis[:upper_case]
       end
 
+      weight += bonus(text)
       weight += vowels(text)
       weight + consonants(text)
     end
 
+    def bonus(text)
+      return 0 if not @emphasis[:bonus_list].kind_of? Highscore::Bonuslist
+      if @emphasis[:bonus_list].include?(text)
+        return @emphasis[:multiplier] * @emphasis[:bonus_multiplier]
+      end
+      return 0
+    end
+
     # weight the vowels on a text
     #
     # @param text String

data/lib/highscore/string.rb

@@ -38,7 +38,7 @@ class String
   # is this a short word?
   #
   # @return TrueClass|FalseClass
-  def short?
-    match(/^[\d]+(\.[\d]+){0,1}$/) or length <= 2
+  def short?(limit=2)
+    match(/^[\d]+(\.[\d]+){0,#{limit - 1}}$/) or length <= limit
   end
 end

data/test/highscore/test_bonuslist.rb

@@ -0,0 +1,61 @@
+require File.dirname(__FILE__) + '/../test_highscore'
+
+class TestBonuslist < Highscore::TestCase
+  def test_is_wordlist
+    bonuslist = Highscore::Bonuslist.new
+    assert bonuslist.kind_of? Highscore::Bonuslist
+  end
+
+  def test_bonus_content
+    bonuslist = Highscore::Bonuslist.load %w{Hacker}
+
+    text = Highscore::Content.new "Cats Cats Cats Cats Ruby Hacker", bonuslist
+
+    results = text.keywords.rank
+
+    assert_equal results[0].text, "Cats"
+    assert_equal results[1].text, "Hacker"
+    assert_equal results[2].text, "Ruby"
+
+    assert_equal results[0].weight, 12.0
+    assert_equal results[1].weight, 6.0
+    assert_equal results[2].weight, 3.0
+  end
+
+  def test_repeated_word
+    bonuslist = Highscore::Bonuslist.load %w{Hacker}
+
+    text = Highscore::Content.new "Cats Hacker Cats Cats Ruby Hacker", bonuslist
+
+    results = text.keywords.rank
+
+    assert_equal results[0].text, "Hacker"
+    assert_equal results[1].text, "Cats"
+    assert_equal results[2].text, "Ruby"
+
+    assert_equal results[0].weight, 12.0
+    assert_equal results[1].weight, 9.0
+    assert_equal results[2].weight, 3.0
+  end
+
+  def test_bonus_option
+
+    bonuslist = Highscore::Bonuslist.load %w{Hacker}
+
+    text = Highscore::Content.new "Cats Hacker Cats Cats Ruby Hacker", bonuslist
+
+    text.configure do
+      set :bonus_multiplier, 4
+    end
+
+    results = text.keywords.rank
+
+    assert_equal results[0].text, "Hacker"
+    assert_equal results[1].text, "Cats"
+    assert_equal results[2].text, "Ruby"
+
+    assert_equal results[0].weight, 14.0
+    assert_equal results[1].weight, 9.0
+    assert_equal results[2].weight, 3.0
+  end
+end

data/test/highscore/test_content.rb

@@ -60,6 +60,14 @@ class TestContent < Highscore::TestCase
     assert_equal 4, keywords.length
   end
 
+  def test_rank_short_words_limit
+    keywords = '56789 as 444 cat is foobar'.keywords do
+      set :short_words_threshold, 3
+    end
+
+    assert_equal 1, keywords.length
+  end
+
   def test_ignore_custom
     keywords = 'foobar a3832'.keywords do
       set :ignore, lambda { |w| w.gsub(/[^0-9]/, '').length > 2 }

data/version.txt

@@ -1 +1 @@
-1.1.0
+1.2.0

metadata

@@ -1,38 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: highscore
 version: !ruby/object:Gem::Version
-  version: 1.1.0
-  prerelease: 
+  version: 1.2.0
 platform: ruby
 authors:
 - Dominik Liebler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-13 00:00:00.000000000 Z
+date: 2013-12-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70314466936760 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.6.4
   type: :development
   prerelease: false
-  version_requirements: *70314466936760
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.4
 - !ruby/object:Gem::Dependency
   name: whatlanguage
-  requirement: &70314466936180 !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: bloomfilter-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
   type: :runtime
   prerelease: false
-  version_requirements: *70314466936180
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.1.1
 description: Find and rank keywords in text.
 email: liebler.dominik@googlemail.com
 executables:
@@ -44,23 +63,25 @@ extra_rdoc_files:
 files:
 - bin/highscore
 - lib/blacklist.txt
-- lib/highscore/blacklist.rb
-- lib/highscore/content.rb
-- lib/highscore/keyword.rb
+- lib/highscore.rb
+- lib/highscore/wordlist.rb
 - lib/highscore/keywords.rb
-- lib/highscore/string.rb
+- lib/highscore/bonuslist.rb
 - lib/highscore/whitelist.rb
-- lib/highscore/wordlist.rb
-- lib/highscore.rb
-- test/fixtures/blacklist.txt
-- test/highscore/test_blacklist.rb
-- test/highscore/test_content.rb
-- test/highscore/test_keyword.rb
-- test/highscore/test_keywords.rb
+- lib/highscore/keyword.rb
+- lib/highscore/string.rb
+- lib/highscore/content.rb
+- lib/highscore/blacklist.rb
+- test/highscore/test_bonuslist.rb
 - test/highscore/test_multiple_blacklists.rb
-- test/highscore/test_string.rb
+- test/highscore/test_keyword.rb
 - test/highscore/test_whitelist.rb
 - test/highscore/test_wordlist.rb
+- test/highscore/test_string.rb
+- test/highscore/test_content.rb
+- test/highscore/test_blacklist.rb
+- test/highscore/test_keywords.rb
+- test/fixtures/blacklist.txt
 - test/test_highscore.rb
 - README.md
 - History.txt
@@ -68,6 +89,7 @@ files:
 - version.txt
 homepage: http://domnikl.github.com/highscore
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options:
 - --main
@@ -75,31 +97,30 @@ rdoc_options:
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: highscore
-rubygems_version: 1.8.16
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 3
 summary: Easily find and rank keywords in long texts.
 test_files:
-- test/fixtures/blacklist.txt
-- test/highscore/test_blacklist.rb
-- test/highscore/test_content.rb
-- test/highscore/test_keyword.rb
-- test/highscore/test_keywords.rb
+- test/highscore/test_bonuslist.rb
 - test/highscore/test_multiple_blacklists.rb
-- test/highscore/test_string.rb
+- test/highscore/test_keyword.rb
 - test/highscore/test_whitelist.rb
 - test/highscore/test_wordlist.rb
+- test/highscore/test_string.rb
+- test/highscore/test_content.rb
+- test/highscore/test_blacklist.rb
+- test/highscore/test_keywords.rb
+- test/fixtures/blacklist.txt
 - test/test_highscore.rb