highscore 0.4.3 → 0.5.0

data/History.txt

@@ -1,3 +1,10 @@
+== 0.5.0 / 2012-02-20
+
+* now supports a whitelist approach
+* <= 2 char words are ignored by default, but is now configurable
+* refactored a huge part of the API
+* added some more inline-documentation
+
 == 0.4.3 / 2012-02-11
 
 * rate words based on the amount of vowels and consonants

data/README.md

@@ -10,6 +10,7 @@ Features
 * rate based on amount (%) of vowels and consonants in a string
 * directly get keywords from String objects
 * blacklist words via a plain text file, String or an Array of words
+* optionally, configure a whitelist and only words from that list will get ranked
 * merge together Keywords from multiple sources
 * contains a CLI tool that operates on STDIN/OUT and is configurable via parameters
 
@@ -79,6 +80,15 @@ blacklist = Highscore::Blacklist.load_default_file
 content = Highscore::Content.new "a string", blacklist
 ```
 
+Using a whitelist instead of ranking all words
+----------------------------------------------
+
+```ruby
+# construct and inject it just like a blacklist
+whitelist = Highscore::Whitelist.load %w{these are valid keywords}
+content = Highscore::Content.new "invalid words", whitelist
+```
+
 Install
 -------
 

data/bin/highscore

@@ -18,7 +18,12 @@ optparse = OptionParser.new do |opts|
 
   # blacklist file
   opts.on('-b', '--blacklist FILEPATH', 'specify a blacklist file that should be used instead of the default') do |filepath|
-    options[:blacklist] = Highscore::Blacklist.load_file(filepath)
+    options[:wordlist] = Highscore::Blacklist.load_file(filepath)
+  end
+
+  # whitelist file
+  opts.on('-w', '--whitelist FILEPATH', 'specify a whitelist file') do |filepath|
+    options[:wordlist] = Highscore::Whitelist.load_file(filepath)
   end
 
   # general multiplier
@@ -26,6 +31,11 @@ optparse = OptionParser.new do |opts|
     options[:emphasis][:multiplier] = multiplier.to_f
   end
 
+  # don't print rank weight
+  opts.on('--no-ignore-short', 'don\'t ignore short words (<= 2 chars)') do
+    options[:emphasis][:ignore_short_words] = false
+  end
+
   # don't print rank weight
   opts.on('-s', '--short', 'don\'t print rank weight') do
     options[:short] = true
@@ -64,7 +74,7 @@ end
 optparse.parse!
 
 text = STDIN.read.to_s
-keywords = text.keywords(options[:blacklist]) do |content|
+keywords = text.keywords(options[:wordlist]) do |content|
   options[:emphasis].each do |key,value|
     content.set key, value
   end

data/lib/highscore/blacklist.rb

@@ -1,72 +1,18 @@
+$:.unshift(File.dirname(__FILE__))
+require "wordlist"
+
 module Highscore
 
   # blacklisted words to be ignored in the resulting keywords
   #
-  class Blacklist
-    include Enumerable
-
-    # load a file of keywords
-    def self.load_file file_path
-      words = File.read(file_path).split(' ')
-      self.load(words)
-    end
+  class Blacklist < Wordlist
 
     # load default file
     #
+    # @return Highscore::Blacklist
     def self.load_default_file
       file_path = File.join(File.dirname(__FILE__), %w{.. blacklist.txt})
       self.load_file(file_path)
     end
-
-    # load a file or array of words
-    #
-    def self.load(data)
-      if data.instance_of?(String)
-        words = data.split(' ')
-      elsif data.instance_of? Array
-        words = data
-      else
-        raise ArgumentError, "don't know how to handle a %s class" % data.class
-      end
-
-      words.map! {|x| x.gsub(/[\!\.\:\,\;\-\+]/, '') }
-
-      self.new(words)
-    end
-
-    attr_reader :words
-
-    def initialize(words = [])
-      @words = words
-    end
-
-    # iterate over words
-    #
-    def each
-      @words.each {|x| yield x }
-    end
-
-    # count of ignored words
-    def length
-      @words.length
-    end
-
-    # get an array of blacklisted words
-    #
-    def to_a
-      @words.to_a
-    end
-
-    # does the blacklist contain this keyword?
-    #
-    def include? keyword
-      @words.include? keyword
-    end
-
-    # add a new word to the blacklist
-    def <<(word)
-      @words << word
-    end
-
   end
 end

data/lib/highscore/content.rb

@@ -5,57 +5,89 @@ module Highscore
   class Content
     attr_reader :content
 
-    def initialize(content, blacklist = nil)
+    # @param content String
+    # @param wordlist Highscore::Wordlist
+    def initialize(content, wordlist = nil)
       @content = content
-
-      unless blacklist
-        blacklist = Highscore::Blacklist.load_default_file
+      @whitelist = @blacklist = nil
+
+      if wordlist.nil?
+        @blacklist = Highscore::Blacklist.load_default_file
+      elsif wordlist.kind_of? Highscore::Blacklist
+        @blacklist = wordlist
+      else
+        @whitelist = wordlist
       end
 
-      @blacklist = blacklist
-
       @emphasis = {
         :multiplier => 1.0,
         :upper_case => 3.0,
         :long_words => 2.0,
         :long_words_threshold => 15,
         :vowels => 0,
-        :consonants => 0
+        :consonants => 0,
+        :ignore_short_words => true
       }
     end
 
     # configure ranking
     #
+    # @param block
     def configure(&block)
       instance_eval(&block)
     end
 
     # set emphasis options to rank the content
     #
+    # @param key Symbol
+    # @param value Object
     def set(key, value)
-      @emphasis[key.to_sym] = value.to_f
+      @emphasis[key.to_sym] = value
     end
 
     # get the ranked keywords
     #
-    # :call-seq:
-    #   keywords -> Keywords
-    #
+    # @return Highscore::Keywords
     def keywords
       keywords = Keywords.new
 
-      Keywords.find_keywords(@content, @blacklist).each do |text|
+      Keywords.find_keywords(@content, wordlist).each do |text|
         text = text.to_s
-        keywords << Highscore::Keyword.new(text, weight(text))
+
+        if not (text.match(/^[\d]+(\.[\d]+){0,1}$/) or text.length <= 2)
+          keywords << Highscore::Keyword.new(text, weight(text))
+        elsif allow_short_words
+          keywords << Highscore::Keyword.new(text, weight(text))
+        end
       end
 
       keywords
     end
 
+    # get the used wordlist
+    #
+    # @return Highscore::Wordlist
+    def wordlist
+      unless @whitelist.nil?
+        @whitelist
+      else
+        @blacklist
+      end
+    end
+
     private
 
+    # allow short words to be rated
+    #
+    # @return TrueClass FalseClass
+    def allow_short_words
+      not @emphasis[:ignore_short_words]
+    end
+
     # weight a single text keyword
     #
+    # @param text String
+    # @return Float
     def weight(text)
       weight = @emphasis[:multiplier]
 
@@ -72,11 +104,19 @@ module Highscore
       weight
     end
 
+    # weight the vowels on a text
+    #
+    # @param text String
+    # @return Float
     def vowels(text)
       percent = text.vowels.length / text.length.to_f
       percent * @emphasis[:vowels]
     end
 
+    # weight the consonants on a text
+    #
+    # @param text String
+    # @return Float
     def consonants(text)
       percent = text.consonants.length / text.length.to_f
       percent * @emphasis[:consonants]

data/lib/highscore/keyword.rb

@@ -6,17 +6,24 @@ module Highscore
     attr_accessor :weight, :text
 
     # init a keyword
+    #
+    # @param text String
+    # @param weight Float
     def initialize(text, weight)
       @text = text
       @weight = weight.to_f
     end
 
     # sort keywords
+    #
+    # @param other Highscore::Keyword
     def <=>(other)
       other.weight <=> @weight
     end
 
     # get the string
+    #
+    # @return String
     def to_s
       @text
     end

data/lib/highscore/keywords.rb

@@ -9,14 +9,19 @@ module Highscore
     include Enumerable
 
     # find keywords in a piece of content
-    def self.find_keywords content, blacklist
+    #
+    # @param content String
+    # @param wordlist Highscore::Wordlist
+    # @return Highscore::Keywords
+    def self.find_keywords content, wordlist
       keywords = content.to_s.scan(/\w+/)
-      keywords.delete_if do |x|
-        x.match(/^[\d]+(\.[\d]+){0,1}$/) or x.length <= 2
-      end
 
       keywords.delete_if do |key, value|
-        blacklist.include?(key.downcase)
+        if wordlist.kind_of? Highscore::Blacklist
+          wordlist.include?(key.downcase)
+        elsif wordlist.kind_of? Highscore::Whitelist
+          not wordlist.include?(key.downcase)
+        end
       end
 
       keywords.sort
@@ -31,21 +36,23 @@ module Highscore
     # ranks the keywords and removes keywords that have a low ranking
     # or are blacklisted
     #
-    # :call-seq:
-    #   rank -> array
-    #
+    # @return Array
     def rank
       sort
     end
 
     # get the top n keywords
     #
+    # @param n Fixnum
+    # @return Array
     def top n = 10
       rank[0..(n - 1)]
     end
 
     # add new keywords
     #
+    # @param keyword String
+    # @return Highscore::Keywords
     def <<(keyword)
       key = Digest::SHA1.hexdigest(keyword.text)
 
@@ -54,9 +61,13 @@ module Highscore
       else
         @keywords[key] = keyword
       end
+
+      @keywords
     end
 
     # sort
+    #
+    # @return Array
     def sort
       sorted = @keywords.sort {|a,b| a[1] <=> b[1] }
 
@@ -66,26 +77,34 @@ module Highscore
 
     # Enumerable
     #
-    def each &block
+    def each
       @keywords.each {|keyword| yield keyword[1] }
     end
 
     # number of Keywords given
+    #
+    # @return Fixnum
     def length
       @keywords.length
     end
 
     # get the keyword with the highest rank
+    #
+    # @return Highscore::Keyword
     def first
       sort.first
     end
 
     # get the keyword with the lowest rank
+    #
+    # @return Highscore::Keyword
     def last
       sort.reverse.first
     end
 
     # merge in another keyword list, operates on self
+    #
+    # @return Highscore::Keywords
     def merge!(other)
       other.each do |keyword|
         self << keyword

data/lib/highscore/string.rb

@@ -1,11 +1,14 @@
-# monkey patch to define a keywords method on arbitrary strings
+# monkey patch to call custom methods on arbitrary strings
 #
 class String
 
   # get keywords from the string
   #
-  def keywords(blacklist = nil, &block)
-    content = Highscore::Content.new(self, blacklist)
+  # @param wordlist Highscore::Wordlist
+  # @param block Block
+  # @return Highscore::Keywords
+  def keywords(wordlist = nil, &block)
+    content = Highscore::Content.new(self, wordlist)
 
     if block_given?
       content.configure do
@@ -17,11 +20,15 @@ class String
   end
 
   # get all vowels from a string
+  #
+  # @return String
   def vowels
     gsub(/[^aeiou]/, '')
   end
 
   # get all consonants from a string
+  #
+  # @return String
   def consonants
     gsub(/[aeiou]/, '')
   end

data/lib/highscore/whitelist.rb

@@ -0,0 +1,10 @@
+$:.unshift(File.dirname(__FILE__))
+require "wordlist"
+
+module Highscore
+  # whitelisted words
+  #
+  class Whitelist < Wordlist
+
+  end
+end

data/lib/highscore/wordlist.rb

@@ -0,0 +1,76 @@
+module Highscore
+
+  # a basic list of words
+  class Wordlist
+    include Enumerable
+
+    # load a file of keywords
+    #
+    # @param file_path String
+    # @return Highscore::Wordlist
+    def self.load_file file_path
+      words = File.read(file_path).split(' ')
+      self.load(words)
+    end
+
+    # load a file or array of words
+    #
+    # @param data String Array
+    # @return Highscore::Wordlist
+    def self.load(data)
+      if data.instance_of?(String)
+        words = data.split(' ')
+      elsif data.instance_of? Array
+        words = data
+      else
+        raise ArgumentError, "don't know how to handle a %s class" % data.class
+      end
+
+      words.map! {|x| x.gsub(/[\!\.\:\,\;\-\+]/, '') }
+
+      self.new(words)
+    end
+
+    attr_reader :words
+
+    # @param words Array
+    def initialize(words = [])
+      @words = words
+    end
+
+    # iterate over words
+    #
+    def each
+      @words.each {|word| yield word }
+    end
+
+    # count of ignored words
+    #
+    # @return Fixnum
+    def length
+      @words.length
+    end
+
+    # get an array of blacklisted words
+    #
+    # @return Array
+    def to_a
+      @words.to_a
+    end
+
+    # does the blacklist contain this keyword?
+    #
+    # @param keyword String
+    # @return true/false
+    def include? keyword
+      @words.include? keyword
+    end
+
+    # add a new word to the blacklist
+    #
+    # @param word String
+    def <<(word)
+      @words << word
+    end
+  end
+end

data/test/highscore/test_blacklist.rb

@@ -3,11 +3,9 @@ require "blacklist"
 require "test/unit"
 
 class TestBlacklist < Test::Unit::TestCase
-  def test_load_file
-    file_path = File.join(File.dirname(__FILE__), %w{.. fixtures blacklist.txt})
-    blacklist = Highscore::Blacklist.load_file(file_path)
-
-    assert_equal 6, blacklist.length
+  def test_is_a_wordlist
+    blacklist = Highscore::Blacklist.new
+    assert blacklist.kind_of? Highscore::Wordlist
   end
 
   def test_load_default_file
@@ -15,62 +13,16 @@ class TestBlacklist < Test::Unit::TestCase
     assert_equal 42, blacklist.length
   end
 
-  def test_load_file_fail
-    assert_raises(Errno::ENOENT) do
-      Highscore::Blacklist.load_file('foobar')
-    end
-  end
-
-  def test_empty_blacklist
-    blacklist = Highscore::Blacklist.new
-    assert_equal 0, blacklist.length
-  end
-
-  def test_add_new_word
-    blacklist = Highscore::Blacklist.new
-    blacklist << 'foo'
-
-    assert_equal ['foo'], blacklist.words
-  end
-
-  def test_load_string_and_remove_special_chars
-    blacklist = Highscore::Blacklist.load "this is an awesome string!"
-    assert_equal 5, blacklist.length
-
-    assert_equal ['this', 'is', 'an', 'awesome', 'string'], blacklist.to_a
-  end
-
-  def test_load_array
-    words = ['foo', 'bar', 'baz']
-
-    blacklist = Highscore::Blacklist.load(words)
-
-    assert_equal words, blacklist.words
-  end
-
-  def test_load_unknown_type
-    assert_raises ArgumentError do
-      Highscore::Blacklist.load(1)
-    end
-  end
-
-  def test_include?
-    blacklist = Highscore::Blacklist.load "foobar baz"
-
-    assert blacklist.include?("foobar")
-    assert !blacklist.include?("bla")
-  end
-
   def test_blacklisting_content
-    keywords = "Foo bar is not bar baz".keywords(Highscore::Blacklist.load(['baz']))
+      keywords = "Foo bar is not bar baz".keywords(Highscore::Blacklist.load(%w(baz)))
 
-    keyword_list = []
-    keywords.rank.each do |k|
-      keyword_list << k.text
-    end
+      keyword_list = []
+      keywords.rank.each do |k|
+        keyword_list << k.text
+      end
 
-    expected_keywords = ['Foo', 'bar', 'not']
+      expected_keywords = %w(Foo bar not)
 
-    assert_equal expected_keywords, keyword_list
+      assert_equal expected_keywords, keyword_list
   end
 end

data/test/highscore/test_content.rb

@@ -42,4 +42,12 @@ class TestContent < Test::Unit::TestCase
     assert_equal 3.75, keywords.first.weight
     assert_equal 3.5, keywords.last.weight
   end
+
+  def test_rank_short_words
+    keywords = 'be as is foobar'.keywords do
+      set :ignore_short_words, false
+    end
+
+    assert_equal 4, keywords.length
+  end
 end

data/test/highscore/test_whitelist.rb

@@ -0,0 +1,17 @@
+$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
+require "whitelist"
+require "test/unit"
+
+class TestBlacklist < Test::Unit::TestCase
+  def test_is_wordlist
+    whitelist = Highscore::Whitelist.new
+    assert whitelist.kind_of? Highscore::Wordlist
+  end
+
+  def test_whitelist_content
+    whitelist = Highscore::Whitelist.load %w{foo bar}
+
+    content = Highscore::Content.new "foo baz bar", whitelist
+    assert_equal 2, content.keywords.length
+  end
+end

data/test/highscore/test_wordlist.rb

@@ -0,0 +1,59 @@
+$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
+require "wordlist"
+require "test/unit"
+
+class TestBlacklist < Test::Unit::TestCase
+
+  def test_load_file
+    file_path = File.join(File.dirname(__FILE__), %w{.. fixtures blacklist.txt})
+    blacklist = Highscore::Wordlist.load_file(file_path)
+
+    assert_equal 6, blacklist.length
+  end
+
+  def test_load_file_fail
+    assert_raises(Errno::ENOENT) do
+      Highscore::Blacklist.load_file('foobar')
+    end
+  end
+
+  def test_empty_blacklist
+    blacklist = Highscore::Wordlist.new
+    assert_equal 0, blacklist.length
+  end
+
+  def test_add_new_word
+    blacklist = Highscore::Wordlist.new
+    blacklist << 'foo'
+
+    assert_equal %w(foo), blacklist.words
+  end
+
+  def test_load_string_and_remove_special_chars
+    blacklist = Highscore::Wordlist.load "this is an awesome string!"
+    assert_equal 5, blacklist.length
+
+    assert_equal %w{this is an awesome string}, blacklist.to_a
+  end
+
+  def test_load_array
+    words = %w{foo bar baz}
+
+    blacklist = Highscore::Wordlist.load(words)
+
+    assert_equal words, blacklist.words
+  end
+
+  def test_load_unknown_type
+    assert_raises ArgumentError do
+      Highscore::Wordlist.load(1)
+    end
+  end
+
+  def test_include?
+    blacklist = Highscore::Wordlist.load "foobar baz"
+
+    assert blacklist.include?("foobar")
+    assert !blacklist.include?("bla")
+  end
+end

data/version.txt

@@ -1 +1 @@
-0.4.3
+0.5.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: highscore
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-11 00:00:00.000000000 Z
+date: 2012-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bones
-  requirement: &70111402827500 !ruby/object:Gem::Requirement
+  requirement: &70127917245680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 3.7.3
   type: :development
   prerelease: false
-  version_requirements: *70111402827500
+  version_requirements: *70127917245680
 description: Find and rank keywords in long texts.
 email: liebler.dominik@googlemail.com
 executables:
@@ -46,12 +46,16 @@ files:
 - lib/highscore/keyword.rb
 - lib/highscore/keywords.rb
 - lib/highscore/string.rb
+- lib/highscore/whitelist.rb
+- lib/highscore/wordlist.rb
 - test/fixtures/blacklist.txt
 - test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb
 - test/highscore/test_string.rb
+- test/highscore/test_whitelist.rb
+- test/highscore/test_wordlist.rb
 - test/test_highscore.rb
 - version.txt
 homepage: http://thewebdev.de
@@ -76,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: highscore
-rubygems_version: 1.8.11
+rubygems_version: 1.8.16
 signing_key: 
 specification_version: 3
 summary: Find and rank keywords in long texts.
@@ -86,4 +90,6 @@ test_files:
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb
 - test/highscore/test_string.rb
+- test/highscore/test_whitelist.rb
+- test/highscore/test_wordlist.rb
 - test/test_highscore.rb