highscore 0.3.1 → 0.3.2

data/History.txt

@@ -1,3 +1,7 @@
+== 0.3.2 / 2012-01-27
+
+* use custom lists of ignored words (blacklist), fallback to a default list
+
 == 0.3.1 / 2012-01-21
 
 * refactored and threw away all that array stuff that nobody wants

data/README.md

@@ -29,6 +29,27 @@ Examples
     end
 
 
+Using a custom blacklist to ignore keywords
+-------------------------------------------
+
+    # setting single words
+    blacklist = Highscore::Blacklist.new
+    blacklist << 'foo'
+
+    # load a string/array
+    blacklist = Highscore::Blacklist.load "a string"
+    blacklist = Highscore::Blacklist.load %w{an array}
+
+    # loading from a file (separated by whitespace)
+    blacklist = Highscore::Blacklist.load_file "blacklist.txt"
+
+    # loading the default blacklist (falls back automatically if not explicit given)
+    blacklist = Highscore::Blacklist.load_default_file
+
+    # inject the blacklist into the content class
+    content = Highscore::Content.new "a string", blacklist
+
+
 Requirements
 ------------
 

data/lib/blacklist.txt

@@ -0,0 +1,3 @@
+the and that post add not see about using some something under our comments comment run
+you want for will file are with end new this use all but can your just get very data blog
+format out first they posts second

data/lib/highscore/blacklist.rb

@@ -0,0 +1,72 @@
+module Highscore
+
+  # blacklisted words to be ignored in the resulting keywords
+  #
+  class Blacklist
+    include Enumerable
+
+    # load a file of keywords
+    def self.load_file file_path
+      words = File.read(file_path).split(' ')
+      self.load(words)
+    end
+
+    # load default file
+    #
+    def self.load_default_file
+      file_path = File.join(File.dirname(__FILE__), %w{.. blacklist.txt})
+      self.load_file(file_path)
+    end
+
+    # load a file or array of words
+    #
+    def self.load(data)
+      if data.instance_of?(String)
+        words = data.split(' ')
+      elsif data.instance_of? Array
+        words = data
+      else
+        raise ArgumentError, "don't know how to handle a %s class" % data.class
+      end
+
+      words.map! {|x| x.gsub(/[\!\.\:\,\;\-\+]/, '') }
+
+      self.new(words)
+    end
+
+    attr_reader :words
+
+    def initialize(words = [])
+      @words = words
+    end
+
+    # iterate over words
+    #
+    def each
+      @words.each {|x| yield x }
+    end
+
+    # count of ignored words
+    def length
+      @words.length
+    end
+
+    # get an array of blacklisted words
+    #
+    def to_a
+      @words.to_a
+    end
+
+    # does the blacklist contain this keyword?
+    #
+    def include? keyword
+      @words.include? keyword
+    end
+
+    # add a new word to the blacklist
+    def <<(word)
+      @words << word
+    end
+
+  end
+end

data/lib/highscore/content.rb

@@ -5,9 +5,15 @@ module Highscore
   class Content
     attr_reader :content
 
-    def initialize content
+    def initialize(content, blacklist = nil)
       @content = content
 
+      unless blacklist
+        blacklist = Highscore::Blacklist.load_default_file
+      end
+
+      @blacklist = blacklist
+
       @emphasis = {
         :multiplier => 1.0,
         :upper_case => 3.0,
@@ -36,7 +42,7 @@ module Highscore
     def keywords
       keywords = Keywords.new
 
-      Keywords.find_keywords(@content).each do |text|
+      Keywords.find_keywords(@content, @blacklist).each do |text|
         weight = @emphasis[:multiplier]
 
         if text.length >= @emphasis[:long_words_threshold]

data/lib/highscore/keywords.rb

@@ -9,14 +9,14 @@ module Highscore
     include Enumerable
 
     # find keywords in a piece of content
-    def self.find_keywords content
+    def self.find_keywords content, blacklist
       keywords = content.scan(/\w+/)
       keywords.delete_if do |x|
         x.match(/^[\d]+(\.[\d]+){0,1}$/) or x.length <= 2
       end
 
       keywords.delete_if do |key, value|
-        %w{the and that post add not see about using some something under our comments comment run you want for will file are with end new this use all but can your just get very data blog format out first they posts second}.include? key.downcase
+        blacklist.include?(key.downcase)
       end
 
       keywords.sort

data/test/fixtures/blacklist.txt

@@ -0,0 +1,2 @@
+this is the test
+blacklist   file

data/test/highscore/test_blacklist.rb

@@ -0,0 +1,63 @@
+$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
+require "blacklist"
+require "test/unit"
+
+class TestBlacklist < Test::Unit::TestCase
+  def test_load_file
+    file_path = File.join(File.dirname(__FILE__), %w{.. fixtures blacklist.txt})
+    blacklist = Highscore::Blacklist.load_file(file_path)
+
+    assert_equal 6, blacklist.length
+  end
+
+  def test_load_default_file
+    blacklist = Highscore::Blacklist.load_default_file
+    assert_equal 42, blacklist.length
+  end
+
+  def test_load_file_fail
+    assert_raises(Errno::ENOENT) do
+      Highscore::Blacklist.load_file('foobar')
+    end
+  end
+
+  def test_empty_blacklist
+    blacklist = Highscore::Blacklist.new
+    assert_equal 0, blacklist.length
+  end
+
+  def test_add_new_word
+    blacklist = Highscore::Blacklist.new
+    blacklist << 'foo'
+
+    assert_equal ['foo'], blacklist.words
+  end
+
+  def test_load_string_and_remove_special_chars
+    blacklist = Highscore::Blacklist.load "this is an awesome string!"
+    assert_equal 5, blacklist.length
+
+    assert_equal ['this', 'is', 'an', 'awesome', 'string'], blacklist.to_a
+  end
+
+  def test_load_array
+    words = ['foo', 'bar', 'baz']
+
+    blacklist = Highscore::Blacklist.load(words)
+
+    assert_equal words, blacklist.words
+  end
+
+  def test_load_unknown_type
+    assert_raises ArgumentError do
+      Highscore::Blacklist.load(1)
+    end
+  end
+
+  def test_include?
+    blacklist = Highscore::Blacklist.load "foobar baz"
+
+    assert blacklist.include?("foobar")
+    assert !blacklist.include?("bla")
+  end
+end

data/version.txt

@@ -1 +1 @@
-0.3.1
+0.3.2

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: highscore
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-21 00:00:00.000000000 Z
+date: 2012-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bones
-  requirement: &70191947101320 !ruby/object:Gem::Requirement
+  requirement: &70200855600060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 3.7.3
   type: :development
   prerelease: false
-  version_requirements: *70191947101320
+  version_requirements: *70200855600060
 description: Rank keywords in long texts.
 email: liebler.dominik@googlemail.com
 executables:
@@ -30,6 +30,8 @@ extensions: []
 extra_rdoc_files:
 - History.txt
 - bin/highscore
+- lib/blacklist.txt
+- test/fixtures/blacklist.txt
 files:
 - .gitignore
 - History.txt
@@ -37,10 +39,14 @@ files:
 - Rakefile
 - bin/highscore
 - highscore.gemspec
+- lib/blacklist.txt
 - lib/highscore.rb
+- lib/highscore/blacklist.rb
 - lib/highscore/content.rb
 - lib/highscore/keyword.rb
 - lib/highscore/keywords.rb
+- test/fixtures/blacklist.txt
+- test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb
@@ -73,6 +79,7 @@ signing_key:
 specification_version: 3
 summary: Rank keywords in long texts.
 test_files:
+- test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb