highscore 0.5.3 → 1.0.0

data/History.txt

@@ -1,3 +1,9 @@
+== 1.0.0 / 2013-02-02
+
+* added per-language support for black- and whitelists (thanks to bobjflong)
+* added support for language recognition
+* fixed tokenization for UTF-8 strings (this is broken in Ruby 1.8.x!) (thanks to pdg)
+
 == 0.5.2 / 2012-02-25
 
 * added (optional) support for word stemming using the fast-stemmer gem

data/README.md

@@ -1,24 +1,21 @@
-highscore
-===========
+# highscore
 
 Easily find and rank keywords in long texts.
 
-[![Build Status](https://secure.travis-ci.org/domnikl/highscore.png?branch=develop)](http://travis-ci.org/domnikl/highscore)
+[![Build Status](https://secure.travis-ci.org/domnikl/highscore.png?branch=develop)](http://travis-ci.org/domnikl/highscore) [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/domnikl/highscore)
 
-Features
---------
+## Features
 
 * configurable to rank different types of words different (uppercase, long words, etc.)
 * rate based on amount (%) of vowels and consonants in a string
 * directly get keywords from String objects
-* blacklist words via a plain text file, String or an Array of words
+* blacklist words via a plain text file, String or an Array of words (per language if needed)
 * optionally, configure a whitelist and only words from that list will get ranked
-* use word stemming (requires the fast-stemmer gem, doesn't work on JRuby platforms!)
+* use word stemming (requires the `fast-stemmer` gem, or the `stemmer` gem)
 * merge together Keywords from multiple sources
 * contains a CLI tool that operates on STDIN/OUT and is configurable via parameters
 
-Examples
---------
+## Examples
 
 ```ruby
 text = Highscore::Content.new "foo bar"
@@ -64,8 +61,7 @@ end
 
 Have a look at bin/highscore, you can run highscore on your CLI and feed it with text on STDIN.
 
-Using a custom blacklist to ignore keywords
--------------------------------------------
+## Using a custom blacklist to ignore keywords
 
 ```ruby
 # setting single words
@@ -86,8 +82,7 @@ blacklist = Highscore::Blacklist.load_default_file
 content = Highscore::Content.new "a string", blacklist
 ```
 
-Using a whitelist instead of ranking all words
-----------------------------------------------
+## Using a whitelist instead of ranking all words
 
 ```ruby
 # construct and inject it just like a blacklist
@@ -95,22 +90,38 @@ whitelist = Highscore::Whitelist.load %w{these are valid keywords}
 content = Highscore::Content.new "invalid words", whitelist
 ```
 
-Install
--------
+## Multiple languages
+
+```ruby
+# Load a default blacklist
+blacklist_default = Highscore::Blacklist.load "mister"
+text = Highscore::Content.new "oui mister interesting", blacklist_default
+text.keywords.top(3).join " "
+
+# Prints "interesting oui"
+
+# Load a rudimentary blacklist for French
+blacklist_francais = Highscore::Blacklist.load "oui"
+text.add_wordlist blacklist_francais, "fr"
+text.keywords(:lang => :fr).top(3).join " "
+
+# Prints "interesting mister"
+```
+
+## Install
 
 * `[sudo] gem install highscore`
 
-To use word stemming, you need to have the fast-stemmer gem installed:
+To use word stemming, you need to have the `fast-stemmer` (C extension) or `stemmer` gem installed:
 
 * `[sudo] gem install fast-stemmer`
+* `[sudo] gem install stemmer`
 
-Author
-------
+## Author
 
 Original author: Dominik Liebler <liebler.dominik@googlemail.com>
 
-License
--------
+## License
 
 (The MIT License)
 

data/Rakefile

@@ -1,18 +1,34 @@
 require 'rubygems'
+require 'rake/testtask'
 
-begin
-  require 'bones'
-rescue LoadError
-  abort '### Please install the "bones" gem ###'
-end
+task :default => 'test'
+
+$LOAD_PATH.unshift File.expand_path("../lib", __FILE__)
+require "highscore"
+
+gem_name = "highscore-#{Highscore::VERSION}.gem"
 
-task :default => 'test:run'
-task 'gem:release' => 'test:run'
+namespace :gem do
+    task :clean do
+      system "rm -f *.gem"
+    end
+  
+    task :build => [:clean, :test] do
+      system "gem build highscore.gemspec"
+    end
+
+    task :install => :build do
+      system "gem install #{gem_name}"
+    end
+
+    task :release => :build do
+      system "gem push #{gem_name}"
+    end
+end
 
-Bones {
-  name     'highscore'
-  authors  'Dominik Liebler'
-  email    'liebler.dominik@googlemail.com'
-  url      'http://thewebdev.de'
-  ignore_file '.gitignore'
-}
+Rake::TestTask.new do |t|
+  t.libs = ["lib"]
+  t.warning = true
+  t.verbose = true
+  t.test_files = FileList['test/**/test_*.rb']
+end

data/bin/highscore

@@ -76,6 +76,11 @@ optparse = OptionParser.new do |opts|
   opts.on('-s', '--short', 'don\'t print rank weight') do
     options[:short] = true
   end
+  
+  opts.on('-v', '--version', 'show version') do
+    puts Highscore::VERSION
+    exit
+  end
 end
 
 optparse.parse!

data/lib/highscore.rb

@@ -1,60 +1,28 @@
+require 'rubygems' if RUBY_VERSION !~ /^1\.9/
 
 module Highscore
-
   # :stopdoc:
   LIBPATH = ::File.expand_path('..', __FILE__) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   VERSION = ::File.read(PATH + 'version.txt').strip
   # :startdoc:
 
-  # Returns the library path for the module. If any arguments are given,
-  # they will be joined to the end of the libray path using
-  # <tt>File.join</tt>.
-  #
-  def self.libpath( *args )
-    rv =  args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
-    if block_given?
-      begin
-        $LOAD_PATH.unshift LIBPATH
-        rv = yield
-      ensure
-        $LOAD_PATH.shift
-      end
-    end
-    return rv
-  end
-
-  # Returns the lpath for the module. If any arguments are given,
-  # they will be joined to the end of the path using
-  # <tt>File.join</tt>.
-  #
-  def self.path( *args )
-    rv = args.empty? ? PATH : ::File.join(PATH, args.flatten)
-    if block_given?
-      begin
-        $LOAD_PATH.unshift PATH
-        rv = yield
-      ensure
-        $LOAD_PATH.shift
-      end
+  def self.load_modules
+    modules = %w(
+      blacklist
+      content
+      keyword
+      keywords
+      string
+      whitelist
+      wordlist
+    )
+    
+    modules.each do |m|
+      require ::File.expand_path('highscore/' + m, LIBPATH)
     end
-    return rv
   end
+end
 
-  # Utility method used to require all files ending in .rb that lie in the
-  # directory below this file that has the same name as the filename passed
-  # in. Optionally, a specific _directory_ name can be passed in such that
-  # the _filename_ does not have to be equivalent to the directory.
-  #
-  def self.require_all_libs_relative_to( fname, dir = nil )
-    dir ||= ::File.basename(fname, '.*')
-    search_me = ::File.expand_path(
-        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
-
-    Dir.glob(search_me).sort.each {|rb| require rb}
-  end
-
-end  # module Highscore
-
-Highscore.require_all_libs_relative_to(__FILE__)
+Highscore.load_modules
 

data/lib/highscore/content.rb

@@ -7,12 +7,14 @@ require 'rubygems'
 module Highscore
   class Content
     attr_reader :content
+    attr_accessor :language_wordlists
 
     # @param content String
     # @param wordlist Highscore::Wordlist
     def initialize(content, wordlist = nil)
       @content = content
       @whitelist = @blacklist = nil
+      @language_wordlists = {}
 
       if wordlist.nil?
         @blacklist = Highscore::Blacklist.load_default_file
@@ -31,9 +33,13 @@ module Highscore
         :consonants => 0,
         :ignore_short_words => true,
         :ignore_case => false,
-        :word_pattern => /\w+/,
+        :word_pattern => /\p{Word}+/u,
         :stemming => false
       }
+      
+      if RUBY_VERSION =~ /^1\.8/
+        @emphasis[:word_pattern] = /\w+/
+      end
     end
 
     # configure ranking
@@ -50,16 +56,32 @@ module Highscore
     def set(key, value)
       @emphasis[key.to_sym] = value
     end
+    
+    # add another wordlist, given a language
+    #
+    # @param language_wordlist Highscore::Wordlist
+    # @param language String
+    def add_wordlist(language_wordlist, language)
+      raise ArgumentError, "Not a valid Wordlist" unless language_wordlist.kind_of? Highscore::Wordlist
+      language_wordlists[language.to_sym] = language_wordlist
+    end
 
     # get the ranked keywords
     #
     # @return Highscore::Keywords
-    def keywords
+    def keywords(opts = {})
+      used_wordlist = nil
+      if opts[:lang]
+        used_wordlist = language_wordlists[opts[:lang].to_sym]
+      else
+        used_wordlist = wordlist
+      end
+        
       @emphasis[:stemming] = use_stemming?
 
       keywords = Keywords.new
 
-      Keywords.find_keywords(processed_content, wordlist, word_pattern).each do |text|
+      Keywords.find_keywords(processed_content, used_wordlist, word_pattern).each do |text|
         text = text.to_s
         text = text.stem if @emphasis[:stemming]
 
@@ -84,6 +106,14 @@ module Highscore
       end
     end
 
+    # guess the language of the content and return a symbol for it
+    # => done via whatlanguage gem
+    #
+    # @return Symbol
+    def language
+      @content.language
+    end
+
     private
 
     # processes the text content applying any necessary transformations
@@ -148,8 +178,6 @@ module Highscore
       percent * @emphasis[:consonants]
     end
 
-    private
-
     # using stemming is only possible, if fast-stemmer is installed
     # doesn't work for JRuby
     def use_stemming?
@@ -158,7 +186,12 @@ module Highscore
           require 'fast_stemmer'
           true
         rescue LoadError
-          false
+          begin
+            require 'stemmer'
+            true
+          rescue LoadError
+            false
+          end
         end
       else
         false

data/lib/highscore/string.rb

@@ -1,3 +1,5 @@
+require 'whatlanguage'
+
 # monkey patch to call custom methods on arbitrary strings
 #
 class String

data/test/highscore/test_blacklist.rb

@@ -1,8 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
-require "blacklist"
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestBlacklist < Test::Unit::TestCase
+class TestBlacklist < Highscore::TestCase
   def test_is_a_wordlist
     blacklist = Highscore::Blacklist.new
     assert blacklist.kind_of? Highscore::Wordlist

data/test/highscore/test_content.rb

@@ -1,9 +1,10 @@
+# encoding: utf-8
 $:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
 require "content"
 require "test/unit"
 require 'rubygems'
 
-class TestContent < Test::Unit::TestCase
+class TestContent < Highscore::TestCase
   def setup
     @text = "This is some text"
     @content = Highscore::Content.new(@text)
@@ -32,6 +33,19 @@ class TestContent < Test::Unit::TestCase
     assert_equal 1, content.keywords.length
   end
 
+  def test_keywords_utf8
+    content = 'Schöne Grüße, caractères, русский'
+
+    content = Highscore::Content.new content
+    
+    if RUBY_VERSION =~ /^1\.8/
+      # Ruby 1.8 doesn't support correct tokenization
+      assert_equal 3, content.keywords.length
+    else
+      assert_equal 4, content.keywords.length
+    end
+  end
+
   def test_vowels_and_consonants
     keywords = 'foobar RubyGems'.keywords do
       set :vowels, 2
@@ -77,9 +91,22 @@ class TestContent < Test::Unit::TestCase
   end
 
   def test_stemming
+    loaded_stemmer = false
+
     begin
       require 'fast_stemmer'
+      loaded_stemmer = true
+    rescue LoadError
+      begin
+        require 'stemmer'
+        loaded_stemmer = true
+      rescue LoadError
+        # skip this test, neither fast_stemmer nor stemmer is installed!
+        fail
+      end
+    end
 
+    if loaded_stemmer
       keywords = 'word words boards board woerter wort'.keywords do
         set :stemming, true
       end
@@ -87,10 +114,17 @@ class TestContent < Test::Unit::TestCase
       assert_equal 4, keywords.length
 
       keywords.each do |k|
-        assert (%w{board word woerter wort}).include?(k.text)
+        assert %w{board word woerter wort}.include?(k.text)
       end
-    rescue LoadError
-      # do nothing, just skip this test
     end
   end
-end
+  
+  def test_language_english
+    assert_equal :english, Highscore::Content.new("this is obviously an english text").language
+  end
+  
+  def test_language_german
+    assert_equal :german, Highscore::Content.new("Das ist sicherlich ein deutscher Text!").language
+  end
+end
+

data/test/highscore/test_keyword.rb

@@ -1,8 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
-require 'keyword'
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestKeyword < Test::Unit::TestCase
+class TestKeyword < Highscore::TestCase
   def setup
     @keyword = Highscore::Keyword.new('Ruby', 2)
   end

data/test/highscore/test_keywords.rb

@@ -1,9 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
-require 'keywords'
-require 'keyword'
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestKeywords < Test::Unit::TestCase
+class TestKeywords < Highscore::TestCase
   def setup
     @keywords = Highscore::Keywords.new
     @keywords << Highscore::Keyword.new('Ruby', 2)

data/test/highscore/test_multiple_blacklists.rb

@@ -0,0 +1,47 @@
+$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
+require "content"
+require "blacklist"
+require "whitelist"
+require "test/unit"
+require 'rubygems'
+
+class TestMultipleBlacklists < Test::Unit::TestCase
+  def setup
+    @nonsense = "Oui monsieur, je suis un programmer. You OK?"
+    
+    @blacklist_francais = Highscore::Blacklist.load %w{oui je un}
+    @blacklist_default = Highscore::Blacklist.load %w{ok you programmer}
+    @whitelist_simple  = Highscore::Whitelist.load %{programmer}
+    
+    @content = Highscore::Content.new @nonsense, @blacklist_default
+    @content.configure do
+      set :ignore_case, true
+    end
+    @content.add_wordlist @blacklist_francais, 'fr'
+    @content.add_wordlist @whitelist_simple, 'si'
+  end
+  
+  def sort_for_ruby18(keywords)
+    keywords.join(' ').split(' ').sort
+  end
+  
+  def test_different_language
+    assert_equal %w(monsieur programmer suis you), sort_for_ruby18(@content.keywords(:lang => :fr).top(4))
+  end
+  
+  def test_bad_wordlist
+    assert_raise ArgumentError do
+      @content.add_wordlist 3.14159, "pi"
+    end
+  end
+  
+  def test_normal_operation
+    assert_equal %w(monsieur oui suis), sort_for_ruby18(@content.keywords.top(3))
+  end
+  
+  def test_added_whitelist
+    assert_equal %w(programmer), sort_for_ruby18(@content.keywords(:lang => :si).top(3))
+  end
+end
+  
+  

data/test/highscore/test_string.rb

@@ -1,8 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib}))
-require 'highscore'
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestString < Test::Unit::TestCase
+class TestString < Highscore::TestCase
   def setup
     blacklist_file = File.join(File.dirname(__FILE__), %w{.. fixtures blacklist.txt})
     @blacklist = Highscore::Blacklist.load_file(blacklist_file)

data/test/highscore/test_whitelist.rb

@@ -1,8 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
-require "whitelist"
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestBlacklist < Test::Unit::TestCase
+class TestBlacklist < Highscore::TestCase
   def test_is_wordlist
     whitelist = Highscore::Whitelist.new
     assert whitelist.kind_of? Highscore::Wordlist

data/test/highscore/test_wordlist.rb

@@ -1,8 +1,6 @@
-$:.unshift(File.join(File.dirname(__FILE__), %w{.. .. lib highscore}))
-require "wordlist"
-require "test/unit"
+require File.dirname(__FILE__) + '/../test_highscore'
 
-class TestBlacklist < Test::Unit::TestCase
+class TestBlacklist < Highscore::TestCase
 
   def test_load_file
     file_path = File.join(File.dirname(__FILE__), %w{.. fixtures blacklist.txt})

data/test/test_highscore.rb

@@ -0,0 +1,19 @@
+$:.unshift(File.join(File.dirname(__FILE__), %w{.. lib highscore}))
+
+begin
+  require 'simplecov'
+  SimpleCov.start
+rescue LoadError
+  # ignore ...
+end
+
+require 'highscore'
+require 'test/unit'
+
+module Highscore
+  class TestCase < Test::Unit::TestCase
+    def test_version
+      assert_not_nil Highscore::VERSION
+    end
+  end
+end

data/version.txt

@@ -1 +1 @@
-0.5.3
+1.0.0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: highscore
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,38 +9,41 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-09 00:00:00.000000000 Z
+date: 2012-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bones
-  requirement: &70276433082620 !ruby/object:Gem::Requirement
+  name: simplecov
+  requirement: &70317557633680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 3.7.3
+        version: 0.6.4
   type: :development
   prerelease: false
-  version_requirements: *70276433082620
-description: Easily find and rank keywords in long texts.
+  version_requirements: *70317557633680
+- !ruby/object:Gem::Dependency
+  name: whatlanguage
+  requirement: &70317557633200 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70317557633200
+description: Find and rank keywords in text.
 email: liebler.dominik@googlemail.com
 executables:
 - highscore
 extensions: []
 extra_rdoc_files:
 - History.txt
-- bin/highscore
 - lib/blacklist.txt
-- test/fixtures/blacklist.txt
 files:
-- .gitignore
-- .travis.yml
-- History.txt
-- README.md
-- Rakefile
 - bin/highscore
 - lib/blacklist.txt
-- lib/highscore.rb
 - lib/highscore/blacklist.rb
 - lib/highscore/content.rb
 - lib/highscore/keyword.rb
@@ -48,17 +51,22 @@ files:
 - lib/highscore/string.rb
 - lib/highscore/whitelist.rb
 - lib/highscore/wordlist.rb
+- lib/highscore.rb
 - test/fixtures/blacklist.txt
 - test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb
+- test/highscore/test_multiple_blacklists.rb
 - test/highscore/test_string.rb
 - test/highscore/test_whitelist.rb
 - test/highscore/test_wordlist.rb
 - test/test_highscore.rb
+- README.md
+- History.txt
+- Rakefile
 - version.txt
-homepage: http://thewebdev.de
+homepage: https://domnikl.github.com/highscore
 licenses: []
 post_install_message: 
 rdoc_options:
@@ -85,10 +93,12 @@ signing_key:
 specification_version: 3
 summary: Easily find and rank keywords in long texts.
 test_files:
+- test/fixtures/blacklist.txt
 - test/highscore/test_blacklist.rb
 - test/highscore/test_content.rb
 - test/highscore/test_keyword.rb
 - test/highscore/test_keywords.rb
+- test/highscore/test_multiple_blacklists.rb
 - test/highscore/test_string.rb
 - test/highscore/test_whitelist.rb
 - test/highscore/test_wordlist.rb

data/.gitignore

@@ -1,12 +0,0 @@
-# general
-.DS_Store
-*.gemspec
-
-# RubyMine
-.idea
-
-# bones
-announcement.txt
-coverage
-doc
-pkg

data/.travis.yml

@@ -1,12 +0,0 @@
-language: ruby
-rvm:
-  - 1.8.7
-  - 1.9.2
-  - 1.9.3
-  - jruby-18mode # JRuby in 1.8 mode
-  - jruby-19mode # JRuby in 1.9 mode
-  - rbx-18mode
-  - rbx-19mode
-
-# fast-stemmer is a C extension, so it won't work with JRuby ...
-before_install: gem install bones; (gem install fast-stemmer &); sleep 10