keyword_prospector 0.8.0

data/History.txt

@@ -0,0 +1,4 @@
+== 0.8.0 2008-08-07
+
+* 1 major enhancement:
+  * Initial release

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Los Angeles Times
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,40 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+config/hoe.rb
+config/requirements.rb
+lib/hyperlink_strategy.rb
+lib/keyword_decorator.rb
+lib/keyword_linker.rb
+lib/keyword_prospector
+lib/keyword_prospector.rb
+lib/lookup_chain.rb
+lib/match.rb
+lib/profile.rb
+lib/search_and_replace.rb
+lib/state.rb
+script/console
+script/destroy
+script/generate
+script/txt2html
+setup.rb
+spec/hyperlink_strategy_spec.rb
+spec/keyword_linker_spec.rb
+spec/keyword_prospector_spec.rb
+spec/lookup_chain_spec.rb
+spec/match_spec.rb
+spec/search_and_replace_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/state_spec.rb
+tasks/deployment.rake
+tasks/environment.rake
+tasks/rspec.rake
+tasks/website.rake
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.html.erb

data/README.txt

@@ -0,0 +1,95 @@
+= keyword_prospector
+
+* http://github.com/latimes/keyword_prospector
+
+== DESCRIPTION:
+
+KeywordProspector is a gem for associating keywords in text with arbitrary
+output objects.  It uses an Aho-Corasick tree to provide matching that scales
+linearly with the length of the text, regardless of the number of keywords.
+
+== FEATURES:
+
+The core KeywordProspector engine has the following properties:
+* Once a tree is built, matching against the tree is o(n) where n is the length
+  of your text, regardless of how mane keywords you have in the tree.
+* Arbitrary output objects can be associated with any keyword or set of
+  keywords.
+
+KeywordLinker can be used to create links to designated url's:
+* You can specify a single keyword to associate with a url.
+* You can specify an array of keywords to associate with a url.
+* Each keyword or group of keywords will be linked only once to the url
+  provided.
+* Hyperlinks are not created within existing hyperlinks.
+* Hyperlinks are not generated anywhere they would be illegal in HTML, such as
+  within attribute values.
+
+== SYNOPSIS:
+
+Use KeywordLinker to create links in HTML text.  KeywordLinker will link only
+the first alternative that appears in text:
+
+  require 'keyword_linker'
+
+  linker = KeywordLinker.new
+  linker.add_url("http://www.latimes.com", ["L.A. Times", "Los Angeles Times"])
+  linker.link_text("'L.A. Times' or 'Los Angeles Times'?")
+  => "'<a href=\"http://www.latimes.com\">L.A. Times</a>' or 'Los Angeles Times'?"
+
+  linker.link_text("'Los Angeles Times' or 'L.A. Times'?")
+  => "'<a href=\"http://www.latimes.com\">Los Angeles Times</a>' or 'L.A. Times'?"
+
+
+You can provide html options when adding url's:
+
+  linker = KeywordLinker.new
+  linker.add_url("http://www.latimes.com", "Los Angeles Times",
+                  :title => "Visit the Los Angeles Times")
+  linker.add_url("http://www.google.com", ["Google", "The Google"],
+                  :title => "Go check it out on The Google!")
+
+  linker.link_text("Do you prefer The Google or the Los Angeles Times?")
+  => "Do you prefer <a href=\"http://www.google.com\" title=\"Go check it out on The Google!\">The Google</a> or the <a href=\"http://www.latimes.com\" title=\"Visit the Los Angeles Times\">Los Angeles Times</a>?"
+
+== DEPENDENCIES:
+
+* Hpricot
+
+== INSTALL:
+
+sudo gem install keyword_prospector
+
+== AUTHOR:
+
+Alf Mikula <amikula@gmail.com>
+
+== ACKNOWLEDGEMENTS:
+
+Thanks to David Johnson for telling me about the Aho-Corasick algorithm, and
+for providing the original Aho-Corasick Ruby implementation.
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 Los Angeles Times
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/config/hoe.rb

@@ -0,0 +1,73 @@
+require 'keyword_prospector/version'
+
+AUTHOR = 'FIXME full name'  # can also be an array of Authors
+EMAIL = "FIXME email"
+DESCRIPTION = "description of gem"
+GEM_NAME = 'keyword_prospector' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'latimes' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+EXTRA_DEPENDENCIES = [
+  ['hpricot', '>= 0.6']
+]    # An array of rubygem dependencies [name, version]
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil
+# UNCOMMENT IF REQUIRED:
+# REV = YAML.load(`svn info`)['Revision']
+VERS = KeywordProspector::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'keyword_prospector documentation',
+    "--opname", "index.html",
+    "--line-numbers",
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' }
+    @extra_deps
+  end
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.developer(AUTHOR, EMAIL)
+  p.description = DESCRIPTION
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = EXTRA_DEPENDENCIES
+
+    #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+  end
+
+CHANGES = $hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+$hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+$hoe.rsync_args = '-av --delete --ignore-errors'
+$hoe.spec.post_install_message = File.open(File.dirname(__FILE__) + "/../PostInstall.txt").read rescue ""

data/config/requirements.rb

@@ -0,0 +1,15 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))

data/lib/hyperlink_strategy.rb

@@ -0,0 +1,56 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require 'set'
+
+class HyperlinkStrategy
+  attr_reader :url
+  attr_reader :options
+
+  def initialize(url=nil, options={})
+    @keywords = Set.new
+    self.options = options
+    self.url = url
+  end
+
+  def keywords=(*keywords)
+    @keywords = Set.new(keywords.flatten)
+  end
+
+  def keywords
+    @keywords
+  end
+
+  def url=(url)
+    @url = url
+    merge_options(@options)
+  end
+
+  def options=(options)
+    merge_options(options)
+  end
+
+  def add_keyword(keyword)
+    @keywords.add(keyword)
+    self
+  end
+
+  def decorate(keyword)
+    attributes = ""
+    options.each_pair do |key, value|
+      attributes += " " unless attributes.length == 0
+      attributes += "#{key}=\"#{value}\""
+    end
+
+    "<a " + attributes + ">#{keyword}</a>"
+  end
+
+  private
+  def merge_options(options)
+    if @url
+      @options = {:href => @url}.merge(options)
+    else
+      @options = options
+    end
+  end
+end

data/lib/keyword_decorator.rb

@@ -0,0 +1,7 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+class KeywordDecorator
+  attr_accessor :keywords
+  attr_accessor :decorator
+end

data/lib/keyword_linker.rb

@@ -0,0 +1,174 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require 'set'
+require 'lookup_chain'
+require 'keyword_prospector'
+require 'hyperlink_strategy'
+require 'rubygems'
+require 'hpricot'
+
+# 
+# Given a set of keywords and url's, and optionally HTML attributes to set
+# on links, takes text and adds hyperlinks from the specified keywords to
+# their associated URL's.  Example:
+#
+#   linker = KeywordLinker.new
+#   linker.add_url('http://www.latimes.com', 'Los Angeles Times')
+#   linker.link_text("Let's check out the Los Angeles Times!")
+#   => "Let's check out the <a href=\"http://www.latimes.com\">Los Angeles Times</a>!"
+#
+# KeywordLinker depends on hpricot for parsing HTML.  This is done to prevent
+# hyperlinks from being added inside of other hyperlinks and inside of
+# attribute text.
+#
+class KeywordLinker  
+  @@blacklist_strategy = Object.new
+  class << @@blacklist_strategy
+    def decorate(keyword)
+      keyword
+    end
+  end
+
+  # Takes an optional array of lookup objects.  A lookup object is anything
+  # that responds to the process method and returns an array of Match objects,
+  # including KeywordLinker, KeywordProspector, and LookupChain objects.  If
+  # multiple objects are specified, a LookupChain is created that gives highest
+  # priority to matches from objects closer to the end of the array.
+  def initialize(*lookups)
+    @tree_initialized=true
+
+    if(lookups)
+      @lookup = LookupChain.new(lookups)
+    end
+  end
+
+  # Takes a url and a keyword String or Array of keywords, and adds it to the
+  # tree of keywords in the KeywordLinker.  Takes an optional hash of html
+  # attributes to be associated with this url.
+  #
+  # Only the first occurrence of the url will be linked.  If multiple keywords
+  # are specified, then only the first occurrence of any of the keywords is
+  # linked to the target url.  ie, if multiple keywords match for this url,
+  # only one instance of one keyword will be linked.
+  def add_url(url, keyword, html_attributes={})
+    init_lookup
+
+    strategy = HyperlinkStrategy.new(url, html_attributes)
+    strategy.keywords = keyword
+
+    @dl.add(strategy)
+  end
+
+  # Blacklist this keyword or array of keywords.  If a keyword is blacklisted,
+  # it will not be linked.  For example, if the "Los Angeles" part of
+  # "Los Angeles Times" is getting linked, you can blacklist
+  # "Los Angeles Times" to keep it from being linked.
+  def blacklist_keyword(keyword)
+    init_lookup
+
+    @dl.add(keyword, @@blacklist_strategy)
+  end
+
+  # Initialize the tree after _all_ url's have been added.  This needs to be
+  # called once.  If you don't call init_tree, it will be called automatically
+  # on the first call to the process or link_text method.  You may find this
+  # annoying or inconvenient if it happens on the first request to your
+  # application and you've constructed a large set of links.  Adding url's
+  # after calling init_tree, process, or link_text is not supported.
+  def init_tree
+    unless @tree_initialized
+      @dl.construct_fail
+      @tree_initialized = true
+    end
+  end
+
+  # Adds links to known url's into the text provided.  Only the first instance
+  # of each keyword or set of keywords associated to a url is linked.  In cases
+  # of overlap, the longest keyword is chosen to resolve the overlap.
+  def link_text(text)
+    init_tree unless @tree_initialized
+
+    linked_outputs = Set.new
+
+    htext = Hpricot(text)
+
+    link_text_in_elem(htext, linked_outputs)
+
+    return htext.to_s
+  end
+
+  # Returns an array of matches in the specified text.  Doesn't filter overlaps
+  # or parse HTML to prevent matches in attribute text or inside of existing
+  # hyperlinks.  Primarily for internal use.
+  def process(text)
+    init_tree unless @tree_initialized
+
+    @lookup.process(text)
+  end
+
+  private
+  # Initialize the KeywordProspector object if needed.  Called only when adding
+  # our own url's, not when aggregating other lookup objects.
+  def init_lookup
+    unless @dl
+      @dl = KeywordProspector.new
+      @tree_initialized = false
+
+      if @lookup
+        @lookup << @dl
+      else
+        @lookup = @dl
+      end
+    end
+  end
+
+  # Given a single hpricot element, link text inside of all child elements.
+  def link_text_in_elem(elem, linked_outputs)
+    elem.children.each do |e|
+      case e
+      when Hpricot::Text
+        text = e.to_s
+
+        results = process(text)
+
+        results.sort!
+        KeywordProspector.filter_overlaps(results)
+
+        unless (results.nil? || results.empty?)
+          e.content = link_text_internal(text, results, linked_outputs)
+        end
+      when Hpricot::Elem
+        link_text_in_elem(e, linked_outputs) if e.stag.name != "a"
+      end
+    end
+  end
+
+  # Called internally to substitute links in element text.
+  def link_text_internal(text, results, linked_outputs = nil)
+    linked_outputs ||= Set.new
+
+    retval = ""
+    cursor = 0
+    results.each do |result|
+      unless linked_outputs.include?(result.output)
+        if(result.start_idx > cursor)
+          retval += text[cursor, result.start_idx - cursor]
+          cursor = result.start_idx
+        end
+
+        retval += result.output.decorate(result.keyword)
+        cursor = result.end_idx
+
+        linked_outputs.add(result.output)
+      end
+    end
+
+    if(cursor < text.size)
+      retval += text[cursor, text.size-cursor]
+    end
+
+    return retval
+  end
+end
+

data/lib/keyword_prospector.rb

@@ -0,0 +1,171 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'state'
+require 'match'
+
+class Position < Struct.new(:begin, :end); end
+
+# KeywordProspector takes a collection of words, and optionally their
+# associated outputs, and builds a match tree for running matches of the
+# keywords against provided text.  While construction of the Aho-Corasick
+# tree takes a long time when there are many keywords, matching runs in time
+# proportional to the length of the text provided.  So, even if you have
+# tens of thousands of keywords to match against, matching will still be
+# very fast.
+class KeywordProspector
+  # If words is provided, each word is added to the tree and the tree is
+  # initialized.  Otherwise, call add for each word to place in the dictionary.
+  def initialize(words=nil)
+    @start = State.new(0, 0, [])
+    if(words)
+      words.each{|word| add word}
+
+      construct_fail
+    end
+  end
+  
+  # Add an entry to the tree.  The optional output parameter can be any object,
+  # and will be returned when this keyword is matched.  If the entry has a
+  # _keywords_ method, it should return a collection of keywords.  In this
+  # case, the output will be added for each keyword provided.  If output is
+  # not provided, the entry is returned.
+  def add(entry, output=nil)
+    output ||= entry
+
+    if (entry.respond_to?(:keywords))
+      entry.keywords.each do |keyword|
+        add_internal(keyword, output)
+      end
+    else
+      add_internal(entry, output)
+    end
+  end
+  
+  # Call once after adding all entries.  This constructs failure links in the
+  # tree, which allow the state machine to move a single step with every input
+  # character instead of backtracking back to the beginning of the tree when
+  # a partial match fails to match the next character.
+  def construct_fail
+    queue = Queue.new
+    @start.values.each do |value|
+      value.fail = @start
+      value.fail_increment = 1
+      queue.push value
+    end
+    
+    prepare_root
+    
+    while !queue.empty?
+      r = queue.pop
+      r.keys.each do |char|
+        s = r[char]
+        queue.push s
+        state = r.fail
+        increment = 0
+        while !state[char]
+          increment += state.fail_increment
+          state = state.fail
+        end
+        s.fail = state[char]
+        s.fail_increment = increment
+      end
+    end
+  end
+  
+  # Process the provided text for matches.  Returns an array of Match objects.
+  # Each Match object contains the keyword matched, the start and end position
+  # in the text, and the output object specified when the keyword was added
+  # to the search tree.  The end position is actually the position of the
+  # character immediately following the end of the keyword, such that end
+  # position minus start position equals the length of the keyword string.
+  #
+  # Options:
+  # * :filter_overlaps - When multiple keywords overlap, filter out overlaps
+  #   by choosing the longest match.
+  def process(bytes, options={})
+    retval = [] unless block_given?
+    state = @start
+    position = Position.new(0, 0)
+    bytes.each_byte do |a|
+      state = state.transition(a, position)
+      if state.keyword && ((position.begin == 0 ||
+                   KeywordProspector.word_delimiter?(bytes[position.begin-1])) &&
+                   (position.end == bytes.length ||
+                    KeywordProspector.word_delimiter?(bytes[position.end])))
+        match = Match.new(state.keyword, position.begin, position.end, state.output)
+
+        # do something with the found item
+        if block_given?
+          yield match
+        else
+          retval << match
+        end
+      end
+    end
+
+    if retval
+      if (options[:filter_overlaps])
+        KeywordProspector.filter_overlaps(retval)
+      end
+    end
+
+    return retval
+  end
+  
+  # Filters overlaps from an array of results.  If two results overlap, the
+  # shorter result is removed.  If both results have the same length, the
+  # second result is removed.
+  def self.filter_overlaps(results)
+    i = 0
+    while (i < results.size-1)
+      a = results[i]
+      b = results[i+1]
+      if a.overlap?(b)
+        if (a.length < b.length)
+          results.delete_at(i)
+        else
+          results.delete_at(i+1)
+        end
+      end
+      i += 1
+    end
+  end
+  
+  private
+  WORD_CHARS=[?a..?z, ?A..?Z, ?0..?9, ?_]
+
+  # Returns true if the character provided is a word character.
+  def self.word_char?(char)
+    WORD_CHARS.each do |spec|
+      return true if spec === char
+    end
+
+    return false
+  end
+
+  # Returns true if the character provided is not a word character.
+  def self.word_delimiter?(char)
+    return !word_char?(char)
+  end
+
+  # Add a single keyword to the tree.
+  def add_internal(keyword, output=nil)
+    cur_state = @start
+    # assuming a string here
+    keyword.each_byte {|c| cur_state = cur_state.insert_next_state(c)}
+    cur_state.keyword = keyword
+    cur_state.output = output
+  end
+
+  # Used internally to create links from root back to itself for all states
+  # that are not beginnings of known keywords.
+  def prepare_root
+    0.upto(255) do |i|
+      @start[i] = @start if !@start[i]    
+    end
+  end
+end