keyword_prospector 0.8.0

data/spec/hyperlink_strategy_spec.rb

@@ -0,0 +1,69 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require File.dirname(__FILE__) + '/spec_helper'
+require 'hyperlink_strategy'
+
+describe HyperlinkStrategy do
+  before :each do
+    @strategy = HyperlinkStrategy.new
+  end
+
+  it "Should create hyperlinks to the provided URL" do
+    @strategy.url="http://travel.latimes.com"
+
+    @strategy.decorate("Foo").should == "<a href=\"#{@strategy.url}\">Foo</a>"
+  end
+
+  it "should accept url and options in the constructor" do
+    tmp = HyperlinkStrategy.new(:url, :foo => :bar)
+    tmp.url.should == :url
+    tmp.options[:foo].should == :bar
+  end
+
+  it "should accept options to specify html attributes" do
+    @strategy.options = {:title => "foo title", :style => "hidden;"}
+    @strategy.url = 'foourl'
+    
+    linked_text = @strategy.decorate("Foo")
+    linked_text.should match(%r{<a .*>Foo</a>})
+    linked_text.should match(%r{href="foourl"})
+    linked_text.should match(%r{title="foo title"})
+    linked_text.should match(%r{style="hidden;"})
+  end
+
+  describe "keywords" do
+    it "should allow setting and retrieving keywords" do
+      keywords = %w{a b c d e}
+      @strategy.keywords = keywords
+
+      @strategy.keywords.should == Set.new(keywords)
+    end
+
+    it "should allow comma-separated strings for setting" do
+      @strategy.keywords = "foo", "bar", "baz"
+      @strategy.keywords.should == Set.new(["foo", "bar", "baz"])
+    end
+
+    it "should allow a single string for setting" do
+      @strategy.keywords = "xyzzy"
+      @strategy.keywords.should == Set.new(["xyzzy"])
+    end
+  end
+
+  describe "add_keyword" do
+    it "should add keywords to empty set" do
+      @strategy.keywords.should == Set.new
+
+      @strategy.add_keyword("foo").keywords.should == Set.new("foo")
+    end
+
+    it "should add keywords to existing set" do
+      keywords = %w{foo, bar, baz}
+      @strategy.keywords = keywords
+
+      @strategy.add_keyword("xyzzy").keywords.should ==
+        Set.new(keywords + ["xyzzy"])
+    end
+  end
+end

data/spec/keyword_linker_spec.rb

@@ -0,0 +1,226 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require File.dirname(__FILE__) + '/spec_helper'
+require 'keyword_linker'
+
+describe KeywordLinker do
+  before(:each) do
+    @kl = KeywordLinker.new
+  end
+
+  describe :add_url do
+    it "should accept a string (url) and a single keyword" do
+      @kl.add_url("url", "keyword")
+    end
+
+    it "should accept a string (url) and an array of keywords" do
+      @kl.add_url("url", ["keyword1", "keyword2"])
+    end
+
+    it "should accept options for html attributes" do
+      @kl.add_url("url", "keyword", :class => "awesome")
+
+      linked_text = @kl.link_text("keyword")
+
+      linked_text.should match(%r{^<a .*>keyword</a>$})
+      linked_text.should match(%r{href="url"})
+      linked_text.should match(%r{class="awesome"})
+    end
+  end
+
+  describe :link_text do
+    it "should init_tree if I forget to" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text("Is there a foo in the house?").should ==
+        "Is there a <a href=\"url\">foo</a> in the house?"
+    end
+
+    it "should return original text when there are no matches" do
+      @kl.add_url("url", "foo")
+      @kl.init_tree
+
+      orig_text = "Is there a bar in the house?"
+      @kl.link_text(orig_text).should == orig_text
+    end
+
+    it "should return linked text when URL's are provided with keyword" do
+      @kl.add_url("url", "foo")
+      @kl.init_tree
+
+      @kl.link_text("Is there a foo in the house?").should ==
+        "Is there a <a href=\"url\">foo</a> in the house?"
+    end
+
+    it "should return linked text when URL's are provided with keyword array" do
+      @kl.add_url("url", %w{foo bar baz})
+      @kl.init_tree
+
+      @kl.link_text("pool bar party").should == "pool <a href=\"url\">bar</a> party"
+    end
+
+    it "should link correctly at the beginning of the text" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text("foo is the word").should == '<a href="url">foo</a> is the word'
+    end
+
+    it "should link correctly at the end of the text" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text("the word is foo").should == 'the word is <a href="url">foo</a>'
+    end
+
+    it "should perform multiple links in the text" do
+      @kl.add_url("url1", "foo")
+      @kl.add_url("url2", "bar")
+
+      @kl.link_text("the foo and the bar are awesome").should ==
+        'the <a href="url1">foo</a> and the <a href="url2">bar</a> are awesome'
+    end
+
+    it "should link only the first instance of each keyword" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text("foo, foo, or foo?").should == '<a href="url">foo</a>, foo, or foo?'
+    end
+
+    it "should link only the first instance of each keyword in separate text elements" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text("<i>foo</i>, <b>foo</b>, or <u>foo</u>?").should == '<i><a href="url">foo</a></i>, <b>foo</b>, or <u>foo</u>?'
+    end
+
+    it "should link only the first instance of each url" do
+      @kl.add_url("url", %w[foo bar baz])
+
+      @kl.link_text("bar, baz, or foo?").should == '<a href="url">bar</a>, baz, or foo?'
+    end
+
+    it "should link longest match in overlapping text" do
+      @kl.add_url("url", ["foo bar", "bar baz xyzzy"])
+
+      @kl.link_text("foo bar baz xyzzy").should == 'foo <a href="url">bar baz xyzzy</a>'
+    end
+
+    describe "with another KeywordLinker in the constructor" do
+      before(:each) do
+        @combo_linker = KeywordLinker.new(@kl)
+      end
+
+      it "should link keywords from both linkers" do
+        @kl.add_url("foourl", "foo")
+        @combo_linker.add_url("barurl", "bar")
+
+        @combo_linker.link_text("foo bar").should == '<a href="foourl">foo</a> <a href="barurl">bar</a>'
+      end
+
+      it "should prioritize its own keywords over the parent's keywords" do
+        @kl.add_url("foourl1", "foo")
+        @combo_linker.add_url("foourl2", "foo")
+
+        @combo_linker.link_text("foo").should == '<a href="foourl2">foo</a>'
+      end
+    end
+
+    describe "with an array of KeywordLinkers as parents in the constructor" do
+      before(:each) do
+        @kl2 = KeywordLinker.new
+        @combo_linker = KeywordLinker.new([@kl, @kl2])
+      end
+
+      it "should link keywords from all linkers" do
+        @kl.add_url("foourl", "foo")
+        @kl2.add_url("barurl", "bar")
+        @combo_linker.add_url("bazurl", "baz")
+
+        @combo_linker.link_text("foo bar baz").should == '<a href="foourl">foo</a> <a href="barurl">bar</a> <a href="bazurl">baz</a>'
+      end
+
+      it "should prioritize its own keywords over the parents' keywords" do
+        @kl.add_url("foourl1", "foo")
+        @kl2.add_url("foourl2", "foo")
+        @combo_linker.add_url("foourl3", "foo")
+
+        @combo_linker.link_text("foo").should == '<a href="foourl3">foo</a>'
+      end
+    end
+
+    describe "with an array of KeywordLinkers as lookups in the constructor" do
+      before(:each) do
+        @kl2 = KeywordLinker.new
+        @combo_linker = KeywordLinker.new([@kl, @kl2])
+      end
+
+      it "should link keywords from all lookups" do
+        @kl.add_url("foourl", "foo")
+        @kl2.add_url("barurl", "bar")
+
+        @combo_linker.link_text("foo bar baz").should == '<a href="foourl">foo</a> <a href="barurl">bar</a> baz'
+      end
+    end
+
+    describe "with an arbitrary lookup object in the constructor" do
+      it "should provide results from the lookup object" do
+        lookup = mock(Object)
+        lookup.should_receive(:process).with(:text).and_return([:result])
+        kl = KeywordLinker.new(lookup)
+        kl.process(:text).should == [:result]
+      end
+
+      it "should reject objects from the constructor if they don't have a process method" do
+        lookup = mock(Object)
+        lambda{KeywordLinker.new(nil, lookup)}.should raise_error(ArgumentError)
+      end
+    end
+
+    describe "with multiple level hierarchy" do
+      before(:each) do
+        @kl2 = KeywordLinker.new(@kl)
+        @kl3 = KeywordLinker.new(@kl2)
+        @combo_linker = KeywordLinker.new(@kl3)
+      end
+
+      it "should link keywords from all linkers" do
+        @kl.add_url("foourl", "foo")
+        @kl2.add_url("barurl", "bar")
+        @kl3.add_url("bazurl", "baz")
+        @combo_linker.add_url("xyzzyurl", "xyzzy")
+
+        @combo_linker.link_text("foo bar baz xyzzy").should == '<a href="foourl">foo</a> <a href="barurl">bar</a> <a href="bazurl">baz</a> <a href="xyzzyurl">xyzzy</a>'
+      end
+    end
+  end
+
+  describe "linking html text" do
+    it "should skip linking inside tag attributes" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text('<td title="another foo for you">foo</td>').should ==
+        '<td title="another foo for you"><a href="url">foo</a></td>'
+    end
+
+    it "should not link inside of <a></a> tags" do
+      @kl.add_url("url", "foo")
+
+      @kl.link_text('<a href="bar">baz foo bar</a> and foo').should ==
+        '<a href="bar">baz foo bar</a> and <a href="url">foo</a>'
+    end
+
+    it "shouldn't choke on bogus etags" do
+      @kl.add_url("url", "foo")
+
+      lambda{@kl.link_text('foo </i>')}.should_not raise_error
+    end
+  end
+
+  describe "blacklisting keywords" do
+    it "should stop linking of every occurrence of the keyword" do
+      @kl.add_url("url", "Los Angeles")
+      @kl.blacklist_keyword("Los Angeles Times")
+
+      @kl.link_text("Los Angeles Times Los Angeles Times").should == "Los Angeles Times Los Angeles Times"
+    end
+  end
+end

data/spec/keyword_prospector_spec.rb

@@ -0,0 +1,232 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require File.dirname(__FILE__) + '/spec_helper'
+require 'set'
+
+describe KeywordProspector do
+  it "should match keywords in text, respecting word boundaries" do
+    dl = KeywordProspector.new
+    
+    dl.add('test')
+    dl.add('fido')
+    dl.add('te')
+    dl.add('fi')
+    dl.add('dot')
+    dl.add('dots')
+    dl.add('sis')
+    
+    dl.construct_fail
+    
+    matches = []
+    dl.process('hello fido this is a test') {|x| matches << x}
+    matches.size.should == 2
+    (matches.collect{|match| match.keyword} & %w{test fido}).
+        size.should == 2
+  end
+
+  it "should give correct location for a single match within a string" do
+    dl = KeywordProspector.new(["foo"])
+
+    match = nil
+    dl.process("A foo and his money are soon parted") {|x| match = x}
+
+    match.start_idx.should == 2
+    match.end_idx.should == 5
+  end
+
+  it "should give correct location for the second match within a string" do
+    dl = KeywordProspector.new(["foo", "bar"])
+
+    match = []
+    dl.process("foo bar") {|x| match << x}
+
+    match
+    match[0].keyword.should == "foo"
+    match[0].start_idx.should == 0
+    match[0].end_idx.should == 3
+    match[1].keyword.should == "bar"
+    match[1].start_idx.should == 4
+    match[1].end_idx.should == 7
+  end
+
+  it "Should include information about where the match is present in the string" do
+    dl = KeywordProspector.new %w{foo oo bar baz xyzzy thud}
+
+    matches = {}
+
+    dl.process('foo, bar, xyzzy and also the baz') {|x| matches[x.keyword] = x}
+
+    matches["foo"].should_not be_nil
+    matches["bar"].should_not be_nil
+    matches["baz"].should_not be_nil
+    matches["xyzzy"].should_not be_nil
+
+    matches["foo"].start_idx.should == 0
+    matches["foo"].end_idx.should == 3
+    matches["bar"].start_idx.should == 5
+    matches["bar"].end_idx.should == 8
+    matches["xyzzy"].start_idx.should == 10
+    matches["xyzzy"].end_idx.should == 15
+    matches["baz"].start_idx.should == 29
+    matches["baz"].end_idx.should == 32
+  end
+
+  it "should match a single word to itself" do
+    dl = KeywordProspector.new(["foo"])
+    count = 0
+    dl.process("foo"){count += 1}
+    count.should == 1
+  end
+
+  it "should not match a single word to a different word" do
+    dl = KeywordProspector.new(["foo"])
+    count = 0
+    dl.process("bar"){count += 1}
+    count.should == 0
+  end
+
+  it "should call the block once for every match" do
+    dl = KeywordProspector.new(["foo"])
+    count = 0
+    dl.process("foo foo foo"){count += 1}
+    count.should == 3
+  end
+
+  it "Should get correct start and end matches with overlapping matches" do
+    keywords = ['Sling Blade', 'Blade Runner', 'foo', 'bar']
+    dl = KeywordProspector.new(keywords)
+    candidate = 'Sling Blade Runner foo bar'
+    matches = {}
+    dl.process(candidate) {|x| matches[x.keyword] = x}
+
+    keywords.each do |keyword|
+      matches[keyword].start_idx.should == candidate.index(keyword)
+      matches[keyword].end_idx.should == candidate.index(keyword) +
+                                           keyword.length
+    end
+  end
+
+  it "returns a sorted array of matches when a block is not given" do
+    keywords = %w{foo bar baz xyzzy thud}
+
+    dl = KeywordProspector.new(keywords)
+    results =
+        dl.process("The best metavariables are thud, xyzzy, and of course foo.")
+
+    results.class.should == Array
+
+    results.should == results.sort
+  end
+
+  it "filters out shorter matches multiple matches overlap" do
+    dl = KeywordProspector.new(["a b c", "c d", "e f", "f g h", "i j k l m",
+                               "k l m n o p q"])
+
+    results = dl.process("a b c d, e f g h, i j k l m n o p q",
+                         :filter_overlaps => true)
+
+    results.size.should == 3
+
+    results[0].keyword.should == 'a b c'
+    results[1].keyword.should == 'f g h'
+    results[2].keyword.should == 'k l m n o p q'
+  end
+
+  it "detects word chars" do
+    KeywordProspector.word_char?(?a).should be_true
+    KeywordProspector.word_char?(?k).should be_true
+    KeywordProspector.word_char?(?z).should be_true
+    KeywordProspector.word_char?(?A).should be_true
+    KeywordProspector.word_char?(?K).should be_true
+    KeywordProspector.word_char?(?Z).should be_true
+    KeywordProspector.word_char?(?0).should be_true
+    KeywordProspector.word_char?(?7).should be_true
+    KeywordProspector.word_char?(?9).should be_true
+    KeywordProspector.word_char?(?_).should be_true
+  end
+
+  it "detects non-word chars" do
+    KeywordProspector.word_char?(?-).should be_false
+    KeywordProspector.word_char?(?>).should be_false
+    KeywordProspector.word_char?(?<).should be_false
+    KeywordProspector.word_char?(?.).should be_false
+    KeywordProspector.word_char?(32).should be_false
+    KeywordProspector.word_char?(9).should be_false
+  end
+
+  it "word_delimiter? is opposite of word_char?" do
+    KeywordProspector.word_delimiter?(?.).should be_true
+    KeywordProspector.word_delimiter?(32).should be_true
+    KeywordProspector.word_delimiter?(?K).should be_false
+  end
+
+  describe "word boundary detection" do
+    before(:each) do
+      keywords = %w{foo bar baz xyzzy thud}
+      @dl = KeywordProspector.new(keywords)
+    end
+
+    describe "allows" do
+      it "matching at beginning of string" do
+        results = @dl.process("foo is the word")
+
+        results.size.should == 1
+        results[0].keyword.should == "foo"
+      end
+
+      it "matching at end of string" do
+        results = @dl.process("the word is bar")
+
+        results.size.should == 1
+        results[0].keyword.should == "bar"
+      end
+    end
+
+    describe "doesn't allow" do
+      it "matches not starting on a word boundary" do
+        results = @dl.process("topaz is a gem but tobaz is not")
+        results.size.should == 0
+      end
+
+      it "matches not ending on a word boundary" do
+        results = @dl.process("are you xyzzypated?")
+        results.size.should == 0
+      end
+
+      it "matches at the beginning of the string and not ending on a word boundary" do
+        results = @dl.process("fooby you too?")
+        results.size.should == 0
+      end
+
+      it "multiple candidate matches in various places" do
+        results = @dl.process("fooby barby bazby tofoo tobar tobaz ambazbafoo")
+        results.size.should == 0
+      end
+    end
+  end
+
+  describe "with decoration strategy objects" do
+    it "should read keywords from the object" do
+      strategy = Object.new
+      strategy.should_receive(:keywords).and_return(Set.new(%w{foo bar baz}))
+
+      dl = KeywordProspector.new([strategy])
+    end
+
+    it "should return strategy objects in results" do
+      strategy = Object.new
+      strategy.should_receive(:keywords).and_return(Set.new(%w{foo bar baz}))
+
+      dl = KeywordProspector.new([strategy])
+
+      results = dl.process("foo, bar, and baz")
+
+      results.size.should == 3
+
+      results.each do |result|
+        result.output.should == strategy
+      end
+    end
+  end
+end

data/spec/lookup_chain_spec.rb

@@ -0,0 +1,104 @@
+# 
+# (C) 2008 Los Angeles Times
+# 
+require File.dirname(__FILE__) + '/spec_helper'
+require 'lookup_chain'
+require 'match'
+
+describe LookupChain do
+  before(:each) do
+    @dl1 = mock(Object, :process => :dummy_method)
+    @dl2 = mock(Object, :process => :dummy_method)
+  end
+
+  describe :initialize do
+    describe "should check all objects for a process method" do
+      it "when given an array of objects" do
+        lambda {LookupChain.new([@dl1])}.should_not raise_error
+        lambda {LookupChain.new([Object.new])}.should raise_error(ArgumentError)
+      end
+
+      it "when given multiple objects in constructor" do
+        lambda {LookupChain.new(@dl1, @dl2)}.should_not raise_error
+        lambda {LookupChain.new(@dl1, Object.new)}.should raise_error(ArgumentError)
+      end
+    end
+  end
+
+  describe :<< do
+    it "should check for a process method" do
+      lambda{LookupChain.new << @dl1}.should_not raise_error
+      lambda{LookupChain.new << Object.new}.should raise_error(ArgumentError)
+    end
+
+    it "should add to the end of the list of lookups" do
+      lc = LookupChain.new
+      lc.lookups.should == []
+
+      lc << @dl1
+      lc.lookups.should == [@dl1]
+
+      lc << @dl2
+      lc.lookups.should == [@dl1, @dl2]
+    end
+  end
+
+  describe :lookups do
+    it "should return an array of lookup objects assigned in constructor" do
+      LookupChain.new(@dl1, @dl2).lookups.should == [@dl1, @dl2]
+      LookupChain.new([@dl2, @dl1]).lookups.should == [@dl2, @dl1]
+    end
+  end
+
+  describe :process do
+    it "should call process on all child objects" do
+      @dl1.should_receive(:process).with(:text).and_return([])
+      @dl2.should_receive(:process).with(:text).and_return([])
+      lc = LookupChain.new(@dl1, @dl2)
+
+      lc.process(:text)
+    end
+
+    it "should return a sorted list of match objects from all lookups" do
+      match1 = Match.new("match1", 0, 3)
+      match2 = Match.new("match2", 5, 7)
+      match3 = Match.new("match3", 11, 13)
+      match4 = Match.new("match4", 19, 31)
+
+      @dl1.should_receive(:process).with(:text).and_return([match1, match4])
+      @dl2.should_receive(:process).with(:text).and_return([match2, match3])
+
+      lc = LookupChain.new(@dl1, @dl2)
+
+      lc.process(:text).should == [match1, match2, match3, match4]
+    end
+
+    it "should return the longest match when there are overlaps, regardless of priority order" do
+      match1 = Match.new("match1", 0, 3)
+      match2 = Match.new("match2", 1, 7)
+
+      @dl1.stub!(:process).with(:text).and_return([match1])
+      @dl2.stub!(:process).with(:text).and_return([match2])
+
+      lc = LookupChain.new(@dl1, @dl2)
+      lc.process(:text).should == [match2]
+
+      lc = LookupChain.new(@dl2, @dl1)
+      lc.process(:text).should == [match2]
+    end
+
+    it "should prioritize the first lookup object in the list when there are overlapping matches of equal length" do
+      match1 = Match.new("match1", 0, 3)
+      match2 = Match.new("match2", 1, 4)
+
+      @dl1.stub!(:process).with(:text).and_return([match1])
+      @dl2.stub!(:process).with(:text).and_return([match2])
+
+      lc = LookupChain.new(@dl1, @dl2)
+      lc.process(:text).should == [match1]
+
+      lc = LookupChain.new(@dl2, @dl1)
+      lc.process(:text).should == [match2]
+    end
+  end
+end