excite 2.1.1

data/spec/excite/citation_spec.rb

@@ -0,0 +1,128 @@
+# encoding: UTF-8
+
+describe Excite do
+
+  context "parse string" do
+    it "handles nil" do
+      Excite.parse_string(nil).should be_nil
+    end
+
+    it "handles empty string" do
+      Excite.parse_string("").should be_nil
+    end
+
+    it "handles string that used to break model" do
+      Excite.parse_string("[if gte mso 10]>\r\n<style>\r\n /* Style Definitions */\r\n table.MsoNormalTable\r\n\t{mso-style-name:\"Table Normal\";\r\n\tmso-tstyle-rowband-size:0;\r\n\tmso-tstyle-colband-size:0;\r\n\tmso-style-noshow:yes;\r\n\tmso-style-priority:99;\r\n\tmso-style-parent:\"\";\r\n\tmso-padding-alt:0in 5.4pt 0in 5.4pt;\r\n\tmso-para-margin:0in;\r\n\tmso-para-margin-bottom:.0001pt;\r\n\tmso-pagination:widow-orphan;\r\n\tfont-size:10.0pt;\r\n\tfont-family:\"Times New Roman\",\"serif\";}\r\n</style>\r\n<![endif]")[:year].should be_nil
+    end
+
+    it "handles non-ASCII unicode characters" do
+      cite = Excite.parse_string("Okuda, Michael, and Denise Okuda. 1993. Star trek chronology » The history of the future りがと. New York: Pocket Books.")
+      title_should_be(cite, "Star trek chronology » The history of the future")
+    end
+
+    it "handles non-citation string" do
+      Excite.parse_string("Recently while contemplating hosting options for my startup I decided to take a look at Heroku.")[:authors].should be_nil
+    end
+
+    it "parses title for APA journal article" do
+      cite = Excite.parse_string("Devine, P. G., & Sherman, S. J. (1992). Intuitive versus rational judgment and the role of stereotyping in the human condition: Kirk or Spock? Psychological Inquiry, 3(2), 153-159. doi:10.1207/s15327965pli0302_13")
+      title_should_be(cite, "Intuitive versus rational judgment and the role of stereotyping in the human condition: Kirk or Spock")
+    end
+
+    it "parses title for Turabian journal article" do
+      cite = Excite.parse_string("Wilcox, Rhonda V. 1991. Shifting roles and synthetic women in Star trek: The next generation. Studies in Popular Culture 13 (June): 53-65.")
+      title_should_be(cite, 'Shifting roles and synthetic women in Star trek: The next generation')
+    end
+
+    it "parses title for Turabian book" do
+      cite = Excite.parse_string("Okuda, Michael, and Denise Okuda. 1993. Star trek chronology: The history of the future. New York: Pocket Books.")
+      title_should_be(cite, "Star trek chronology: The history of the future")
+    end
+
+    it "parses title for MLA newspaper article" do
+      cite = Excite.parse_string('Di Rado, Alicia. "Trekking through College: Classes Explore Modern Society Using the World of Star Trek." Los Angeles Times 15 Mar. 1995: A3+. Print.')
+      title_should_be(cite, 'Trekking through College: Classes Explore Modern Society Using the World of Star Trek')
+    end
+
+    it "parses title for Chicago journal article" do
+      cite = Excite.parse_string('Wilcox, Rhonda V. 1991. Shifting roles and synthetic women in Star trek: The next generation. Studies in Popular Culture 13 (2): 53-65.')
+      title_should_be(cite, 'Shifting roles and synthetic women in Star trek: The next generation')
+    end
+
+    it "parses title for journal article with volume" do
+      cite = Excite.parse_string("Watts, S. & Bagnoli, M. (2010). Oligopoly, Disclosure and Earnings Management. The Accounting Review, vol. 85 (4), 1191-1214.")
+      title_should_be(cite, "Oligopoly, Disclosure and Earnings Management")
+    end
+
+    it "parses title for MLA journal article" do
+      cite = Excite.parse_string('Hodges, F. M. "The Promised Planet: Alliances and Struggles of the Gerontocracy in American Television Science Fiction of the 1960s." Aging Male 6.3 (2003)')
+      title_should_be(cite, "The Promised Planet: Alliances and Struggles of the Gerontocracy in American Television Science Fiction of the 1960s")
+    end
+
+    it "parses title for AMA journal article" do
+      cite = Excite.parse_string("Wilcox RV. Shifting roles and synthetic women in Star trek: The next generation. Stud Pop Culture. 1991;13:53-65.")
+      title_should_be(cite, 'Shifting roles and synthetic women in Star trek: The next generation')
+    end
+
+    it "parses quoted journal article title" do
+      cite = Excite.parse_string('“Standing in Livestock’s ‘Long Shadow’: The Ethics of Eating Meat on a Small Planet,” Ethics & the Environment 16 (2011): 63-93. (pdf)')
+      title_should_be(cite, 'Standing in Livestock\'s `Long Shadow\': The Ethics of Eating Meat on a Small Planet')
+    end
+
+    it "parses citation prefixed by number" do
+      cite = Excite.parse_string('1. “Mechanisms of network collapse in GeO2 glass: high-pressure neutron diffraction with isotope substitution as arbitrator of competing models ” Kamil Wezka ,Philip Salmon, Anita Ziedler, Dean Whittaker, James Drewitt, Stefan Klotz, Harry Fisher and D Marrocchelli, Journal of Physics: Condensed Matter 24 502101 (2012)')
+      title_should_be(cite, 'Mechanisms of network collapse in GeO2 glass: high-pressure neutron diffraction with isotope substitution as arbitrator of competing models')
+    end
+
+    it "parses citation prefixed by number without space" do
+      cite = Excite.parse_string("3.“ High pressure neutron diffraction study of GeO2 glass up to 17.5 GPa ” Philip Salmon, James Drewitt, Dean Whittaker, Anita Ziedler, Kamil Wezka, Craig Bull, Mathew Tucker, Martin Wilding, Malcon Guthrie and D Marrocchelli, Journal of Physics: Condensed Matter 24 415102 (2012)")
+      title_should_be(cite, 'High pressure neutron diffraction study of GeO2 glass up to 17.5 GPa')
+    end
+
+    it "parses citation with name not in dict" do
+      cite = Excite.parse_string("John Xkcd, Analyzing Phonetic Variation. Journal of Digital Scholarship\nNov. 2011", "John Xkcd")
+      title_should_be(cite, "Analyzing Phonetic Variation")
+    end
+
+    it "parses citation with parenthetical comment" do
+      cite = Excite.parse_string('The Ethics of Creativity: Beauty, Morality, and Nature in a Processive Cosmos (University of Pittsburgh Press 2005). (Awarded the Metaphysical Society of America’s 2007 John N. Findlay Book Prize.)')
+      title_should_be(cite, 'The Ethics of Creativity: Beauty, Morality, and Nature in a Processive Cosmos')
+    end
+
+  end
+
+  context "parse html" do
+
+    it "parses cleanly marked-up cite" do
+      cite_str = %{
+			<h3 class="PaperTitle">
+				<span class="AuthorList">Wangyi Liu, Andrea Bertozzi, and Theodore Kolokolnikov,</span>
+					<a class="Title" href="http://www.math.ucla.edu/~bertozzi/papers/CMS-Bobby12-galley.pdf">“Diffuse interface surface tension models in an expanding flow”,</a>
+				<span class="Source">Communications in Mathematical Sciences,</span>
+				<span class="DisplayDate">2012,</span>
+				<span class="Volume">10(1)</span>:<span class="Page">387-418,</span>
+			</h3> }
+
+      cite = Excite.parse_html(cite_str)
+      title_should_be(cite, "Diffuse interface surface tension models in an expanding flow")
+
+      cite[:authors].to_set.should == ["Wangyi Liu", "Andrea Bertozzi", "Theodore Kolokolnikov"].to_set
+      cite[:journal].should == "Communications in Mathematical Sciences"
+    end
+
+    it "parses cite wihout much punctuation" do
+      cite_str = "<span>Devine, PG, & Sherman, SJ</span><span>(1992)</span><strong>Intuitive versus rational judgment and the role of stereotyping in the human condition: Kirk or Spock?</strong><em>Psychological Inquiry</em><span>3(2), 153-159</span>"
+
+      cite = Excite.parse_html(cite_str)
+      title_should_be(cite, 'Intuitive versus rational judgment and the role of stereotyping in the human condition: Kirk or Spock')
+    end
+
+  end
+
+  def title_should_be(cite, title)
+    cite[:title].should == title
+    cite.overall_probability.should be_within(0.5).of(0.5)
+    cite.probabilities[:title].should be_within(0.5).of(0.5)
+  end
+
+end

data/spec/excite/crfparser_spec.rb

@@ -0,0 +1,118 @@
+# encoding: UTF-8
+
+module Excite
+
+  describe CRFParser do
+
+    before do
+      @parser = CRFParser.new
+    end
+
+    describe "normalize_input_author" do
+
+      it "handles blank" do
+        @parser.normalize_input_author(nil).should be_nil
+        @parser.normalize_input_author('').should be_nil
+      end
+
+      it "handles name with junk punctuation" do
+        res = @parser.normalize_input_author("'Gertjan van Noord'")
+        res.should == ['gertjan', 'van', 'noord']
+      end
+
+    end
+
+    describe "tokenizing" do
+
+      describe "html training data" do
+        TAGGED_HTML = "<author> &lt;li&gt;González-Bailón, S. </author> <date> (2009) </date> <title> &lt;a&gt;Traps on the Web&lt;/a&gt;. </title> <journal> Information, Communication &amp; Society </journal> <volume> 12 (8) </volume> <pages> 1149-1173.&lt;/li&gt; </pages>"
+
+        it "is labeled correctly" do
+          toks = CRFParser.new(:html).prepare_token_data(TAGGED_HTML, true)
+
+          expected = [
+            ['González-Bailón','li','author'],
+            [',','li','author'],
+            ['S.','li','author'],
+            ['(','li','date'],
+            ['2009','li','date'],
+            [')','li', 'date'],
+            ['Traps','a','title'],
+            ['on','a','title'],
+            ['the','a','title'],
+            ['Web','a','title'],
+            ['.','li','title'],
+            ['Information','li','journal'],
+            [',','li','journal'],
+            ['Communication','li','journal'],
+            ['&','li','journal'],
+            ['Society','li','journal'],
+            ['12','li','volume'],
+            ['(','li','volume'],
+            ['8','li','volume'],
+            [')','li','volume'],
+            ['1149-1173', 'li', 'pages'],
+            ['.','li','pages']
+          ]
+
+          toks.length.should == expected.length
+
+          expected.each_with_index do |e, i|
+            t = toks[i]
+            t.raw.should == e[0]
+            t.node.parent.name.should == e[1]
+            t.label.should == e[2]
+          end
+        end
+
+      end
+
+      describe "html test data" do
+        HTML = "<li><b>Author Name</b> (2012) <a>Paper Title.</a><!-- This is a comment -->Journal Title 15:2 123-234.<span>&nbsp;</span></li>"
+
+        it "is stripped of empty tags and comments" do
+          toks = CRFParser.new(:html).prepare_token_data(HTML)
+
+          toks.each do |tok|
+            tok.node.name.should == 'text'
+            tok.node.parent.name.should_not == 'comment'
+            tok.node.parent.name.should_not == 'span'
+          end
+        end
+
+        it "is tokenized correctly" do
+          expected = [
+            ['Author','b'],
+            ['Name','b'],
+            ['(', 'li'],
+            ['2012', 'li'],
+            [')','li'],
+            ['Paper','a'],
+            ['Title','a'],
+            ['.','a'],
+            ['Journal','li'],
+            ['Title','li'],
+            ['15','li'],
+            [':2','li'],
+            ['123-234','li'],
+            ['.','li']
+          ]
+
+          toks = CRFParser.new(:html).prepare_token_data(HTML)
+
+          toks.length.should == expected.length
+
+          expected.each_with_index do |e, i|
+            t = toks[i]
+            t.raw.should == e[0]
+            t.node.parent.name.should == e[1]
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/spec/excite/postprocessor_spec.rb

@@ -0,0 +1,68 @@
+# encoding: UTF-8
+
+require 'excite/postprocessor'
+
+module Excite
+
+  describe Postprocessor do
+
+    describe "normalize_title" do
+
+      it "strips whitespace" do
+        normalize(' a title .').should == 'a title'
+      end
+
+      it "strips punctuation" do
+        normalize('(a title) ').should == 'a title'
+      end
+
+      it "strips leading numerals" do
+        normalize('1. A title').should == 'A title'
+      end
+
+      it "doesn't strip numerals part of the title" do
+        normalize('1 is the best number of titles').should == '1 is the best number of titles'
+      end
+
+      it "strips leading roman numerals" do
+        normalize('xiv. A title').should == 'A title'
+      end
+
+      it "doesn't strip roman numeral-like title starts" do
+        normalize('IVs are needles not titles').should == 'IVs are needles not titles'
+      end
+
+      it "strips leading enumerating letters" do
+        normalize('A. My title').should == 'My title'
+      end
+
+      it "doesn't strip leading single letters" do
+        normalize('A title').should == 'A title'
+      end
+
+      it "extracts title from between quotes" do
+        normalize('"A title" which is cool').should == 'A title'
+      end
+
+      it "doesn't reduce title to quote part" do
+        normalize('This title comments on "some other title": a crappy work').should == 'This title comments on "some other title": a crappy work'
+      end
+
+      it "chops content after a newline" do
+        normalize("A title\nActually an author or journal").should == 'A title'
+      end
+
+      it "doesn't chop content after a newline if there's not enough before the newline" do
+        normalize("A\ntitle mostly after the newline").should == "A\ntitle mostly after the newline"
+      end
+
+      def normalize(title)
+        hsh = { "title" => title }
+        CRFParser.new.normalize_title(hsh)
+        hsh["title"]
+      end
+
+    end
+
+  end
+end