biblicit 1.0 → 2.0.3

data/parscit/resources/sectLabel/funcWord

@@ -0,0 +1,320 @@
+a
+about
+above
+after
+again
+ago
+all
+almost
+along
+already
+also
+although
+always
+am
+among
+an
+and
+another
+any
+anybody
+anything
+anywhere
+are
+aren't
+around
+as
+at
+back
+else
+be
+been
+before
+being
+below
+beneath
+beside
+between
+beyond
+billion
+billionth
+both
+each
+but
+by
+can
+can't
+could
+couldn't
+did
+didn't
+do
+does
+doesn't
+doing
+done
+don't
+down
+during
+eight
+eighteen
+eighteenth
+eighth
+eightieth
+eighty
+either
+eleven
+eleventh
+enough
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+except
+far
+few
+fewer
+fifteen
+fifteenth
+fifth
+fiftieth
+fifty
+first
+five
+for
+fortieth
+forty
+four
+fourteen
+fourteenth
+fourth
+hundred
+from
+get
+gets
+getting
+got
+had
+hadn't
+has
+hasn't
+have
+haven't
+having
+he
+he'd
+he'll
+hence
+her
+here
+hers
+herself
+he's
+him
+himself
+his
+hither
+how
+however
+near
+hundredth
+i
+i'd
+if
+i'll
+i'm
+in
+into
+is
+i've
+isn't
+it
+its
+it's
+itself
+last
+less
+many
+me
+may
+might
+million
+millionth
+mine
+more
+most
+much
+must
+mustn't
+my
+myself
+near
+nearby
+nearly
+neither
+never
+next
+nine
+nineteen
+nineteenth
+ninetieth
+ninety
+ninth
+no
+nobody
+none
+noone
+nothing
+nor
+not
+now
+nowhere
+of
+off
+often
+on
+or
+once
+one
+only
+other
+others
+ought
+oughtn't
+our
+ours
+ourselves
+out
+over
+quite
+rather
+round
+second
+seven
+seventeen
+seventeenth
+seventh
+seventieth
+seventy
+shall
+shan't
+she'd
+she
+she'll
+she's
+should
+shouldn't
+since
+six
+sixteen
+sixteenth
+sixth
+sixtieth
+sixty
+so
+some
+somebody
+someone
+something
+sometimes
+somewhere
+soon
+still
+such
+ten
+tenth
+than
+that
+that
+that's
+the
+their
+theirs
+them
+themselves
+these
+then
+thence
+there
+therefore
+they
+they'd
+they'll
+they're
+third
+thirteen
+thirteenth
+thirtieth
+thirty
+this
+thither
+those
+though
+thousand
+thousandth
+three
+thrice
+through
+thus
+till
+to
+towards
+today
+tomorrow
+too
+twelfth
+twelve
+twentieth
+twenty
+twice
+two
+under
+underneath
+unless
+until
+up
+us
+very
+when
+was
+wasn't
+we
+we'd
+we'll
+were
+we're
+weren't
+we've
+what
+whence
+where
+whereas
+which
+while
+whither
+who
+whom
+whose
+why
+will
+with
+within
+without
+won't
+would
+wouldn't
+yes
+yesterday
+yet
+you
+your
+you'd
+you'll
+you're
+yours
+yourself
+yourselves
+you've

data/parscit/resources/sectLabel/sectLabel.config

@@ -0,0 +1,42 @@
+1token=1
+2token=1
+3token=1
+4token=1
+parscit=1
+parscit_char=1
+tokenCapital=1
+tokenNumber=1
+tokenName=1
+tokenPunct=1
+tokenKeyword=0
+
+lineNum=1
+linePunct=1
+lineLength=1
+lineCapital=0
+linePos=1
+
+1gram=0
+2gram=0
+3gram=0
+4gram=0
+
+xmlLoc=0
+xmlAlign=0
+
+xmlBold=0
+xmlItalic=0
+xmlFontSize=0
+
+xmlPic=0
+xmlTable=0
+xmlBullet=0
+
+bi_xmlA=0
+bi_xmlF=0
+bi_xmlSF=0
+bi_xmlSFBI=0
+bi_xmlSFBIA=0
+bi_xmlPara=0
+
+xmlSpace=0

data/parscit/resources/sectLabel/sectLabel.configXml

@@ -0,0 +1,42 @@
+1token=1
+2token=1
+3token=1
+4token=1
+parscit=1
+parscit_char=1
+tokenCapital=1
+tokenNumber=1
+tokenName=1
+tokenPunct=1
+tokenKeyword=0
+
+lineNum=1
+linePunct=1
+lineLength=1
+lineCapital=0
+linePos=1
+
+1gram=0
+2gram=0
+3gram=0
+4gram=0
+
+xmlLoc=1
+xmlAlign=0
+
+xmlBold=1
+xmlItalic=1
+xmlFontSize=1
+
+xmlPic=1
+xmlTable=1
+xmlBullet=1
+
+bi_xmlA=0
+bi_xmlF=0
+bi_xmlSF=0
+bi_xmlSFBI=0
+bi_xmlSFBIA=1
+bi_xmlPara=1
+
+xmlSpace=0

data/sh/convert_to_text.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ -z "$2" ]; then
+  echo "Usage: ./convert_to_text.sh in.pdf out.txt";
+  exit 1;
+fi
+
+INPUT=$1
+OUTPUT=$2
+TYPE=`file -b --mime-type "$INPUT"`
+
+if [[ "$TYPE" == "text/plain" ]]; then
+  cp "$INPUT" "$OUTPUT";
+elif [[ "$TYPE" == "application/pdf" && -n `which pdftotext` ]]; then
+  pdftotext -raw -enc UTF-8 "$INPUT" "$OUTPUT";
+elif [[ "$TYPE" == "application/postscript" && -n `which ps2ascii` ]]; then
+  ps2ascii "$INPUT" > "$OUTPUT";
+elif [[ -n `which abiword` ]]; then
+  abiword -t txt "$INPUT" -o "$OUTPUT";
+fi

data/spec/biblicit/extractor_spec.rb

@@ -0,0 +1,121 @@
+# encoding: UTF-8
+
+module Biblicit
+
+  describe Extractor do
+
+    unless ENV['LOCAL']
+      it "handles 'Multi-scale collaborative...' using remote cb2bib" do
+        result = Extractor.extract(file: "#{FIXTURES_DIR}/pdf/ICINCO_2010.pdf", tools: [:cb2bib], remote: true)
+
+        cb2bib = result[:cb2bib]
+        cb2bib[:title].should == "Multiscale collaborative searching through swarming"
+        cb2bib[:authors].should == ["W. Liu", "M. B. Short", "Y. E. Taima", "A. L. Bertozzi"]
+        cb2bib[:year].should == 2010
+      end
+    end
+
+    it "handles 'Multi-scale collaborative...' " do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/pdf/ICINCO_2010.pdf")
+
+      citeseer = result[:citeseer]
+      citeseer[:valid].should be_true
+      citeseer[:title].should == 'MULTI-SCALE COLLABORATIVE SEARCHING THROUGH SWARMING'
+      citeseer[:authors].should == ["Wangyi Liu", "Yasser E. Taima", "Martin B. Short", "Andrea L. Bertozzi"]
+
+      parshed = result[:parshed]
+      parshed[:title].should == 'MULTI-SCALE COLLABORATIVE SEARCHING THROUGH SWARMING'
+      parshed[:authors].should == ["Wangyi Liu", "Yasser E Taima", "Martin B Short", "Andrea L Bertozzi"]
+   end
+
+    it "handles 'Multi-scale collaborative...' " do
+      content = IO.read("#{FIXTURES_DIR}/pdf/ICINCO_2010.pdf")
+      result = Extractor.extract(content: content)
+
+      citeseer = result[:citeseer]
+      citeseer[:valid].should be_true
+      citeseer[:title].should == 'MULTI-SCALE COLLABORATIVE SEARCHING THROUGH SWARMING'
+      citeseer[:authors].should == ["Wangyi Liu", "Yasser E. Taima", "Martin B. Short", "Andrea L. Bertozzi"]
+
+      parshed = result[:parshed]
+      parshed[:title].should == 'MULTI-SCALE COLLABORATIVE SEARCHING THROUGH SWARMING'
+      parshed[:authors].should == ["Wangyi Liu", "Yasser E Taima", "Martin B Short", "Andrea L Bertozzi"]
+    end
+
+    it "handles 'Oligopoly, Disclosure...'" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/pdf/Bagnoli Watts TAR 2010.pdf")
+
+      citeseer = result[:citeseer]
+      citeseer[:valid].should be_true
+      citeseer[:title].should == 'Oligopoly, Disclosure, and Earnings Management'
+      citeseer[:authors].should == ["Mark Bagnoli", "Susan G. Watts"]
+
+      parshed = result[:parshed]
+      parshed[:title].should == 'Oligopoly, Disclosure, and Earnings Management'
+      parshed[:authors].should == ["Mark Bagnoli Susan G Watts"]
+    end
+
+    it "handles Google paper" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/pdf/10.1.1.109.4049.pdf")
+
+      citeseer = result[:citeseer]
+      citeseer[:valid].should be_true
+      citeseer[:title].should == 'The Anatomy of a Large-Scale Hypertextual Web Search Engine'
+      citeseer[:authors].should == ['Sergey Brin']
+
+      parshed = result[:parshed]
+      parshed[:title].should == 'The Anatomy of a Large-Scale Hypertextual Web Search Engine'
+      parshed[:authors].should == ['Sergey Brin', 'Lawrence Page']
+    end
+
+    it "handles docx file" do
+      pending "Fails because Abiword can't be installed on current versions of OS X"
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/Review_of_Michael_Tyes_Consciousness_Revisited.docx")
+    end
+
+    it "handles ps file" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/critical-infrastructures.ps")
+
+      citeseer = result[:citeseer]
+      citeseer[:valid].should be_true
+      citeseer[:title].should == "Critical Infrastructures You Can Trust:  Where Telecommunications Fits"
+      citeseer[:authors].should == ["Productname Gpl Ghostscript", "Fred B. Schneider", "Steven M. Bellovin", "Alan S. Inouye"]
+
+      parshed = result[:parshed]
+      parshed[:title].should == "Critical Infrastructures You Can Trust: Where Telecommunications Fits"
+      parshed[:authors].should == ["Fred B Schneider", "Cornell University Steven M Bellovin", "AT&T Labs-Research Alan S Inouye", "Computer Science", "Telecommunications Board"] # not ideal, but ok to match against
+    end
+
+    it "handles ParsCit sample1" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/txt/sample1.txt")
+
+      result[:citeseer][:title].should == "A Calculus of Program Transformations and Its Applications"
+      result[:citeseer][:authors].should == ["Rahma Ben Ayed", "Jules Desharnais", "Ali Mili"]
+
+      result[:parshed][:title].should == "A Calculus of Program Transformations and Its Applications"
+      result[:parshed][:authors].should == ["Rahma Ben Ayed", "Jules Desharnais", "Marc Frappier", "Ali Mili"]
+    end
+
+    it "handles ParsCit sample2" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/txt/sample2.txt")
+
+      result[:citeseer][:title].should ==  "Explanation-Based Learning of Indirect Speech Act Interpretation Rules"
+      result[:citeseer][:authors].should == ["David Schulenburg", "Michael J. Pazzani"]
+
+      result[:parshed][:title].should ==  "Explanation-Based Learning of Indirect Speech Act Interpretation Rules"
+      result[:parshed][:authors].should == ["David Schulenburg Michael J Pazzani"]
+   end
+
+    it "handles ParsCit sample E06" do
+      result = Extractor.extract(file: "#{FIXTURES_DIR}/txt/E06-1050.txt")
+
+      result[:citeseer][:title].should == "A Probabilistic Answer Type Model"
+      result[:citeseer][:authors].should == ["Christopher Pinchak", "Dekang Lin"]
+
+      result[:parshed][:title].should == "A Probabilistic Answer Type Model"
+      result[:parshed][:authors].should == ["Christopher Pinchak", "Dekang Lin"]
+   end
+
+  end
+
+end