excite 2.1.1

data/lib/excite/resources/trainingdata/verify.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+#
+# Script to assist in verifying tagged references
+
+f = ARGV[0]
+
+cleaned = IO.readlines(f).map(&:strip)
+
+tags = []
+tag_contents = {}
+
+annotation_tags = %w{ author title date booktitle journal volume pages editor workid link publisher location institution bullet tech note }.map { |t| "<#{t}>" }
+
+cleaned.each_with_index do |line, index|
+  open_tags = line.scan(/<\s*\w+\s*>/).map(&:downcase)
+  for tag in open_tags
+    i = line.downcase.index(tag)
+    j = line.downcase.index(tag.sub('<','</'))
+    if tag != "<br>" && (j.nil? || j <= i)
+      puts "Missing close tag for #{tag} on line #{index+1}: #{line}"
+    end
+  end
+
+  close_tags = line.scan(/<\/\s*\w+\s*>/).map(&:downcase)
+  for tag in close_tags
+    i = line.downcase.index(tag)
+    j = line.downcase.index(tag.sub(/<\/\s*/, ''))
+    if j.nil? || j >= i
+      puts "Missing open tag for #{tag} on line #{index+1}: #{line}"
+    end
+  end
+
+  tags += open_tags
+
+  toks = line.split(/(\s+)|(?=<)|(?<=>)/)
+
+  start_tag = nil
+  tag_content = ''
+
+  for tok in toks
+    if annotation_tags.include?(tok)
+      if !start_tag.nil?
+        puts "Started #{tok} within #{start_tag} on line #{index+1}: #{line}"
+        start_tag = nil
+      else
+        start_tag = tok
+      end
+    elsif annotation_tags.include?(tok.sub(/<\/\s*/, '<'))
+      if start_tag.nil?
+        puts "End tag #{tok} without a corresponding start tag on line #{index+1}: #{line}"
+      elsif start_tag != tok.sub(/<\/s*/, '<')
+        puts "End tag #{tok} doesn't match start tag #{start_tag} on line #{index+1}: #{line}"
+        start_tag = nil
+      else
+        tag_contents[start_tag] ||= []
+        tag_contents[start_tag] << tag_content.strip
+
+        tag_content = ''
+        start_tag = nil
+      end
+    elsif start_tag.nil?
+      puts "Token '#{tok}' is not tagged in line #{index+1}: #{line}" unless tok.strip.empty?
+    else
+      tag_content += tok
+    end
+  end
+
+  for tag in annotation_tags
+    if open_tags.count(tag) > 1
+      puts "(Might be ok but...) More than one #{tag} in line #{index+1}: #{line}"
+    end
+  end
+
+  for tag in open_tags
+    if open_tags.count(tag) != close_tags.count(tag.sub('<','</'))
+      puts "Unequal numbers of open and close tags for #{tag} in line #{index+1}: #{line}" unless tag.match(/<\/?br>/)
+    end
+  end
+
+  if !open_tags.include?("<title>")
+    puts "Missing title on line #{index+1}: #{line}"
+  end
+end
+
+tag_counts = tags.inject({}) do |counts, tag|
+  counts[tag] ||= 0
+  counts[tag] += 1
+  counts
+end
+
+puts "\n\nAnnotation tags used: #{tag_counts.select { |t,c| annotation_tags.include?(t) } }"
+puts "Other tags: #{tag_counts.reject { |t,c| annotation_tags.include?(t) }}\n\n\n"
+
+tag_contents.each do |tag, contents|
+  puts "#{tag}s:"
+  contents.each { |c| puts "\t\t#{c}" }
+end

data/lib/excite/token_features.rb

@@ -0,0 +1,313 @@
+# encoding: UTF-8
+
+module Excite
+
+  module TokenFeatures
+
+    module DictFlags
+      PUBLISHER_NAME = 32
+      PLACE_NAME = 16
+      MONTH_NAME = 8
+      LAST_NAME = 4
+      FIRST_NAME = 1
+    end
+
+    def TokenFeatures.read_dict_files(dir_name)
+      dict = {}
+      [
+        ['first-names',DictFlags::FIRST_NAME],
+        ['surnames',DictFlags::LAST_NAME],
+        ['months',DictFlags::MONTH_NAME],
+        ['places',DictFlags::PLACE_NAME],
+        ['publishers',DictFlags::PUBLISHER_NAME],
+      ].each do |file_name, flag|
+        filename = File.join(dir_name, file_name)
+        f = File.open(filename, 'r')
+
+        while l = f.gets
+          l.strip!
+          if !l.match(/^\#/)
+            dict[l] ||= 0
+            unless dict[l] & flag > 0
+              dict[l] += flag
+            end
+          end
+        end
+
+        f.close
+      end
+      dict
+    end
+
+    DIR = File.dirname(__FILE__)
+    DICT = TokenFeatures.read_dict_files("#{DIR}/resources/dicts")
+
+    private_class_method :read_dict_files
+
+    def clear
+      @possible_editor = nil
+      @possible_chapter = nil
+      @dict_status = nil
+      @is_proceeding = nil
+    end
+
+    def last_char(toks, idx, author_names=nil)
+      case toks[idx].raw[-1,1]
+        when /[a-z]/
+          'a'
+        when /[A-Z]/
+          'A'
+        when /[0-9]/
+          0
+        else
+          toks[idx].raw[-1,1]
+      end
+    end
+
+    def first_1_char(toks, idx, author_names=nil); toks[idx].raw[0,1]; end
+    def first_2_chars(toks, idx, author_names=nil); toks[idx].raw[0,2]; end
+    def first_3_chars(toks, idx, author_names=nil); toks[idx].raw[0,3]; end
+    def first_4_chars(toks, idx, author_names=nil); toks[idx].raw[0,4]; end
+    def first_5_chars(toks, idx, author_names=nil); toks[idx].raw[0,5]; end
+
+    def last_1_char(toks, idx, author_names=nil); toks[idx].raw[-1,1]; end
+    def last_2_chars(toks, idx, author_names=nil); toks[idx].raw[-2,2] || toks[idx].raw; end
+    def last_3_chars(toks, idx, author_names=nil); toks[idx].raw[-3,3] || toks[idx].raw; end
+    def last_4_chars(toks, idx, author_names=nil); toks[idx].raw[-4,4] || toks[idx].raw; end
+
+    def toklcnp(toks, idx, author_names=nil); toks[idx].lcnp; end
+
+    def capitalization(toks, idx, author_names=nil)
+      case toks[idx].np
+        when "EMPTY"
+          "others"
+        when /^[[:upper:]]$/
+          "singleCap"
+        when /^[[:upper:]][[:lower:]]+/
+          "InitCap"
+        when /^[[:upper:]]+$/
+          "AllCap"
+        else
+          "others"
+      end
+    end
+
+    def numbers(toks, idx, author_names=nil)
+      (toks[idx].raw           =~ /[0-9]\-[0-9]/)          ? "possiblePage" :
+        (toks[idx].raw         =~ /^\D*(19|20)[0-9][0-9]\D*$/)   ? "year"         :
+        (toks[idx].np       =~ /^(19|20)[0-9][0-9]$/)   ? "year"         :
+        (toks[idx].np       =~ /^[0-9]$/)               ? "1dig"         :
+        (toks[idx].np       =~ /^[0-9][0-9]$/)          ? "2dig"         :
+        (toks[idx].np       =~ /^[0-9][0-9][0-9]$/)     ? "3dig"         :
+        (toks[idx].np       =~ /^[0-9]+$/)              ? "4+dig"        :
+        (toks[idx].np       =~ /^[0-9]+(th|st|nd|rd)$/) ? "ordinal"      :
+        (toks[idx].np       =~ /[0-9]/)                 ? "hasDig"       : "nonNum"
+    end
+
+    # ignores idx
+    def possible_editor(toks, idx=nil, author_names=nil)
+      if !@possible_editor.nil?
+        @possible_editor
+      else
+        @possible_editor =
+          (toks.any? { |t|  %w(ed editor editors eds edited).include?(t.lcnp) } ?
+            "possibleEditors" : "noEditors")
+      end
+    end
+
+    # if there is possible editor entry and "IN" preceeded by punctuation
+    # this citation may be a book chapter
+    #
+    # ignores idx
+    def possible_chapter(toks, idx=nil, author_names=nil)
+      if !@possible_chapter.nil?
+        @possible_chapter
+      else
+        has_editor = possible_editor(toks) == 'possibleEditors'
+        has_chapter = toks.each_with_index.any? do |t, i|
+          if i > 0 && i < (toks.length-1) && t.lcnp == 'in'
+            prev_is_separator = ['pp','ppr','ppc','pps'].include?(toks[i-1].part_of_speech)
+            next_is_separator = ['ppl','ppc','pps'].include?(toks[i+1].part_of_speech)
+            prev_is_separator && (has_editor || next_is_separator)
+          end
+        end
+        has_chapter ? "possibleChapter" : "noChapter"
+      end
+    end
+
+    # ignores idx
+    def is_proceeding(toks, idx=nil, author_names=nil)
+      if !@is_proceeding.nil?
+        @is_proceeding
+      else
+        @is_proceeding =
+          (toks.any? { |t|
+            %w( proc proceeding proceedings ).include?(t.lcnp.strip)
+          } ? 'isProc' : 'noProc')
+      end
+    end
+
+    # TODO remove duplication with possible_chapter
+    def is_in(toks, idx, author_names=nil)
+      is_in = if idx > 0 && idx < (toks.length-1) && toks[idx].lcnp == 'in'
+        prev_is_separator = ['pp','ppr','ppc','pps'].include?(toks[idx-1].part_of_speech)
+        next_is_separator = ['ppl','ppc','pps'].include?(toks[idx+1].part_of_speech)
+        prev_is_separator && (next_is_separator || toks[idx+1].np =~ /^[A-Z]/)
+      end
+      is_in ? "inBook" : "notInBook"
+    end
+
+    def location(toks, idx, author_names=nil)
+      r = ((idx.to_f / toks.length) * 10).round
+    end
+
+    def punct(toks, idx, author_names=nil)
+      (toks[idx].raw =~ /\-.*\-/)              ? "multiHyphen" :
+      (toks[idx].raw =~ /[[:alpha:]].*\-$/)    ? "truncated"   :
+      (toks[idx].raw =~ /[[:alpha:]].*\.$/)    ? "abbrev"      :
+      (toks[idx].np != toks[idx].raw)          ? "hasPunct"    : "others"
+    end
+
+    def possible_volume(toks, idx, author_names=nil)
+      if possible_vol_with_str(toks, idx)
+        'volume'
+      elsif possible_vol_with_str(toks, idx-1) && possible_issue_with_str(toks, idx)
+        'issue'
+      elsif possible_vol_with_str(toks, idx-2) && possible_issue_with_str(toks, idx-1) && possible_issue_with_str(toks, idx)
+        'issue'
+      elsif possible_vol_with_parens(toks, idx)
+        'volume'
+      elsif (1..3).any? { |i| possible_vol_with_parens(toks, idx-i) }
+        'issue'
+      elsif possible_vol_with_colon(toks, idx)
+        'volume'
+      else
+        'noVolume'
+      end
+    end
+
+    # TODO this method is weirdly named b/c of alphabetical ordering hack: remove that
+    def a_is_in_dict(toks, idx, author_names=nil)
+      dict_status(toks, idx)
+    end
+
+    def publisherName(toks, idx, author_names=nil)
+      (dict_status(toks, idx) & DictFlags::PUBLISHER_NAME) > 0 ? 'publisherName' : 'noPublisherName'
+    end
+
+    def placeName(toks, idx, author_names=nil)
+      (dict_status(toks, idx) & DictFlags::PLACE_NAME) > 0 ? 'placeName' : 'noPlaceName'
+    end
+
+    def monthName(toks, idx, author_names=nil)
+      (dict_status(toks, idx) & DictFlags::MONTH_NAME) > 0 ? 'monthName' : 'noMonthName'
+    end
+
+    def lastName(toks, idx, author_names=nil)
+      return 'lastName' if author_names && author_names.last == toks[idx].lcnp
+      (dict_status(toks, idx) & DictFlags::LAST_NAME) > 0 ? 'lastName' : 'noLastName'
+    end
+
+    def firstName(toks, idx, author_names=nil)
+      return 'firstName' if author_names && author_names.first == toks[idx].lcnp
+      (dict_status(toks, idx) & DictFlags::FIRST_NAME) > 0 ? 'firstName' : 'noFirstName'
+    end
+
+    def dict_status(toks, idx)
+      @dict_status ||= [nil]*toks.length
+      @dict_status[idx] ||= (DICT[toks[idx].lcnp] || 0)
+    end
+
+    NODE_TYPES_BY_NAME = {
+      'div'=>'div',
+      'p'=>'p',
+      'ul'=>'div', # lump with div - higher-level structure
+      'li'=>'li',
+      'tr'=>'div', # lump with div - higher-level structure
+      'td'=>'td',
+      'span'=>'span',
+      'font'=>'span',
+      'em'=>'em',
+      'i'=>'em',
+      'strong'=>'strong',
+      'b'=>'strong',
+      'u'=>'u',
+      'h1'=>'h',
+      'h2'=>'h',
+      'h3'=>'h',
+      'h4'=>'h',
+      'h5'=>'h',
+      'h6'=>'h',
+      'a'=>'a',
+      '#document-fragment'=>'unknown' # the actual tag wasn't captured in the fragment we're parsing
+    }
+
+    def tag_name(toks, idx, author_names=nil)
+      name = toks[idx].node.parent.name # node is always a text node; the informative one is the parent
+      NODE_TYPES_BY_NAME[name.downcase] || 'other'
+    end
+
+    def location_in_node(toks, idx, author_names=nil)
+      ((toks[idx].idx_in_node.to_f / toks[idx].node_token_count) * 10).round
+    end
+
+    def part_of_speech(toks, idx, author_names=nil)
+      toks[idx].part_of_speech
+    end
+
+  private
+
+    def possible_issue_with_str(toks, idx)
+      return unless toks[idx]
+
+      possible_issue_str(toks, idx) ||
+        (possible_issue_str(toks, idx-1) && toks[idx].raw =~ /^\d+$/)
+    end
+
+    def possible_issue_str(toks, idx)
+      if toks[idx]
+        if toks[idx].raw =~ /^(no)|(issue)?\.?\d+.?$/i
+          return true
+        elsif toks[idx+1]
+          return ['no','issue'].include?(toks[idx].lcnp) && toks[idx+1].raw =~ /^\d+$/
+        end
+      end
+    end
+
+    def possible_vol_with_str(toks, idx)
+      return unless toks[idx]
+
+      possible_vol_str(toks, idx) ||
+        (possible_vol_str(toks, idx-1) && (toks[idx].raw =~ /^\d+$/ || toks[idx].raw == ',')) ||
+        (possible_vol_str(toks, idx-2) && toks[idx-1].raw =~ /^\d+$/ && toks[idx].raw == ',')
+    end
+
+    def possible_vol_str(toks, idx)
+      if toks[idx]
+        if toks[idx].raw =~ /^vol(ume)?\.?\d+.?$/i
+          return true
+        elsif toks[idx+1]
+          return ['vol','volume'].include?(toks[idx].lcnp) && toks[idx+1].raw =~ /^\d+$/
+        end
+      end
+    end
+
+    def possible_vol_with_parens(toks, idx)
+      if toks[idx] && toks[idx+3]
+        toks[idx].raw =~ /^\d+$/ && toks[idx+1].raw == '(' && toks[idx+2].raw =~ /^\d+$/ && toks[idx+3].raw == ')'
+      end
+    end
+
+    def possible_vol_with_colon(toks, idx)
+      if toks[idx] && toks[idx+1]
+        # case of <year>: something is common so make sure we exclude it
+        if toks[idx].np =~ /^\d{1,3}$/ && toks[idx+1].raw =~ /^:/
+          # at this point it's likely a volume, but exclude it if it's not followed by an apparent page or issue
+          toks[idx+1].np =~ /^\d+$/ || (toks[idx+1].raw == ':' && toks[idx+2] && toks[idx+2].np =~ /^\d+/)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/excite/version.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Excite
+
+  VERSION = '2.1.1'
+
+end

data/lib/excite.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+
+module Excite
+end
+
+require 'crfpp'
+
+require 'excite/array_helpers'
+require 'excite/citation'
+require 'excite/preprocessor'
+require 'excite/postprocessor'
+require 'excite/token_features'
+require 'excite/crfparser'

data/model/test/analysis.csv

@@ -0,0 +1,54 @@
+Results for model
+ branch: add_parts_of_speech
+ version: fe881b78325f48e511a2d015ba80edb55ba9a64c Rename module to Excite
+Test run on:,2013-03-06 15:15:20 -0800
+K-fold x-validation:,10
+Corpus size:,500
+
+truth\test,author,booktitle,date,editor,institution,journal,location,note,pages,publisher,tech,title,volume
+author,3894,6,0,7,0,0,0,3,0,0,0,1,0
+booktitle,0,2053,0,8,3,52,8,9,0,0,3,31,6
+date,1,0,1547,1,0,0,6,2,4,0,0,1,3
+editor,35,5,0,442,0,4,0,0,0,0,0,1,0
+institution,3,8,0,4,368,0,6,0,0,1,1,8,0
+journal,0,35,0,5,0,731,0,0,0,3,0,26,4
+location,0,21,2,0,14,0,479,0,0,0,0,3,3
+note,0,32,1,5,2,6,0,99,0,0,7,3,9
+pages,0,0,2,0,0,0,0,0,772,1,2,0,6
+publisher,0,10,0,0,11,2,6,0,0,271,0,4,0
+tech,0,0,0,0,2,0,4,0,4,0,199,30,5
+title,1,32,0,4,0,4,2,0,0,0,0,4215,2
+volume,0,3,1,0,0,3,0,0,7,0,3,0,609
+author,0.9956532856047047,0.001534134492457172,0.0,0.0017898235745333673,0.0,0.0,0.0,0.000767067246228586,0.0,0.0,0.0,0.00025568908207619537,0.0
+booktitle,0.0,0.9447768062586286,0.0,0.0036815462494247586,0.0013805798435342844,0.02393005062126093,0.0036815462494247586,0.0041417395306028535,0.0,0.0,0.0013805798435342844,0.014265991716520939,0.0027611596870685687
+date,0.0006389776357827476,0.0,0.9884984025559106,0.0006389776357827476,0.0,0.0,0.0038338658146964857,0.0012779552715654952,0.0025559105431309905,0.0,0.0,0.0006389776357827476,0.0019169329073482429
+editor,0.07186858316221766,0.01026694045174538,0.0,0.9075975359342916,0.0,0.008213552361396304,0.0,0.0,0.0,0.0,0.0,0.002053388090349076,0.0
+institution,0.007518796992481203,0.020050125313283207,0.0,0.010025062656641603,0.9223057644110275,0.0,0.015037593984962405,0.0,0.0,0.002506265664160401,0.002506265664160401,0.020050125313283207,0.0
+journal,0.0,0.043532338308457715,0.0,0.006218905472636816,0.0,0.9092039800995025,0.0,0.0,0.0,0.0037313432835820895,0.0,0.03233830845771144,0.004975124378109453
+location,0.0,0.040229885057471264,0.0038314176245210726,0.0,0.02681992337164751,0.0,0.9176245210727969,0.0,0.0,0.0,0.0,0.005747126436781609,0.005747126436781609
+note,0.0,0.1951219512195122,0.006097560975609756,0.03048780487804878,0.012195121951219513,0.036585365853658534,0.0,0.6036585365853658,0.0,0.0,0.042682926829268296,0.018292682926829267,0.054878048780487805
+pages,0.0,0.0,0.002554278416347382,0.0,0.0,0.0,0.0,0.0,0.9859514687100894,0.001277139208173691,0.002554278416347382,0.0,0.007662835249042145
+publisher,0.0,0.03289473684210526,0.0,0.0,0.03618421052631579,0.006578947368421052,0.019736842105263157,0.0,0.0,0.8914473684210527,0.0,0.013157894736842105,0.0
+tech,0.0,0.0,0.0,0.0,0.00819672131147541,0.0,0.01639344262295082,0.0,0.01639344262295082,0.0,0.8155737704918032,0.12295081967213115,0.020491803278688523
+title,0.00023474178403755868,0.007511737089201878,0.0,0.0009389671361502347,0.0,0.0009389671361502347,0.00046948356807511736,0.0,0.0,0.0,0.0,0.9894366197183099,0.00046948356807511736
+volume,0.0,0.004792332268370607,0.001597444089456869,0.0,0.0,0.004792332268370607,0.0,0.0,0.011182108626198083,0.0,0.004792332268370607,0.0,0.9728434504792333
+
+Label,Precision,Recall,F-measure
+author,0.9898322318251144,0.9956532856047047,0.9927342256214148
+booktitle,0.9310657596371882,0.9447768062586286,0.9378711740520785
+date,0.9961365099806826,0.9884984025559106,0.9923027581783195
+editor,0.9285714285714286,0.9075975359342916,0.9179646936656283
+institution,0.92,0.9223057644110275,0.9211514392991239
+journal,0.9114713216957606,0.9092039800995025,0.9103362391033625
+location,0.9373776908023483,0.9176245210727969,0.9273959341723136
+note,0.8761061946902655,0.6036585365853658,0.7148014440433211
+pages,0.9809402795425667,0.9859514687100894,0.9834394904458599
+publisher,0.9818840579710145,0.8914473684210527,0.9344827586206896
+tech,0.9255813953488372,0.8155737704918032,0.8671023965141613
+title,0.9750173490631506,0.9894366197183099,0.9821740650122335
+volume,0.9412673879443586,0.9728434504792333,0.9567949725058915
+
+Average accuracy by reference:,0.9621987290789997
+STD of Average accuracy by reference:,0.09112313872147432
+Perfect parses:,383,0.766
+Accuracy:, 0.9653367811845832

data/model/test/array_helpers.rb

@@ -0,0 +1,30 @@
+module ArrayHelpers
+
+  def sum
+    inject(0, :+)
+  end
+
+  def mean
+    (size > 0) ? sum.to_f / size : 0
+  end
+
+  def stddev
+    m = mean
+    devsum = inject( 0 ) { |ds,x| ds += (x - m)**2 }
+    (size > 0) ? (devsum.to_f / size) ** 0.5 : 0
+  end
+
+  def cov(other)
+    zip(other).map {|a,b| a*b }.mean - (mean * other.mean)
+  end
+
+  def pearson_r(other)
+    unless size == other.size
+      raise "Vectors must be of same length to calculate pearson_r"
+    end
+    devp = stddev * other.stddev
+    (devp > 0) ? cov(other) / devp : 0.0
+  end
+
+end
+

data/model/test/html-analysis.csv

@@ -0,0 +1,60 @@
+Results for model
+ branch: add_parts_of_speech
+ version: 19c853489cbc259793dd24d89e1e16c78222a14b Add possible_volume based on regex (not actually usable unfortunately)
+Test run on:,2013-03-06 14:52:26 -0800
+K-fold x-validation:,10
+Corpus size:,500
+
+truth\test,author,booktitle,bullet,date,editor,institution,journal,link,location,note,pages,publisher,title,volume,workid
+author,3503,8,4,2,32,0,2,3,0,7,2,0,72,0,0
+booktitle,3,1271,0,20,27,15,82,0,13,21,4,13,171,2,0
+bullet,3,1,31,1,0,0,1,0,0,3,0,0,1,2,0
+date,1,24,0,1274,5,0,10,0,4,8,5,8,3,8,0
+editor,35,19,0,0,1012,0,0,4,5,0,0,9,23,0,0
+institution,0,34,0,2,11,0,18,0,0,10,0,36,29,0,0
+journal,3,45,0,3,2,4,1130,1,0,8,0,11,62,2,0
+link,3,16,0,1,0,0,0,326,0,48,7,2,2,1,0
+location,9,13,0,3,4,0,5,2,441,7,0,34,9,0,0
+note,9,61,0,13,21,0,16,19,0,175,3,20,107,2,0
+pages,0,1,0,14,0,0,0,4,0,9,693,3,4,16,0
+publisher,6,31,0,4,16,0,15,1,12,13,2,532,14,0,0
+title,54,159,0,12,4,0,20,0,10,34,2,8,6129,0,0
+volume,0,6,0,18,0,0,8,0,3,2,7,0,0,861,0
+workid,0,0,0,7,0,0,0,2,4,7,8,0,3,4,64
+author,0.9636863823933975,0.002200825309491059,0.0011004126547455295,0.0005502063273727648,0.008803301237964236,0.0,0.0005502063273727648,0.0008253094910591472,0.0,0.0019257221458046766,0.0005502063273727648,0.0,0.019807427785419534,0.0,0.0
+booktitle,0.0018270401948842874,0.7740560292326432,0.0,0.012180267965895249,0.016443361753958587,0.009135200974421437,0.049939098660170524,0.0,0.007917174177831911,0.012789281364190013,0.00243605359317905,0.007917174177831911,0.10414129110840438,0.001218026796589525,0.0
+bullet,0.06976744186046512,0.023255813953488372,0.7209302325581395,0.023255813953488372,0.0,0.0,0.023255813953488372,0.0,0.0,0.06976744186046512,0.0,0.0,0.023255813953488372,0.046511627906976744,0.0
+date,0.0007407407407407407,0.017777777777777778,0.0,0.9437037037037037,0.003703703703703704,0.0,0.007407407407407408,0.0,0.002962962962962963,0.005925925925925926,0.003703703703703704,0.005925925925925926,0.0022222222222222222,0.005925925925925926,0.0
+editor,0.031616982836495035,0.017163504968383016,0.0,0.0,0.9141824751580849,0.0,0.0,0.0036133694670280035,0.004516711833785004,0.0,0.0,0.008130081300813009,0.02077687443541102,0.0,0.0
+institution,0.0,0.24285714285714285,0.0,0.014285714285714285,0.07857142857142857,0.0,0.12857142857142856,0.0,0.0,0.07142857142857142,0.0,0.2571428571428571,0.20714285714285716,0.0,0.0
+journal,0.0023603461841070024,0.03540519276160504,0.0,0.0023603461841070024,0.0015735641227380016,0.003147128245476003,0.8890637293469709,0.0007867820613690008,0.0,0.006294256490952006,0.0,0.00865460267505901,0.04878048780487805,0.0015735641227380016,0.0
+link,0.007389162561576354,0.03940886699507389,0.0,0.0024630541871921183,0.0,0.0,0.0,0.8029556650246306,0.0,0.11822660098522167,0.017241379310344827,0.0049261083743842365,0.0049261083743842365,0.0024630541871921183,0.0
+location,0.017077798861480076,0.024667931688804556,0.0,0.0056925996204933585,0.007590132827324478,0.0,0.009487666034155597,0.003795066413662239,0.8368121442125237,0.013282732447817837,0.0,0.06451612903225806,0.017077798861480076,0.0,0.0
+note,0.020179372197309416,0.1367713004484305,0.0,0.02914798206278027,0.04708520179372197,0.0,0.03587443946188341,0.042600896860986545,0.0,0.3923766816143498,0.006726457399103139,0.04484304932735426,0.2399103139013453,0.004484304932735426,0.0
+pages,0.0,0.0013440860215053765,0.0,0.01881720430107527,0.0,0.0,0.0,0.005376344086021506,0.0,0.012096774193548387,0.9314516129032258,0.004032258064516129,0.005376344086021506,0.021505376344086023,0.0
+publisher,0.009287925696594427,0.047987616099071206,0.0,0.006191950464396285,0.02476780185758514,0.0,0.02321981424148607,0.0015479876160990713,0.018575851393188854,0.020123839009287926,0.0030959752321981426,0.8235294117647058,0.021671826625386997,0.0,0.0
+title,0.008395522388059701,0.024720149253731342,0.0,0.0018656716417910447,0.0006218905472636816,0.0,0.003109452736318408,0.0,0.001554726368159204,0.005286069651741294,0.0003109452736318408,0.0012437810945273632,0.9528917910447762,0.0,0.0
+volume,0.0,0.0066298342541436465,0.0,0.019889502762430938,0.0,0.0,0.008839779005524863,0.0,0.0033149171270718232,0.0022099447513812156,0.0077348066298342545,0.0,0.0,0.9513812154696133,0.0
+workid,0.0,0.0,0.0,0.0707070707070707,0.0,0.0,0.0,0.020202020202020204,0.04040404040404041,0.0707070707070707,0.08080808080808081,0.0,0.030303030303030304,0.04040404040404041,0.6464646464646465
+
+Label,Precision,Recall,F-measure
+author,0.9652796913750344,0.9636863823933975,0.9644823788546256
+booktitle,0.7525162818235642,0.7740560292326432,0.7631341939357552
+bullet,0.8857142857142857,0.7209302325581395,0.7948717948717948
+date,0.9272197962154294,0.9437037037037037,0.9353891336270191
+editor,0.892416225749559,0.9141824751580849,0.9031682284694332
+institution,0.0,0.0,NaN
+journal,0.864575363427697,0.8890637293469709,0.8766485647788983
+link,0.9005524861878453,0.8029556650246306,0.8489583333333334
+location,0.8963414634146342,0.8368121442125237,0.8655544651619235
+note,0.4971590909090909,0.3923766816143498,0.43859649122807015
+pages,0.9454297407912687,0.9314516129032258,0.938388625592417
+publisher,0.7869822485207101,0.8235294117647058,0.8048411497730711
+title,0.9245738422084779,0.9528917910447762,0.9385192557997091
+volume,0.9587973273942093,0.9513812154696133,0.9550748752079867
+workid,1.0,0.6464646464646465,0.7852760736196319
+
+Average accuracy by reference:,0.8980503362135145
+STD of Average accuracy by reference:,0.150249693017486
+Perfect parses:,239,0.478
+Accuracy:, 0.899396689527149