opener-coreference-base 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 91b868225cc8ddadc6790ae9ff424de0d5e13591
+  data.tar.gz: 7d2944067210631815353e6e9520521309996a23
+SHA512:
+  metadata.gz: bb964bf0f8ec83884b1876785285c42827422aee9e65a917518d37ef96386b8b09c33e47d1fb106b2f1558b19a33d1f0cb8c8bc383a68b59e8d3e86e4ae8997a
+  data.tar.gz: e4585ebcf72ec6992853893daf37103b1960364ba0952e97ac3fb63b816a1c2a0c75ba0766f50a351cc8bb92e85d36b5b8df8b3e0c829cef54ec7e5e61fa3b87

data/README.md

@@ -0,0 +1,97 @@
+[![Build Status](https://drone.io/github.com/opener-project/coreference-base/status.png)](https://drone.io/github.com/opener-project/coreference-base/latest)
+
+# Coreference
+
+This Gem provides coreference resolution for various languages such as English
+and Spanish.
+
+The CorefGraph-en module provides an implementation of the Multi-Sieve Pass
+system for for Coreference Resolution system originally proposed by the
+Stanford NLP Group (Raghunathan et al., 2010; Lee et al., 2011) and (Lee et
+al., 2013).  This system proposes a number of deterministic passes, ranging
+from high precision to higher recall, each dealing with a different manner in
+which coreference manifests itself in running text.
+
+Although more sieves are available, in order to facilitate the integration of
+the coreference system for the 6 languages of OpeNER we have included here 4
+sieves: Exact String Matching, Precise Constructs, Strict Head Match and
+Pronoun Match (the sieve nomenclature follows Lee et al (2013)). Furthermore,
+as it has been reported, this sieves are responsible for most of the
+performance in the Stanford system.
+
+The implementation is a result of a collaboration between the IXA NLP
+(http://ixa.si.ehu.es) and LinguaMedia Groups (http://linguamedia.deusto.es).
+
+## Requirements
+
+* Ruby 1.9.2 or newer
+* Python 2.7 or newer
+* Pip 1.3.1 or newer
+
+## Installation
+
+Installing as a regular Gem:
+
+    gem install opener-coreference-base
+
+Using Bundler:
+
+    gem 'opener-coreference-base',
+      :git    => 'git@github.com:opener-project/coreference-base.git',
+      :branch => 'master'
+
+Using specific install:
+
+    gem install specific_install
+    gem specific_install opener-coreference-base \
+       -l https://github.com/opener-project/coreference-base.git
+
+## Usage
+
+To run the program execute:
+
+    coreference-base -l (de|en|es|fr|it|nl) -i input.kaf 
+
+Corefgraph will output KAF via standard output with the <coreference> clusters
+added to the KAF input received. Note that for the full functionality of
+CorefGraph you will need to provide the <constituents> elements with the heads of (at
+least) the Noun Phrases marked, as it can be seen in the treebank input
+examples in the resource/examples directory. If you do not provide heads, only
+Exact String Match will work properly, whereas Precise Constructs, Strict Head
+Match and Pronoun Match will not.
+
+For a full explanation of how the Multi Sieve Pass system works see
+documentation in resources/doc.
+
+# Adapting CorefGraph-en to your language
+
+There are a number of changes needed to be made to make CorefGraph works for
+other languages. Although we have try to keep the language dependent features
+to a minimum, you will still need to create some dictionaries for your own
+language and make some very minor changes in the code. Here is the list of very
+file in the Corefgraph module that needs to be changed.  Every change except
+one (see below) to be done in the **$project/core/corefgraph/resources** directory:
+
+* dictionaries/$lang\_determiners.py
+* dictionaries/$lang\_pronouns.py
+* dictionaries/$lang\_verbs.py
+* dictionaries/$lang\_stopwords.py
+* dictionaries/$lang\_temporals.py
+
+* tagset/$TAGSETNAME\_pos.py
+* tagset/$TAGSETNAME\_constituent.py
+
+* files/animate/$lang.animate.txt
+* files/animate/$lang.inanimate.txt
+
+* files/demonym/$lang.txt
+
+* files/gender/$lang.male.unigrams.txt
+* files/gender/$lang.female.unigrams.txt
+* files/gender/$lang.neutral.unigrams.txt
+* files/gender/$lang.namegender.combine.txt
+* files/gender/$lang.gender.data
+
+* files/number/$lang.plural.unigrams.txt
+* files/number/$lang.singular.unigrams.txt
+

data/bin/coreference-de

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'de')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/bin/coreference-en

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'en')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/bin/coreference-es

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'es')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/bin/coreference-fr

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'fr')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/bin/coreference-it

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'it')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/bin/coreference-nl

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/opener/coreferences/base'
+
+kernel = Opener::Coreferences::Base.new(:args => ARGV, :language => 'nl')
+input  = STDIN.tty? ? nil : STDIN.read
+
+kernel.run!(input)

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: VUKafParserPy
+Version: 1.0
+Summary: Library in python to parse kaf files
+Home-page: UNKNOWN
+Author: Ruben Izquierdo
+Author-email: r.izquierdobevia@vu.nl
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+VUKafParserPy/KafDataObjectsMod.py
+VUKafParserPy/KafParserMod.py
+VUKafParserPy/__init__.py
+VUKafParserPy.egg-info/PKG-INFO
+VUKafParserPy.egg-info/SOURCES.txt
+VUKafParserPy.egg-info/dependency_links.txt
+VUKafParserPy.egg-info/top_level.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt

@@ -0,0 +1,11 @@
+../VUKafParserPy/KafParserMod.py
+../VUKafParserPy/__init__.py
+../VUKafParserPy/KafDataObjectsMod.py
+../VUKafParserPy/KafParserMod.pyc
+../VUKafParserPy/__init__.pyc
+../VUKafParserPy/KafDataObjectsMod.pyc
+./
+top_level.txt
+SOURCES.txt
+PKG-INFO
+dependency_links.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+VUKafParserPy

data/core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py

@@ -0,0 +1,165 @@
+class KafTermSentiment:
+  def __init__(self):
+    self.resource=None
+    self.polarity=None
+    self.strength=None
+    self.subjectivity=None
+    
+  def simpleInit(self,r,p,st,su,sm=None):
+    self.resource=r
+    self.polarity=p
+    self.strength=st
+    self.subjectivity=su
+    self.sentiment_modifier = sm
+    
+  def getPolarity(self):
+    return self.polarity
+  
+  def getSentimentModifier(self):
+    return self.sentiment_modifier
+    
+    
+class KafToken:
+  def __init__(self,wid, value, sent=None, para=None):
+    self.token_id = wid
+    self.value = value
+    self.sent = sent
+    self.para = para
+  
+
+class KafOpinionExpression:
+  def __init__(self,polarity,strength,targets):
+    self.polarity = polarity
+    self.strength = strength
+    self.targets = targets
+    
+  def __str__(self):
+    return 'Op_exp==> pol:'+self.polarity+' Str:'+self.strength+' ids:'+'-'.join(self.targets)
+    
+class KafOpinion:
+  def __init__(self,id,holders, targets, opi_exp):
+    self.id = id
+    self.holders = holders
+    self.targets = targets
+    self.opi_exp = opi_exp
+    
+  def __str__(self):
+    c='Opinion id'+self.id+'\n'
+    c+='  Holders: '+'-'.join(self.holders)+'\n'
+    c+='  Targets: '+'-'.join(self.targets)+'\n'
+    c+=str(self.opi_exp)
+    return c
+    
+    
+
+class KafSingleProperty:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+
+class KafSingleEntity:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+class KafTerm:
+  def __init__(self):
+    self.tid = None
+    self.lemma = None
+    self.pos = None
+    self.morphofeat = None
+    self.sentiment = None
+    self.list_span_id = []
+    
+  def get_morphofeat(self):
+    return self.morphofeat
+    
+  def set_list_span_id(self, L):
+    self.list_span_id = L
+    
+  def get_list_span(self):
+    return self.list_span_id
+    
+  def get_polarity(self):
+    if self.sentiment != None:
+      return self.sentiment.getPolarity()
+    else:
+      return None
+      
+  def get_sentiment_modifier(self):
+    if self.sentiment != None:
+      return self.sentiment.getSentimentModifier()
+    else:
+      return None
+    
+    
+  def setSentiment(self,my_sent):
+    self.sentiment = my_sent
+    
+  def getSentiment(self):
+    return self.sentiment
+    
+  def getLemma(self): 
+      return self.lemma
+    
+  def setLemma(self,lemma):
+      self.lemma = lemma
+    
+  def getPos(self):
+      return self.pos
+    
+  def setPos(self,pos):
+      self.pos = pos
+      
+  def getId(self):
+      return self.tid
+      
+  def setId(self,id):
+      self.tid = id
+
+  def getShortPos(self):
+    if self.pos==None:
+      return None
+    auxpos=self.pos.lower()[0]
+    if auxpos == 'g': auxpos='a'
+    elif auxpos == 'a': auxpos='r'
+    return auxpos
+    
+  def __str__(self):
+    if self.tid and self.lemma and self.pos:
+        return self.tid+'\n\t'+self.lemma.encode('utf-8')+'\n\t'+self.pos
+    else:
+        return 'None'
+      
+    
+    
+    
+    

data/core/site-packages/pre_build/VUKafParserPy/KafParserMod.py

@@ -0,0 +1,439 @@
+########################################################################
+# 14 Jan 2013: added function add_attrs_to_layer
+########################################################################
+
+###################
+# List of changes #
+###################
+# 14 Jan 2013: added function add_attrs_to_layer
+# 27 Feb 2013: added code for comply with DTD
+# 18 Jun 2013: getSingleProperties adapted to the structure KAF/features/properties/property/references/span/target
+# 18 Jun 2013: funcion add_property created for adding the properties to the KAF
+
+
+from lxml import etree
+from KafDataObjectsMod import *
+import time
+
+class KafParser:
+  def __init__(self,filename=None):
+	self.tree=None
+	self.__pathForToken={}
+	self.__term_ids_for_token_id = None
+    
+	if filename:
+		#self.tree = etree.parse(filename,etree.XMLParser(remove_blank_text=True))
+		self.tree = etree.parse(filename,etree.XMLParser(remove_blank_text=True, strip_cdata=False))
+		## Do the text tokenization
+		self.__textTokenization()
+	else:
+		root = etree.Element('KAF')
+		root.set('version','v1.opener')
+		root.set('{http://www.w3.org/XML/1998/namespace}lang','en')
+		self.tree = etree.ElementTree(element=root)
+		
+  def __textTokenization(self):
+	for wf in self.tree.findall('text/wf'):
+	  wid = wf.get('wid')
+	  self.__pathForToken[wid] = self.tree.getpath(wf)
+	 
+  
+  def getToken(self,tid):
+      if tid in self.__pathForToken:
+          path = self.__pathForToken[tid]
+          return self.tree.xpath(self.__pathForToken[tid])[0]
+      return None
+
+  
+  def getLanguage(self):
+	  lang = self.tree.getroot().get('{http://www.w3.org/XML/1998/namespace}lang','nl')
+	  return lang
+	  
+  ## Return a list of (sentence_id, TOKENS) where tokens is a list of (token_id,token)
+  ## [(s_id1, T1), (sent_id2, T2)....]
+  ## T1 --> [(tokenid, token), (tokenid2,token2)....]
+  def get_tokens_in_sentences(self):
+      sents = []
+      current = []
+      previous_sent = None
+      for element in self.tree.findall('text/wf'):
+          w_id = element.get('wid')
+          s_id = element.get('sent')
+          word = element.text
+          
+          if previous_sent is not None and s_id != previous_sent:
+              sents.append((previous_sent,current))
+              current = []
+          current.append((w_id,word))
+          previous_sent = s_id
+      ####
+      sents.append((s_id,current)) 
+      return sents
+  
+  def get_term_ids_for_token_id(self,tok_id):
+      if self.__term_ids_for_token_id is None:
+          self.__term_ids_for_token_id = {}
+          for element in self.tree.findall('terms/term'):
+              term_id = element.get('tid')
+              for target in element.findall('span/target'):
+                  token_id = target.get('id')
+                  if token_id not in self.__term_ids_for_token_id:
+                      self.__term_ids_for_token_id[token_id] = [term_id]
+                  else:
+                      self.__term_ids_for_token_id[token_id].append(term_id)
+      return self.__term_ids_for_token_id.get(tok_id,[])
+  
+          
+      
+  def getTokens(self):
+	for element in self.tree.findall('text/wf'):
+	  w_id = element.get('wid')
+	  s_id = element.get('sent','0')
+	  word = element.text
+	  yield (word, s_id, w_id)
+	
+    
+    
+  def getTerms(self):
+	 if self.tree:
+	   for element in self.tree.findall('terms/term'):
+		   kafTermObj = KafTerm()
+		   kafTermObj.setId(element.get('tid'))
+		   kafTermObj.setLemma(element.get('lemma'))
+		   kafTermObj.setPos(element.get('pos'))
+		   kafTermObj.morphofeat = element.get('morphofeat')
+		   		   
+		   ## Parsing sentiment
+		   sentiment = element.find('sentiment')
+		   if sentiment is not None:
+			 resource = sentiment.get('resource','')
+			 polarity = sentiment.get('polarity',None)
+			 strength = sentiment.get('strength','')
+			 subjectivity = sentiment.get('subjectivity','')
+			 sentiment_modifier = sentiment.get('sentiment_modifier')
+			 
+			 my_sent = KafTermSentiment()
+			 my_sent.simpleInit(resource,polarity,strength,subjectivity,sentiment_modifier)
+			 kafTermObj.setSentiment(my_sent)
+		  
+		   ## Parsing the span
+		   span = element.find('span')
+		   if span is not None:
+			list_ids = [target.get('id') for target in span.findall('target')]
+			kafTermObj.set_list_span_id(list_ids)
+		
+			 
+		   yield kafTermObj
+	 else:
+	   return
+	  
+	  
+  def getSentimentTriples(self):
+	data = []
+	if self.tree:
+	  for term_element in self.tree.findall('terms/term'):
+		lemma = term_element.get('lemma')
+		polarity = None
+		sentiment_modifier = None
+		
+		sentiment_element = term_element.find('sentiment')
+		if sentiment_element is not None:
+			polarity = sentiment_element.get('polarity',None)
+			sentiment_modifier = sentiment_element.get('sentiment_modifier')
+		data.append( (lemma,polarity,sentiment_modifier))
+	return data
+	  
+   
+	  
+  def addPolarityToTerm(self,termid,my_sentiment_attribs,polarity_pos=None):
+	if self.tree:
+	  for element in self.tree.find('terms'):
+		if element.get('tid','')==termid:
+		  
+		  #In case there is no pos info, we use the polarityPos
+		  if not element.get('pos') and polarity_pos is not None:
+			element.set('pos',polarity_pos)
+		  sentEle = etree.Element('sentiment',attrib=my_sentiment_attribs)
+		  element.append(sentEle)  
+	  
+  def saveToFile(self,filename,myencoding='UTF-8'):
+	if self.tree:
+	  self.tree.write(filename,encoding=myencoding,pretty_print=True,xml_declaration=True)
+	  
+  
+  def addLinguisticProcessor(self,name,version, layer, time_stamp=True):
+	aux = self.tree.findall('kafHeader')
+	if len(aux)!=0:
+	  kaf_header = aux[0]
+	else:
+	  kaf_header = etree.Element('kafHeader')
+	  self.tree.getroot().insert(0,kaf_header)
+
+        aux2= kaf_header.findall('linguisticProcessors')
+        if len(aux2) == 0:
+          new_lp = etree.Element('linguisticProcessors')
+          new_lp.set('layer',layer)
+          kaf_header.append(new_lp)
+          
+	## Check if there is already element for the layer
+	my_lp_ele = None
+	
+	for element in kaf_header.findall('linguisticProcessors'):
+	  if element.get('layer','')==layer:
+		my_lp_ele = element
+		break
+	  
+	if time_stamp:  
+	  my_time = time.strftime('%Y-%m-%dT%H:%M:%S%Z')
+	else:
+	  my_time = '*'
+	  
+	my_lp = etree.Element('lp')
+	my_lp.set('timestamp',my_time)
+	my_lp.set('version',version)
+	my_lp.set('name',name)
+
+	if my_lp_ele is not None: #Already an element for linguisticProcessor with the layer
+	  my_lp_ele.append(my_lp)
+	else:
+	  # Create a new element for the LP layer
+	  my_lp_ele = etree.Element('linguisticProcessors')
+	  my_lp_ele.set('layer',layer)
+	  my_lp_ele.append(my_lp)
+	  #my_lp_ele.tail=my_lp_ele.text='\n'
+	  ## Should be inserted after the last linguisticProcessor element (stored in variable element)
+	  idx = kaf_header.index(element)
+	  kaf_header.insert(idx+1,my_lp_ele)
+		
+	  
+  def addLayer(self,type,element,first_char_id=None):
+	if first_char_id is None:
+		first_char_id = type[0]
+		
+	## Check if there is already layer for the type
+	layer_element = self.tree.find(type)
+	
+	if layer_element is None:
+	  layer_element = etree.Element(type)
+	  self.tree.getroot().append(layer_element)
+	  ## The id is going to be the first one
+	  new_id = first_char_id+'1'
+	else:
+	  ## We need to know how many elements there are in the layer
+	  current_n = len(layer_element.getchildren())
+	  new_id = first_char_id+''+str(current_n+1)
+	  
+	  
+	## In this point layer_element points to the correct element, existing or created
+	
+	element.set(first_char_id+'id',new_id)
+	layer_element.append(element)
+	return new_id
+	
+  def addElementToLayer(self,layer, element,first_char_id=None):
+	return self.addLayer(layer,element,first_char_id)
+
+  def add_attrs_to_layer(self,layer,attrs):
+	layer_element = self.tree.find(layer)
+	if layer_element is not None:
+	  for att, val in attrs.items():
+		layer_element.set(att,val)
+		
+
+  def addAttributeToElement(self,path,str_id, id, attribute, value,sub_path=None):
+	  for element in self.tree.findall(path):
+		if id is not None and element.get(str_id,None) == id:
+		  if sub_path is not None:
+			elements = element.findall(sub_path)
+			if len(elements)!=0: element = elements[0]
+		  element.set(attribute,value)
+		  return
+
+  
+  ## This works with the original definition of the property layer
+  ## KAF -> properties -> property* -> span* -> target*
+  def getSingleProperties_old(self):
+	  for element in self.tree.findall('properties/property'):
+		  my_id = element.get('pid')
+		  my_type = element.get('type')
+		  ref = element.find('references')
+		  if ref is not None:
+			element = ref
+		  for span_element in element.findall('span'):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleProperty(my_id,my_type,target_ids)
+			  yield my_prop
+			   
+  ## 18-June-2013
+  def getSingleProperties(self):
+	  for property in self.tree.findall('features/properties/property'):
+		  my_id = property.get('pid')
+		  if my_id is None:
+			  my_id = property.get('fpid')
+		  my_type = property.get('lemma')
+		  for span_element in property.findall('references/span'):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleProperty(my_id,my_type,target_ids)
+			  yield my_prop
+
+  # This function adds a new property of the type given with the list of ids given
+  # my_type -> 'sleeping comfort'	list_ids = ['id1','id2']
+  # It creates the features/properties layers in case 
+  # Agglomerates all the properties for the same TYPE under the same property element
+  # It calculates automatically the number for the identifier depending on the number
+  # of properties existing
+  def add_property(self,my_type,list_ids,comment=None):
+      
+      #Looking for feature layer or creating it
+      feature_layer = self.tree.find('features')
+      if feature_layer is None:
+          feature_layer = etree.Element('features')
+          self.tree.getroot().append(feature_layer)
+          
+      #Looking for properties layer
+      properties_layer = feature_layer.find('properties')
+      if properties_layer is None:
+          properties_layer = etree.Element('properties')
+          feature_layer.append(properties_layer)
+          
+      num_props = 0
+      property_layer = None
+      for property in properties_layer.findall('property'):
+          num_props += 1
+          prop_type = property.get('lemma')
+          if prop_type == my_type:
+              property_layer = property
+              break
+          
+      if property_layer is None:  # There is no any property for that type, let's create one
+          property_layer = etree.Element('property')
+          property_layer.set('pid','p'+str(num_props+1))
+          property_layer.set('lemma',my_type)    
+          properties_layer.append(property_layer)
+          
+          
+      references = property_layer.find('references')
+      if references is None:
+          references = etree.Element('references')
+          property_layer.append(references)
+      ## Create the new span
+      if comment is not None:
+        references.append(etree.Comment(comment))
+      span = etree.Element('span')
+      references.append(span)
+      for my_id in list_ids:
+           span.append(etree.Element('target',attrib={'id':my_id}))
+       
+   
+			  
+  
+  def getSingleEntities(self):
+	  for element in self.tree.findall('entities/entity'):
+		  my_id = element.get('eid')
+		  my_type = element.get('type')
+		  my_path_to_span = None
+		  ref = element.find('references')
+		  if ref is not None:
+                      my_path_to_span = 'references/span'
+                  else:
+                      my_path_to_span = 'span'
+
+		  for span_element in element.findall(my_path_to_span):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleEntity(my_id,my_type,target_ids)
+			  yield my_prop
+
+
+  def getOpinions(self):
+	for element in self.tree.findall('opinions/opinion'):
+	  my_id = element.get('oid')
+	  
+	  tar_ids_hol = []
+	  tar_ids_tar = []
+	  polarity = strenght = ''
+	  tar_ids_exp = []
+	  
+	  #Holder
+	  opi_hol_eles = element.findall('opinion_holder')
+	  if len(opi_hol_eles)!=0:
+		  opi_hol_ele = opi_hol_eles[0]
+		  tar_ids_hol = [t_ele.get('id') for t_ele in opi_hol_ele.findall('span/target')]
+	  
+	  #Target
+	  opi_tar_eles = element.findall('opinion_target')
+	  if len(opi_tar_eles) != 0:
+		opi_tar_ele = opi_tar_eles[0]
+		tar_ids_tar = [t_ele.get('id') for t_ele in opi_tar_ele.findall('span/target')]
+		
+	  ## Opinion expression
+	  opi_exp_eles = element.findall('opinion_expression')
+	  if len(opi_exp_eles) != 0:
+		  opi_exp_ele = opi_exp_eles[0]
+		  polarity = opi_exp_ele.get('polarity','')
+		  strength = opi_exp_ele.get('strength','')
+		  tar_ids_exp = [t_ele.get('id') for t_ele in opi_exp_ele.findall('span/target')]
+
+	  yield KafOpinion(my_id,tar_ids_hol, tar_ids_tar, KafOpinionExpression(polarity, strength,tar_ids_exp))
+
+
+  
+  def remove_opinion_layer(self):
+      opinion_layer = self.tree.find('opinions')
+      if opinion_layer is not None:
+          self.tree.getroot().remove(opinion_layer)
+  
+  ## This function add an opinion to the opinion layer, creating it if does not exist
+  ## The id is calculated automatically according to the number of elements and ensring there is no repetition
+  def add_opinion(self,hol_ids,tar_ids,polarity,strength,exp_ids):
+      
+      #Looking for opinion layer or creating it
+      opinion_layer = self.tree.find('opinions')
+      if opinion_layer is None:
+          opinion_layer = etree.Element('opinions')
+          self.tree.getroot().append(opinion_layer)
+          
+      ## Generating unique id
+      list_of_oids = [opi.get('oid') for opi in opinion_layer]
+      
+      n = 1
+      while True:
+          my_id = 'o'+str(n)
+          if my_id not in list_of_oids:
+              break
+          n += 1
+      #####
+          
+      op_ele = etree.Element('opinion')
+      opinion_layer.append(op_ele)
+      op_ele.set('oid',my_id)
+
+      ## Holder
+      op_hol = etree.Element('opinion_holder')
+      op_ele.append(op_hol)
+      span_op_hol = etree.Element('span')
+      op_hol.append(span_op_hol)
+      for my_id in hol_ids:
+        span_op_hol.append(etree.Element('target',attrib={'id':my_id}))
+
+      ## TARGET
+      op_tar = etree.Element('opinion_target')
+      op_ele.append(op_tar)
+      span_op_tar = etree.Element('span')
+      op_tar.append(span_op_tar)
+      for my_id in tar_ids:
+        span_op_tar.append(etree.Element('target',attrib={'id':my_id}))
+
+      ## Expression
+    
+      op_exp = etree.Element('opinion_expression',attrib={'polarity':polarity,
+                                                       'strength':str(strength)})
+      op_ele.append(op_exp)
+      span_exp = etree.Element('span')
+      op_exp.append(span_exp)
+      for my_id in exp_ids:
+        span_exp.append(etree.Element('target',attrib={'id':my_id}))
+
+
+
+	
+	

data/core/site-packages/pre_build/VUKafParserPy/__init__.py

@@ -0,0 +1,7 @@
+## version = 0.2
+## Added timestamp to function addLinguisitcProcessor
+## 24-april-2013 --> getSingleEntieies and getSingleProperties reads both entities/props in format
+## entities -> entity -> span -> target and entities -> entity -> references -> span
+####
+
+from KafParserMod import KafParser

data/ext/hack/Rakefile

@@ -0,0 +1,16 @@
+require 'rake'
+require_relative 'support'
+
+desc 'Verifies the requirements'
+task :requirements do
+  verify_requirements
+  require_executable("make")
+end
+
+task :compile => :requirements
+
+task :default => [:compile] do
+  path = File.join(PYTHON_SITE_PACKAGES, 'pre_install')
+
+  pip_install(PRE_INSTALL_REQUIREMENTS, path)
+end

data/ext/hack/support.rb

@@ -0,0 +1,38 @@
+require 'opener/build-tools'
+
+include Opener::BuildTools::Requirements
+include Opener::BuildTools::Python
+include Opener::BuildTools::Files
+
+# Directory where packages will be installed to.
+PYTHON_SITE_PACKAGES = File.expand_path(
+  '../../../core/site-packages',
+  __FILE__
+)
+
+# Directory containing the temporary files.
+TMP_DIRECTORY = File.expand_path('../../../tmp', __FILE__)
+
+# Path to the pip requirements file used to install requirements before
+# packaging the Gem.
+PRE_BUILD_REQUIREMENTS = File.expand_path(
+  '../../../pre_build_requirements.txt',
+  __FILE__
+)
+
+# Path to the pip requirements file used to install requirements upon Gem
+# installation.
+PRE_INSTALL_REQUIREMENTS = File.expand_path(
+  '../../../pre_install_requirements.txt',
+  __FILE__
+)
+
+##
+# Verifies the requirements to install thi Gem.
+#
+def verify_requirements
+  require_executable('python')
+  require_version('python', python_version, '2.6.0')
+  require_executable('pip')
+  require_version('pip', pip_version, '1.3.1')
+end

data/lib/opener/coreferences/base.rb

@@ -0,0 +1,122 @@
+require 'open3'
+require 'nokogiri'
+
+require_relative 'base/version'
+
+module Opener
+  module Coreferences
+    ##
+    # Coreference class for various languages such as English and Spanish.
+    #
+    # @!attribute [r] args
+    #  @return [Array]
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    class Base
+      attr_reader :args, :options
+
+      ##
+      # Returns the default language to use.
+      #
+      # @return [String]
+      #
+      DEFAULT_LANGUAGE = 'en'.freeze
+
+      ##
+      # @param [Hash] options
+      #
+      # @option options [Array] :args The commandline arguments to pass to the
+      #  underlying Java code.
+      #
+      def initialize(options = {})
+        
+        @args    = options.delete(:args) || []
+        @options = options
+      end
+
+      ##
+      # Returns a String containing the command used to execute the kernel.
+      #
+      # @return [String]
+      #
+      def command
+        return "#{adjust_python_path} python -E -OO -m #{kernel} #{args.join(' ')}"
+      end
+
+      ##
+      # Runs the command and returns the output of STDOUT, STDERR and the
+      # process information.
+      #
+      # @param [String] input The input to process.
+      # @return [Array]
+      #
+      def run(input)
+        @args << ["--language #{language(input)}"]
+        Dir.chdir(core_dir) do
+          capture(input)
+        end
+      end
+
+      ##
+      # Runs the command and takes care of error handling/aborting based on the
+      # output.
+      #
+      # @see #run
+      #
+      def run!(input)
+        stdout, stderr, process = run(input)
+
+        if process.success?
+          puts stdout
+
+          STDERR.puts(stderr) unless stderr.empty?
+        else
+          abort stderr
+        end
+      end
+
+      protected
+      ##
+      # @return [String]
+      #
+      def adjust_python_path
+        site_packages =  File.join(core_dir, 'site-packages')
+        "env PYTHONPATH=#{site_packages}:$PYTHONPATH"
+      end
+      
+      def capture(input)
+        Open3.popen3(*command.split(" ")) {|i, o, e, t|
+          out_reader = Thread.new { o.read }
+          err_reader = Thread.new { e.read }
+          i.write input
+          i.close
+          [out_reader.value, err_reader.value, t.value]
+        }
+      end
+
+      ##
+      # @return [String]
+      #
+      def core_dir
+        return File.expand_path('../../../../core', __FILE__)
+      end
+
+      ##
+      # @return [String]
+      #
+      def kernel
+        return 'corefgraph.process.file'
+      end
+      
+      ##
+      # @return the language from the KAF
+      #
+      def language(input)
+        document = Nokogiri::XML(input)
+        language = document.at('KAF').attr('xml:lang')
+        return language
+      end
+    end # Base
+  end # Coreferences
+end # Opener

data/lib/opener/coreferences/base/version.rb

@@ -0,0 +1,7 @@
+module Opener
+  module Coreferences
+    class Base
+      VERSION = '2.0.0'
+    end # Base
+  end # Coreferences
+end # Opener

data/opener-coreference-base.gemspec

@@ -0,0 +1,31 @@
+require File.expand_path('../lib/opener/coreferences/base/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = 'opener-coreference-base'
+  gem.version     = Opener::Coreferences::Base::VERSION
+  gem.authors     = ['development@olery.com']
+  gem.summary     = 'Coreference resolution for various languages.'
+  gem.description = gem.summary
+  gem.has_rdoc    = 'yard'
+  gem.extensions  = ['ext/hack/Rakefile']
+
+  gem.required_ruby_version = '>= 1.9.2'
+
+  gem.files = Dir.glob([
+    'core/site-packages/pre_build/**/*',
+    'ext/**/*',
+    'lib/**/*',
+    '*.gemspec',
+    '*_requirements.txt',
+    'README.md'
+  ]).select { |file| File.file?(file) }
+
+  gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
+
+  gem.add_dependency 'rake'
+  gem.add_dependency 'opener-build-tools', ['>= 0.2.7']
+  gem.add_dependency 'nokogiri'
+
+  gem.add_development_dependency 'cucumber'
+  gem.add_development_dependency 'rspec'
+end

data/pre_build_requirements.txt

@@ -0,0 +1 @@
+git+ssh://git@github.com/opener-project/VU-kaf-parser.git#egg=VUKafParserPy

data/pre_install_requirements.txt

@@ -0,0 +1,2 @@
+networkx
+pyyaml

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: opener-coreference-base
+version: !ruby/object:Gem::Version
+  version: 2.0.0
+platform: ruby
+authors:
+- development@olery.com
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: opener-build-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.7
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Coreference resolution for various languages.
+email: 
+executables:
+- coreference-es
+- coreference-nl
+- coreference-fr
+- coreference-de
+- coreference-it
+- coreference-en
+extensions:
+- ext/hack/Rakefile
+extra_rdoc_files: []
+files:
+- README.md
+- bin/coreference-de
+- bin/coreference-en
+- bin/coreference-es
+- bin/coreference-fr
+- bin/coreference-it
+- bin/coreference-nl
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt
+- core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt
+- core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py
+- core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.pyc
+- core/site-packages/pre_build/VUKafParserPy/KafParserMod.py
+- core/site-packages/pre_build/VUKafParserPy/KafParserMod.pyc
+- core/site-packages/pre_build/VUKafParserPy/__init__.py
+- core/site-packages/pre_build/VUKafParserPy/__init__.pyc
+- ext/hack/Rakefile
+- ext/hack/support.rb
+- lib/opener/coreferences/base.rb
+- lib/opener/coreferences/base/version.rb
+- opener-coreference-base.gemspec
+- pre_build_requirements.txt
+- pre_install_requirements.txt
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Coreference resolution for various languages.
+test_files: []
+has_rdoc: yard