opener-opinion-detector-base 2.0.0

data/core/site-packages/pre_build/VUA_pylib/lexicon/__init__.py

@@ -0,0 +1 @@
+from lexicon import *

data/core/site-packages/pre_build/VUA_pylib/lexicon/lexicon.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+import os
+import re
+from VUA_pylib.common import normalize_pos
+
+__this_folder__ = os.path.dirname(os.path.realpath(__file__))
+
+class MPQA_subjectivity_lexicon:
+    def __init__(self):
+        self.__filename=os.path.join(__this_folder__,'data','subjclueslen1-HLTEMNLP05.tff')
+        self.stemmed = {}
+        self.stemmed_anypos = {}
+        self.no_stemmed = {}
+        self.no_stemmed_anypos = {}
+
+        self.__load()
+        
+    def __load(self):
+        # Format of lines: 
+        # type=weaksubj len=1 word1=abandoned pos1=adj stemmed1=n priorpolarity=negative
+        fic = open(self.__filename)
+        for line in fic:
+            line=line.strip()+' '
+            this_type = re.findall('type=([^ ]+)', line)[0]
+            word = re.findall('word1=([^ ]+)', line)[0]
+            pos = re.findall('pos1=([^ ]+)', line)[0]
+            stemmed = re.findall('stemmed1=([^ ]+)', line)[0]
+            prior_polarity = re.findall('priorpolarity=([^ ]+)', line)[0]
+            pos = normalize_pos(pos)
+            if stemmed == 'y':
+                self.stemmed[(word,pos)] = (this_type,prior_polarity)
+                if True or pos == '*':  #anypos
+                    self.stemmed_anypos[word] = (this_type,prior_polarity)
+
+            elif stemmed == 'n':  
+                self.no_stemmed[(word,pos)] = (this_type,prior_polarity) 
+                if True or pos == '*':
+                    self.no_stemmed_anypos[word] = (this_type,prior_polarity)
+
+        fic.close()
+        
+    def get_type_and_polarity(self,word,pos=None):
+        res = None
+        if pos is not None:
+            pos = normalize_pos(pos)
+            
+            # Try no stemmed with the given pos
+            res = self.no_stemmed.get((word,pos))
+            
+            # Try stemmed with the given pos
+            if res is None:
+                res = self.stemmed.get((word,pos))
+            
+        # Try no stemmed with any pos    
+        if res is None:
+            res = self.no_stemmed_anypos.get(word)
+
+        # Try stemm with any pos
+        if res is None:
+            res = self.stemmed_anypos.get(word)
+            
+            
+        
+        return res
+            
+            
+if __name__ == '__main__':
+    o = MPQA_subjectivity_lexicon()
+    print o.get_type_and_polarity('abidance','adj')
+    
+    

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: VUKafParserPy
+Version: 1.0
+Summary: Library in python to parse kaf files
+Home-page: UNKNOWN
+Author: Ruben Izquierdo
+Author-email: r.izquierdobevia@vu.nl
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+VUKafParserPy/KafDataObjectsMod.py
+VUKafParserPy/KafParserMod.py
+VUKafParserPy/__init__.py
+VUKafParserPy.egg-info/PKG-INFO
+VUKafParserPy.egg-info/SOURCES.txt
+VUKafParserPy.egg-info/dependency_links.txt
+VUKafParserPy.egg-info/top_level.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/installed-files.txt

@@ -0,0 +1,11 @@
+../VUKafParserPy/KafParserMod.py
+../VUKafParserPy/__init__.py
+../VUKafParserPy/KafDataObjectsMod.py
+../VUKafParserPy/KafParserMod.pyc
+../VUKafParserPy/__init__.pyc
+../VUKafParserPy/KafDataObjectsMod.pyc
+./
+top_level.txt
+SOURCES.txt
+PKG-INFO
+dependency_links.txt

data/core/site-packages/pre_build/VUKafParserPy-1.0-py2.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+VUKafParserPy

data/core/site-packages/pre_build/VUKafParserPy/KafDataObjectsMod.py

@@ -0,0 +1,165 @@
+class KafTermSentiment:
+  def __init__(self):
+    self.resource=None
+    self.polarity=None
+    self.strength=None
+    self.subjectivity=None
+    
+  def simpleInit(self,r,p,st,su,sm=None):
+    self.resource=r
+    self.polarity=p
+    self.strength=st
+    self.subjectivity=su
+    self.sentiment_modifier = sm
+    
+  def getPolarity(self):
+    return self.polarity
+  
+  def getSentimentModifier(self):
+    return self.sentiment_modifier
+    
+    
+class KafToken:
+  def __init__(self,wid, value, sent=None, para=None):
+    self.token_id = wid
+    self.value = value
+    self.sent = sent
+    self.para = para
+  
+
+class KafOpinionExpression:
+  def __init__(self,polarity,strength,targets):
+    self.polarity = polarity
+    self.strength = strength
+    self.targets = targets
+    
+  def __str__(self):
+    return 'Op_exp==> pol:'+self.polarity+' Str:'+self.strength+' ids:'+'-'.join(self.targets)
+    
+class KafOpinion:
+  def __init__(self,id,holders, targets, opi_exp):
+    self.id = id
+    self.holders = holders
+    self.targets = targets
+    self.opi_exp = opi_exp
+    
+  def __str__(self):
+    c='Opinion id'+self.id+'\n'
+    c+='  Holders: '+'-'.join(self.holders)+'\n'
+    c+='  Targets: '+'-'.join(self.targets)+'\n'
+    c+=str(self.opi_exp)
+    return c
+    
+    
+
+class KafSingleProperty:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+
+class KafSingleEntity:
+  def __init__(self,id,type,targets):
+    self.id = id
+    self.type = type
+    self.targets = targets
+    
+  def get_id(self):
+    return self.id
+    
+  def get_type(self):
+    return self.type
+  
+  def get_span(self):
+    return self.targets
+  
+  def __str__(self):
+    return 'Id: '+self.id+' Type: '+self.type+' ids:'+' '.join(self.targets)
+
+class KafTerm:
+  def __init__(self):
+    self.tid = None
+    self.lemma = None
+    self.pos = None
+    self.morphofeat = None
+    self.sentiment = None
+    self.list_span_id = []
+    
+  def get_morphofeat(self):
+    return self.morphofeat
+    
+  def set_list_span_id(self, L):
+    self.list_span_id = L
+    
+  def get_list_span(self):
+    return self.list_span_id
+    
+  def get_polarity(self):
+    if self.sentiment != None:
+      return self.sentiment.getPolarity()
+    else:
+      return None
+      
+  def get_sentiment_modifier(self):
+    if self.sentiment != None:
+      return self.sentiment.getSentimentModifier()
+    else:
+      return None
+    
+    
+  def setSentiment(self,my_sent):
+    self.sentiment = my_sent
+    
+  def getSentiment(self):
+    return self.sentiment
+    
+  def getLemma(self): 
+      return self.lemma
+    
+  def setLemma(self,lemma):
+      self.lemma = lemma
+    
+  def getPos(self):
+      return self.pos
+    
+  def setPos(self,pos):
+      self.pos = pos
+      
+  def getId(self):
+      return self.tid
+      
+  def setId(self,id):
+      self.tid = id
+
+  def getShortPos(self):
+    if self.pos==None:
+      return None
+    auxpos=self.pos.lower()[0]
+    if auxpos == 'g': auxpos='a'
+    elif auxpos == 'a': auxpos='r'
+    return auxpos
+    
+  def __str__(self):
+    if self.tid and self.lemma and self.pos:
+        return self.tid+'\n\t'+self.lemma.encode('utf-8')+'\n\t'+self.pos
+    else:
+        return 'None'
+      
+    
+    
+    
+    

data/core/site-packages/pre_build/VUKafParserPy/KafParserMod.py

@@ -0,0 +1,439 @@
+########################################################################
+# 14 Jan 2013: added function add_attrs_to_layer
+########################################################################
+
+###################
+# List of changes #
+###################
+# 14 Jan 2013: added function add_attrs_to_layer
+# 27 Feb 2013: added code for comply with DTD
+# 18 Jun 2013: getSingleProperties adapted to the structure KAF/features/properties/property/references/span/target
+# 18 Jun 2013: funcion add_property created for adding the properties to the KAF
+
+
+from lxml import etree
+from KafDataObjectsMod import *
+import time
+
+class KafParser:
+  def __init__(self,filename=None):
+	self.tree=None
+	self.__pathForToken={}
+	self.__term_ids_for_token_id = None
+    
+	if filename:
+		#self.tree = etree.parse(filename,etree.XMLParser(remove_blank_text=True))
+		self.tree = etree.parse(filename,etree.XMLParser(remove_blank_text=True, strip_cdata=False))
+		## Do the text tokenization
+		self.__textTokenization()
+	else:
+		root = etree.Element('KAF')
+		root.set('version','v1.opener')
+		root.set('{http://www.w3.org/XML/1998/namespace}lang','en')
+		self.tree = etree.ElementTree(element=root)
+		
+  def __textTokenization(self):
+	for wf in self.tree.findall('text/wf'):
+	  wid = wf.get('wid')
+	  self.__pathForToken[wid] = self.tree.getpath(wf)
+	 
+  
+  def getToken(self,tid):
+      if tid in self.__pathForToken:
+          path = self.__pathForToken[tid]
+          return self.tree.xpath(self.__pathForToken[tid])[0]
+      return None
+
+  
+  def getLanguage(self):
+	  lang = self.tree.getroot().get('{http://www.w3.org/XML/1998/namespace}lang','nl')
+	  return lang
+	  
+  ## Return a list of (sentence_id, TOKENS) where tokens is a list of (token_id,token)
+  ## [(s_id1, T1), (sent_id2, T2)....]
+  ## T1 --> [(tokenid, token), (tokenid2,token2)....]
+  def get_tokens_in_sentences(self):
+      sents = []
+      current = []
+      previous_sent = None
+      for element in self.tree.findall('text/wf'):
+          w_id = element.get('wid')
+          s_id = element.get('sent')
+          word = element.text
+          
+          if previous_sent is not None and s_id != previous_sent:
+              sents.append((previous_sent,current))
+              current = []
+          current.append((w_id,word))
+          previous_sent = s_id
+      ####
+      sents.append((s_id,current)) 
+      return sents
+  
+  def get_term_ids_for_token_id(self,tok_id):
+      if self.__term_ids_for_token_id is None:
+          self.__term_ids_for_token_id = {}
+          for element in self.tree.findall('terms/term'):
+              term_id = element.get('tid')
+              for target in element.findall('span/target'):
+                  token_id = target.get('id')
+                  if token_id not in self.__term_ids_for_token_id:
+                      self.__term_ids_for_token_id[token_id] = [term_id]
+                  else:
+                      self.__term_ids_for_token_id[token_id].append(term_id)
+      return self.__term_ids_for_token_id.get(tok_id,[])
+  
+          
+      
+  def getTokens(self):
+	for element in self.tree.findall('text/wf'):
+	  w_id = element.get('wid')
+	  s_id = element.get('sent','0')
+	  word = element.text
+	  yield (word, s_id, w_id)
+	
+    
+    
+  def getTerms(self):
+	 if self.tree:
+	   for element in self.tree.findall('terms/term'):
+		   kafTermObj = KafTerm()
+		   kafTermObj.setId(element.get('tid'))
+		   kafTermObj.setLemma(element.get('lemma'))
+		   kafTermObj.setPos(element.get('pos'))
+		   kafTermObj.morphofeat = element.get('morphofeat')
+		   		   
+		   ## Parsing sentiment
+		   sentiment = element.find('sentiment')
+		   if sentiment is not None:
+			 resource = sentiment.get('resource','')
+			 polarity = sentiment.get('polarity',None)
+			 strength = sentiment.get('strength','')
+			 subjectivity = sentiment.get('subjectivity','')
+			 sentiment_modifier = sentiment.get('sentiment_modifier')
+			 
+			 my_sent = KafTermSentiment()
+			 my_sent.simpleInit(resource,polarity,strength,subjectivity,sentiment_modifier)
+			 kafTermObj.setSentiment(my_sent)
+		  
+		   ## Parsing the span
+		   span = element.find('span')
+		   if span is not None:
+			list_ids = [target.get('id') for target in span.findall('target')]
+			kafTermObj.set_list_span_id(list_ids)
+		
+			 
+		   yield kafTermObj
+	 else:
+	   return
+	  
+	  
+  def getSentimentTriples(self):
+	data = []
+	if self.tree:
+	  for term_element in self.tree.findall('terms/term'):
+		lemma = term_element.get('lemma')
+		polarity = None
+		sentiment_modifier = None
+		
+		sentiment_element = term_element.find('sentiment')
+		if sentiment_element is not None:
+			polarity = sentiment_element.get('polarity',None)
+			sentiment_modifier = sentiment_element.get('sentiment_modifier')
+		data.append( (lemma,polarity,sentiment_modifier))
+	return data
+	  
+   
+	  
+  def addPolarityToTerm(self,termid,my_sentiment_attribs,polarity_pos=None):
+	if self.tree:
+	  for element in self.tree.find('terms'):
+		if element.get('tid','')==termid:
+		  
+		  #In case there is no pos info, we use the polarityPos
+		  if not element.get('pos') and polarity_pos is not None:
+			element.set('pos',polarity_pos)
+		  sentEle = etree.Element('sentiment',attrib=my_sentiment_attribs)
+		  element.append(sentEle)  
+	  
+  def saveToFile(self,filename,myencoding='UTF-8'):
+	if self.tree:
+	  self.tree.write(filename,encoding=myencoding,pretty_print=True,xml_declaration=True)
+	  
+  
+  def addLinguisticProcessor(self,name,version, layer, time_stamp=True):
+	aux = self.tree.findall('kafHeader')
+	if len(aux)!=0:
+	  kaf_header = aux[0]
+	else:
+	  kaf_header = etree.Element('kafHeader')
+	  self.tree.getroot().insert(0,kaf_header)
+
+        aux2= kaf_header.findall('linguisticProcessors')
+        if len(aux2) == 0:
+          new_lp = etree.Element('linguisticProcessors')
+          new_lp.set('layer',layer)
+          kaf_header.append(new_lp)
+          
+	## Check if there is already element for the layer
+	my_lp_ele = None
+	
+	for element in kaf_header.findall('linguisticProcessors'):
+	  if element.get('layer','')==layer:
+		my_lp_ele = element
+		break
+	  
+	if time_stamp:  
+	  my_time = time.strftime('%Y-%m-%dT%H:%M:%S%Z')
+	else:
+	  my_time = '*'
+	  
+	my_lp = etree.Element('lp')
+	my_lp.set('timestamp',my_time)
+	my_lp.set('version',version)
+	my_lp.set('name',name)
+
+	if my_lp_ele is not None: #Already an element for linguisticProcessor with the layer
+	  my_lp_ele.append(my_lp)
+	else:
+	  # Create a new element for the LP layer
+	  my_lp_ele = etree.Element('linguisticProcessors')
+	  my_lp_ele.set('layer',layer)
+	  my_lp_ele.append(my_lp)
+	  #my_lp_ele.tail=my_lp_ele.text='\n'
+	  ## Should be inserted after the last linguisticProcessor element (stored in variable element)
+	  idx = kaf_header.index(element)
+	  kaf_header.insert(idx+1,my_lp_ele)
+		
+	  
+  def addLayer(self,type,element,first_char_id=None):
+	if first_char_id is None:
+		first_char_id = type[0]
+		
+	## Check if there is already layer for the type
+	layer_element = self.tree.find(type)
+	
+	if layer_element is None:
+	  layer_element = etree.Element(type)
+	  self.tree.getroot().append(layer_element)
+	  ## The id is going to be the first one
+	  new_id = first_char_id+'1'
+	else:
+	  ## We need to know how many elements there are in the layer
+	  current_n = len(layer_element.getchildren())
+	  new_id = first_char_id+''+str(current_n+1)
+	  
+	  
+	## In this point layer_element points to the correct element, existing or created
+	
+	element.set(first_char_id+'id',new_id)
+	layer_element.append(element)
+	return new_id
+	
+  def addElementToLayer(self,layer, element,first_char_id=None):
+	return self.addLayer(layer,element,first_char_id)
+
+  def add_attrs_to_layer(self,layer,attrs):
+	layer_element = self.tree.find(layer)
+	if layer_element is not None:
+	  for att, val in attrs.items():
+		layer_element.set(att,val)
+		
+
+  def addAttributeToElement(self,path,str_id, id, attribute, value,sub_path=None):
+	  for element in self.tree.findall(path):
+		if id is not None and element.get(str_id,None) == id:
+		  if sub_path is not None:
+			elements = element.findall(sub_path)
+			if len(elements)!=0: element = elements[0]
+		  element.set(attribute,value)
+		  return
+
+  
+  ## This works with the original definition of the property layer
+  ## KAF -> properties -> property* -> span* -> target*
+  def getSingleProperties_old(self):
+	  for element in self.tree.findall('properties/property'):
+		  my_id = element.get('pid')
+		  my_type = element.get('type')
+		  ref = element.find('references')
+		  if ref is not None:
+			element = ref
+		  for span_element in element.findall('span'):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleProperty(my_id,my_type,target_ids)
+			  yield my_prop
+			   
+  ## 18-June-2013
+  def getSingleProperties(self):
+	  for property in self.tree.findall('features/properties/property'):
+		  my_id = property.get('pid')
+		  if my_id is None:
+			  my_id = property.get('fpid')
+		  my_type = property.get('lemma')
+		  for span_element in property.findall('references/span'):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleProperty(my_id,my_type,target_ids)
+			  yield my_prop
+
+  # This function adds a new property of the type given with the list of ids given
+  # my_type -> 'sleeping comfort'	list_ids = ['id1','id2']
+  # It creates the features/properties layers in case 
+  # Agglomerates all the properties for the same TYPE under the same property element
+  # It calculates automatically the number for the identifier depending on the number
+  # of properties existing
+  def add_property(self,my_type,list_ids,comment=None):
+      
+      #Looking for feature layer or creating it
+      feature_layer = self.tree.find('features')
+      if feature_layer is None:
+          feature_layer = etree.Element('features')
+          self.tree.getroot().append(feature_layer)
+          
+      #Looking for properties layer
+      properties_layer = feature_layer.find('properties')
+      if properties_layer is None:
+          properties_layer = etree.Element('properties')
+          feature_layer.append(properties_layer)
+          
+      num_props = 0
+      property_layer = None
+      for property in properties_layer.findall('property'):
+          num_props += 1
+          prop_type = property.get('lemma')
+          if prop_type == my_type:
+              property_layer = property
+              break
+          
+      if property_layer is None:  # There is no any property for that type, let's create one
+          property_layer = etree.Element('property')
+          property_layer.set('pid','p'+str(num_props+1))
+          property_layer.set('lemma',my_type)    
+          properties_layer.append(property_layer)
+          
+          
+      references = property_layer.find('references')
+      if references is None:
+          references = etree.Element('references')
+          property_layer.append(references)
+      ## Create the new span
+      if comment is not None:
+        references.append(etree.Comment(comment))
+      span = etree.Element('span')
+      references.append(span)
+      for my_id in list_ids:
+           span.append(etree.Element('target',attrib={'id':my_id}))
+       
+   
+			  
+  
+  def getSingleEntities(self):
+	  for element in self.tree.findall('entities/entity'):
+		  my_id = element.get('eid')
+		  my_type = element.get('type')
+		  my_path_to_span = None
+		  ref = element.find('references')
+		  if ref is not None:
+                      my_path_to_span = 'references/span'
+                  else:
+                      my_path_to_span = 'span'
+
+		  for span_element in element.findall(my_path_to_span):
+			  target_ids = [target_element.get('id') for target_element in span_element.findall('target')]
+			  my_prop = KafSingleEntity(my_id,my_type,target_ids)
+			  yield my_prop
+
+
+  def getOpinions(self):
+	for element in self.tree.findall('opinions/opinion'):
+	  my_id = element.get('oid')
+	  
+	  tar_ids_hol = []
+	  tar_ids_tar = []
+	  polarity = strenght = ''
+	  tar_ids_exp = []
+	  
+	  #Holder
+	  opi_hol_eles = element.findall('opinion_holder')
+	  if len(opi_hol_eles)!=0:
+		  opi_hol_ele = opi_hol_eles[0]
+		  tar_ids_hol = [t_ele.get('id') for t_ele in opi_hol_ele.findall('span/target')]
+	  
+	  #Target
+	  opi_tar_eles = element.findall('opinion_target')
+	  if len(opi_tar_eles) != 0:
+		opi_tar_ele = opi_tar_eles[0]
+		tar_ids_tar = [t_ele.get('id') for t_ele in opi_tar_ele.findall('span/target')]
+		
+	  ## Opinion expression
+	  opi_exp_eles = element.findall('opinion_expression')
+	  if len(opi_exp_eles) != 0:
+		  opi_exp_ele = opi_exp_eles[0]
+		  polarity = opi_exp_ele.get('polarity','')
+		  strength = opi_exp_ele.get('strength','')
+		  tar_ids_exp = [t_ele.get('id') for t_ele in opi_exp_ele.findall('span/target')]
+
+	  yield KafOpinion(my_id,tar_ids_hol, tar_ids_tar, KafOpinionExpression(polarity, strength,tar_ids_exp))
+
+
+  
+  def remove_opinion_layer(self):
+      opinion_layer = self.tree.find('opinions')
+      if opinion_layer is not None:
+          self.tree.getroot().remove(opinion_layer)
+  
+  ## This function add an opinion to the opinion layer, creating it if does not exist
+  ## The id is calculated automatically according to the number of elements and ensring there is no repetition
+  def add_opinion(self,hol_ids,tar_ids,polarity,strength,exp_ids):
+      
+      #Looking for opinion layer or creating it
+      opinion_layer = self.tree.find('opinions')
+      if opinion_layer is None:
+          opinion_layer = etree.Element('opinions')
+          self.tree.getroot().append(opinion_layer)
+          
+      ## Generating unique id
+      list_of_oids = [opi.get('oid') for opi in opinion_layer]
+      
+      n = 1
+      while True:
+          my_id = 'o'+str(n)
+          if my_id not in list_of_oids:
+              break
+          n += 1
+      #####
+          
+      op_ele = etree.Element('opinion')
+      opinion_layer.append(op_ele)
+      op_ele.set('oid',my_id)
+
+      ## Holder
+      op_hol = etree.Element('opinion_holder')
+      op_ele.append(op_hol)
+      span_op_hol = etree.Element('span')
+      op_hol.append(span_op_hol)
+      for my_id in hol_ids:
+        span_op_hol.append(etree.Element('target',attrib={'id':my_id}))
+
+      ## TARGET
+      op_tar = etree.Element('opinion_target')
+      op_ele.append(op_tar)
+      span_op_tar = etree.Element('span')
+      op_tar.append(span_op_tar)
+      for my_id in tar_ids:
+        span_op_tar.append(etree.Element('target',attrib={'id':my_id}))
+
+      ## Expression
+    
+      op_exp = etree.Element('opinion_expression',attrib={'polarity':polarity,
+                                                       'strength':str(strength)})
+      op_ele.append(op_exp)
+      span_exp = etree.Element('span')
+      op_exp.append(span_exp)
+      for my_id in exp_ids:
+        span_exp.append(etree.Element('target',attrib={'id':my_id}))
+
+
+
+	
+