opener-opinion-detector-base 2.0.0

data/core/site-packages/pre_build/KafNafParser/feature_extractor/dependency.py

@@ -0,0 +1,309 @@
+from operator import itemgetter
+from VUA_pylib.common import get_max_distr_dict
+import sys
+
+class Cdependency_extractor:
+    def __init__(self,knaf_obj):
+        self.naf = knaf_obj
+        self.relations_for_term = {}
+        self.reverse_relations_for_term = {}
+        self.prefix_for_reverse = ''
+        
+       
+        already_linked = {}
+        for dep in knaf_obj.get_dependencies():
+            term_from = dep.get_from()
+            term_to = dep.get_to()
+            rfunc = dep.get_function()
+            
+            # Dependencies reversed are skipped...
+            #if rfunc.startswith('rhd/') or rfunc.startswith('whd/'):
+            #    continue
+            
+            #  For detecting cycles like:
+            #       <!-- rhd/body(geef,wat) -->  
+            #       <dep from="t19" to="t15" rfunc="rhd/body"/>
+            #       <!-- hd/su(wat,geef) -->
+            #      <dep from="t15" to="t19" rfunc="hd/su"/>
+            
+            '''
+            if term_from in already_linked and term_to in already_linked[term_from]:
+                #There could be a cycle, skip this 
+                print>>sys.stderr,'Skipped from',term_from,'to',term_to,'func',rfunc,' cycle detected'
+                continue
+            else:
+                #Include term_from as linked with term_to for future...
+                if term_to not in already_linked:
+                    already_linked[term_to] = set()
+                already_linked[term_to].add(term_from)
+            ''' 
+                
+                        
+
+            
+            if term_from in self.relations_for_term:
+                self.relations_for_term[term_from].append((rfunc,term_to))
+            else:
+                self.relations_for_term[term_from] = [(rfunc,term_to)]
+        
+            if term_to in self.reverse_relations_for_term:                
+                self.reverse_relations_for_term[term_to].append((self.prefix_for_reverse+rfunc,term_from))
+            else:
+                self.reverse_relations_for_term[term_to] = [(self.prefix_for_reverse+rfunc,term_from)]
+        
+        
+        self.paths_for_termid={}      
+        self.sentence_for_termid={}  
+        self.top_relation_for_term = {}     ## termid --> (relation,topnode)
+        self.root_for_sentence = {}         ## sentenceid --> termid
+        
+        for term_obj in knaf_obj.get_terms():
+            termid = term_obj.get_id()
+            
+            #Calculating the sentence id for the term id
+            span_ids = term_obj.get_span().get_span_ids()
+            token_obj = knaf_obj.get_token(span_ids[0])
+            if token_obj is None:
+                continue
+                
+            sentence = token_obj.get_sent()
+            
+            self.sentence_for_termid[termid] = sentence
+            ###########################################
+            
+            #paths = self.__propagate_node(termid,[])
+            #inversed = self.__reverse_propagate_node(termid)
+            
+            ## Due to the change on direction of dependencies...
+            inversed = self.__propagate_node(termid,already_propagated=[])
+            paths = self.__reverse_propagate_node(termid,already_propagated=[])
+
+            ##Calculate the top relation for the node, the relation with the main root of the tree
+            if len(inversed) != 0:
+                for ip in inversed:
+                    if len(ip)!=0:
+                        self.top_relation_for_term[termid] = ip[-1] ## ex. ('NMOD', 't2')
+                        root = ip[-1][1]
+                        if sentence not in self.root_for_sentence:
+                            self.root_for_sentence[sentence] = {}
+                        
+                        if root not in self.root_for_sentence[sentence]:
+                            self.root_for_sentence[sentence][root]=0
+                        else:
+                            self.root_for_sentence[sentence][root]+=1
+                        break
+                    
+            self.paths_for_termid[termid] = paths + inversed
+            
+            '''
+            print termid
+            print 'DIRECT RELS'
+            for p in paths:
+                print ' ',p 
+                
+            print 'INDIRECT RELS'
+            for p in inversed:
+                print ' ',p 
+            '''
+        ####
+
+        for sent_id, distr in self.root_for_sentence.items():
+            ## get_max_distr_dict imported from VUA_pylib.common
+            most_freq,c = get_max_distr_dict(distr)
+            self.root_for_sentence[sent_id] = most_freq
+
+        
+        
+        
+    def __propagate_node(self,node,already_propagated=[]):
+        paths = []
+        
+        relations = self.relations_for_term.get(node)
+        #print 'Propagate ',node,relations
+        if relations is None:   ##Case base
+            paths = [[]] 
+        elif node in already_propagated:
+            paths = [[]]
+
+        else:    
+            already_propagated.append(node)
+            for func, target_node in relations:
+                new_paths = self.__propagate_node(target_node, already_propagated)
+                for new_path in new_paths:
+                    new_path.insert(0,(func,target_node))
+                    paths.append(new_path)
+        return paths
+
+    def __reverse_propagate_node(self,node,already_propagated=[]):
+        paths = []
+        relations = self.reverse_relations_for_term.get(node)
+        #print 'Propagate reverse',node,relations,already_propagated
+        if relations is None:   ##Case base
+            paths = [[]] 
+        elif node in already_propagated:
+            paths = [[]]
+        else:    
+            already_propagated.append(node)
+            for func, target_node in relations:
+                new_paths = self.__reverse_propagate_node(target_node,already_propagated)
+                for new_path in new_paths:
+                    new_path.insert(0,(func,target_node))
+                    paths.append(new_path)
+        return paths        
+
+
+    # Get the shortest path between 2 term ids
+    def get_shortest_path(self,term1,term2):
+        dep_path = None
+        if term1 == term2: dep_path = []
+        else:
+            paths1 = self.paths_for_termid[term1]
+            paths2 = self.paths_for_termid[term2]
+
+            ##Check if term2 is on paths1 
+            hits = [] ## list of (common_id,idx1,idx2,numpath1,numpath2)
+            for num1, p1 in enumerate(paths1):
+                ids1 = [ my_id for my_func, my_id in p1]
+                if term2 in ids1:
+                    idx1 = ids1.index(term2)
+                    hits.append((term2,idx1+0,idx1,0,num1,None))
+                    
+            for num2,p2 in enumerate(paths2):
+                ids2 = [ my_id for my_func, my_id in p2]
+                if term1 in ids2:
+                    idx2=ids2.index(term1)
+                    hits.append((term1,0+idx2,0,idx2,None,num2))
+                    
+            #Pair by pair
+            for num1, p1 in enumerate(paths1):
+                #print 'Path1',term1, p1
+                ids1 = [ my_id for my_func, my_id in p1]
+                #print 'IDS1',ids1
+                for num2, p2 in enumerate(paths2):
+                    #print '\t',term2,p2
+                    ids2 = [ my_id for my_func, my_id in p2]
+                    #print '  IDS2',ids2
+                    common_ids = set(ids1) & set(ids2)
+                    #print '  cmmon',common_ids
+                    for common_id in common_ids:
+                        idx1 = ids1.index(common_id)
+                        idx2 = ids2.index(common_id)
+                        hits.append((common_id,idx1+idx2,idx1,idx2,num1,num2))
+
+                        
+            if len(hits) != 0:
+                dep_path = []
+                hits.sort(key=itemgetter(1))
+                best_hit = hits[0]
+                common_id, _, idx1, idx2, numpath1, numpath2 = best_hit
+                
+                if numpath2 is None:  #term2 is in one of the paths of t1
+                    path1 = paths1[numpath1]
+                    my_rels1 = path1[:idx1+1]
+                    ##complete_path = ''
+                    ##complete_path_ids = ''
+                    for func,node in my_rels1:
+                        dep_path.append(func)
+                        ##complete_path+=func+'#'
+                        ##complete_path_ids+=node+'#'
+                        
+                    #===========================================================
+                    # print 'CASE1',best_hit
+                    # print complete_path
+                    # print complete_path_ids
+                    #===========================================================
+                elif numpath1 is None: #term1 is in one of the paths of t2
+                    path2 = paths2[numpath2]
+                    my_rels2 = path2[:idx2+1]
+                    ##complete_path = ''
+                    ##complete_path_ids = ''
+                    for func,node in my_rels2:
+                        dep_path.append(func)
+                        #complete_path+=func+'#'
+                        #complete_path_ids+=node+'#'
+                        
+                    #===========================================================
+                    # print 'CASE2',best_hit
+                    # print complete_path
+                    # print complete_path_ids
+                    #===========================================================
+                else:   #There is a common node linking both
+                    path1 = paths1[numpath1]
+                    my_rels1 = path1[:idx1+1]
+                    
+                    path2 = paths2[numpath2]
+                    my_rels2 = path2[:idx2+1]
+                    
+                    ##complete_path = ''
+                    #complete_path_ids = ''
+                    for func,node in my_rels1:
+                        dep_path.append(func)
+                        ##complete_path+=func+'#'
+                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
+                        
+                    for func,node in my_rels2[-1::-1]:
+                        dep_path.append(func)
+                        ##complete_path+=func+'#'
+                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
+                    #===========================================================
+                    #    
+                    # print complete_path
+                    # print complete_path_ids
+                    # print path2
+                    # print my_rels1
+                    # print my_rels2
+                    # print 'CASE3',best_hit
+                    #===========================================================
+        return dep_path
+    
+    ## Get the shortest dependency path between 2 sets of spans
+    def get_shortest_path_spans(self,span1,span2):
+        shortest_path = None
+        
+        for term1 in span1:
+            for term2 in span2:
+                this_path = self.get_shortest_path(term1, term2)
+                #print term1,term2, this_path
+                if shortest_path is None or (this_path is not None and len(this_path)<len(shortest_path)):
+                    shortest_path = this_path
+        return shortest_path
+    
+    # Get the dependency path to the sentence root for a term id
+    def get_path_to_root(self,termid):
+        # Get the sentence for the term
+        root = None
+        sentence = self.sentence_for_termid.get(termid)
+        
+        if sentence is None:    #try with the top node
+            top_node = self.top_relation_for_term.get(termid)
+            if top_node is not None:
+                root = top_node[1]
+            else:
+                return None
+        else:
+            if sentence in self.root_for_sentence:
+                root = self.root_for_sentence[sentence]
+            else:
+                ##There is no root for this sentence
+                return None
+        # In this point top_node should be properly set
+        path = self.get_shortest_path(termid, root)
+        return path
+    
+    # Get the shortest dependency path to the sentence root for a span of ids
+    # extractor.get_shortest_path_to_root_span(['t444','t445','t446'])
+    def get_shortest_path_to_root_span(self,span):
+        shortest_path = None
+        for termid in span:
+            this_path = self.get_path_to_root(termid)
+            ## In case of , or . or whatever, the path to the root usually is None, there are no dependencies...
+            if shortest_path is None or (this_path is not None and len(this_path) < len(shortest_path)):
+                shortest_path = this_path
+        return shortest_path
+        
+
+
+                
+                           
+            
+       

data/core/site-packages/pre_build/KafNafParser/features_data.py

@@ -0,0 +1,131 @@
+from lxml import etree
+from lxml.objectify import dump
+from references_data import *
+
+
+
+class Cproperty:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('property')
+        else:
+            self.node = node
+    
+    def get_node(self):
+        return self.node
+    
+    def get_id(self):
+        if self.type == 'KAF':
+            return self.node.get('pid')
+        elif self.type == 'NAF':
+            return self.node.get('id')
+  
+    def set_id(self,pid):
+        if self.type == 'KAF':
+            return self.node.set('pid',pid)
+        elif self.type == 'NAF':
+            return self.node.set('id',pid)
+          
+    def get_type(self):
+        return self.node.get('lemma')
+
+    def set_type(self,t):
+        return self.node.set('lemma',t)
+    
+    def get_references(self):
+        for ref_node in self.node.findall('references'):
+            yield Creferences(ref_node)
+            
+    def set_reference(self,ref):
+        self.node.append(ref.get_node())
+                             
+                        
+                
+class Cproperties:
+    def __init__(self,node=None,type='NAF'):
+        self.type=type
+        if node is None:
+            self.node = etree.Element('properties')
+        else:
+            self.node = node
+            
+    def get_node(self):
+        return self.node
+            
+    def __iter__(self):
+        for prop_node in self.node.findall('property'):
+            yield Cproperty(prop_node,self.type)
+            
+    def add_property(self,pid, label,term_span):
+        new_property = Cproperty(type=self.type)
+        self.node.append(new_property.get_node())
+        ##Set the id
+        if pid is None:
+            ##Generate a new pid
+            existing_pids = [property.get_id() for property in self]
+            n = 0
+            new_pid = ''
+            while True:
+                new_pid = 'p'+str(n)
+                if new_pid not in existing_pids: break
+                n += 1
+            pid = new_pid
+        new_property.set_id(pid)
+        
+        new_property.set_type(label)
+        
+        new_ref = Creferences()
+        new_ref.add_span(term_span)
+        new_property.set_reference(new_ref)
+ 
+        
+                
+class Cfeatures:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('features')
+        else:
+            self.node = node
+        
+    def get_node(self):
+        return self.node
+    
+    def to_kaf(self):
+        if self.type == 'NAF':
+            ##convert all the properties
+            for node in self.node.findall('properties/property'):
+                node.set('pid',node.get('id'))
+                del node.attrib['id']
+
+    def to_naf(self):
+        if self.type == 'KAF':
+            ##convert all the properties
+            for node in self.node.findall('properties/property'):
+                node.set('id',node.get('pid'))
+                del node.attrib['pid']                
+            
+    def add_property(self,pid, label,term_span):
+        node_prop = self.node.find('properties')
+        if node_prop is None:
+            properties = Cproperties(type=self.type)
+            self.node.append(properties.get_node())
+        else:
+            properties = Cproperties(node=node_prop,type=self.type)
+            
+        properties.add_property(pid, label,term_span)
+        
+        
+    def get_properties(self):
+        node_prop = self.node.find('properties')
+        if node_prop is not None:
+            obj_properties = Cproperties(node_prop,self.type)
+            for prop in obj_properties:
+                yield prop
+                
+    def remove_properties(self):
+        node_prop = self.node.find('properties')
+        if node_prop is not None:
+            self.node.remove(node_prop)
+            

data/core/site-packages/pre_build/KafNafParser/header_data.py

@@ -0,0 +1,127 @@
+# Modified to KAF / NAF
+
+from lxml import etree
+import time
+
+class CfileDesc:
+    def __init__(self,node=None):
+        self.type = 'KAF/NAF'
+        if node is None:
+            self.node = etree.Element('fileDesc')
+        else:
+            self.node = node
+    
+    #self.title=''    #self.author=''    #self.creationtime=''    #self.filename=''    #self.filetype=''    #self.pages=''
+    
+ 
+class Cpublic:
+    def __init__(self,node=None):
+        self.type = 'KAF/NAF'
+        if node is None:
+            self.node = etree.Element('public')
+        else:
+            self.node = node
+            
+        #self.publicId = ''
+        #slf.uri = ''
+
+   
+class Clp:
+    def __init__(self,node=None,name="",version="",timestamp=None):
+        self.type = 'KAF/NAF'
+        if node is None:
+            self.node = etree.Element('lp')
+            self.set_name(name)
+            self.set_version(version)
+            self.set_timestamp(timestamp)
+        else:
+            self.node = node
+            
+    def set_name(self,name):
+        self.node.set('name',name)
+        
+    def set_version(self,version):
+        self.node.set('version',version)
+        
+    def set_timestamp(self,timestamp=None):
+        if timestamp is None:
+            import time
+            timestamp = time.strftime('%Y-%m-%dT%H:%M:%S%Z')
+        self.node.set('timestamp',timestamp)
+        
+
+    def get_node(self):
+        return self.node
+        
+    
+class ClinguisticProcessors:
+    def __init__(self,node=None):
+        self.type = 'KAF/NAF'
+        if node is None:
+            self.node = etree.Element('linguisticProcessors')
+        else:
+            self.node = node
+            
+    def get_layer(self):
+        return self.node.get('layer')
+    
+    def set_layer(self,layer):
+        self.node.set('layer',layer)
+    
+    def add_linguistic_processor(self,my_lp):
+        self.node.append(my_lp.get_node())
+        
+    def get_node(self):
+        return self.node
+
+    
+class CHeader:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            if self.type == 'NAF':
+                self.node = etree.Element('nafHeader')
+            elif self.type == 'KAF':
+                self.node = etree.Element('kafHeader')
+        else:
+            self.node = node
+    
+    def to_kaf(self):
+        if self.type == 'NAF':
+            self.node.tag = 'kafHeader'
+            self.type = 'KAF'
+        
+    def to_naf(self):
+        if self.type == 'KAF':
+            self.node.tag = 'nafHeader'
+            self.type = 'NAF'
+      
+    def add_linguistic_processors(self,linpro):
+        self.node.append(linpro.get_node())
+        
+    def remove_lp(self,layer):
+        for this_node in self.node.findall('linguisticProcessors'):
+            if this_node.get('layer') == layer:
+                self.node.remove(this_node)
+                break
+            
+        
+    def add_linguistic_processor(self, layer ,my_lp):
+        ## Locate the linguisticProcessor element for taht layer
+        found_lp_obj = None
+        for this_lp in self.node.findall('linguisticProcessors'):
+            lp_obj = ClinguisticProcessors(this_lp)
+            if lp_obj.get_layer() == layer:
+                found_lp_obj = lp_obj
+                break
+        
+        if found_lp_obj is None:    #Not found
+            found_lp_obj = ClinguisticProcessors()
+            found_lp_obj.set_layer(layer)
+            self.add_linguistic_processors(found_lp_obj)
+            
+        found_lp_obj.add_linguistic_processor(my_lp)
+        
+        
+            
+