opener-kaf-naf-parser 1.0.1 → 1.0.2

data/core/site-packages/pre_build/KafNafParser/feature_extractor/dependency.py

@@ -1,300 +0,0 @@
-from operator import itemgetter
-from VUA_pylib.common import get_max_distr_dict
-import sys
-
-class Cdependency_extractor:
-    def __init__(self,knaf_obj):
-        self.naf = knaf_obj
-        self.relations_for_term = {}
-        self.reverse_relations_for_term = {}
-        self.prefix_for_reverse = ''
-        
-       
-        already_linked = {}
-        for dep in knaf_obj.get_dependencies():
-            term_from = dep.get_from()
-            term_to = dep.get_to()
-            rfunc = dep.get_function()
-            
-            # Dependencies reversed are skipped...
-            #if rfunc.startswith('rhd/') or rfunc.startswith('whd/'):
-            #    continue
-            
-            #  For detecting cycles like:
-            #       <!-- rhd/body(geef,wat) -->  
-            #       <dep from="t19" to="t15" rfunc="rhd/body"/>
-            #       <!-- hd/su(wat,geef) -->
-            #      <dep from="t15" to="t19" rfunc="hd/su"/>
-            
-            '''
-            if term_from in already_linked and term_to in already_linked[term_from]:
-                #There could be a cycle, skip this 
-                print>>sys.stderr,'Skipped from',term_from,'to',term_to,'func',rfunc,' cycle detected'
-                continue
-            else:
-                #Include term_from as linked with term_to for future...
-                if term_to not in already_linked:
-                    already_linked[term_to] = set()
-                already_linked[term_to].add(term_from)
-            ''' 
-                
-                        
-
-            
-            if term_from in self.relations_for_term:
-                self.relations_for_term[term_from].append((rfunc,term_to))
-            else:
-                self.relations_for_term[term_from] = [(rfunc,term_to)]
-        
-            if term_to in self.reverse_relations_for_term:                
-                self.reverse_relations_for_term[term_to].append((self.prefix_for_reverse+rfunc,term_from))
-            else:
-                self.reverse_relations_for_term[term_to] = [(self.prefix_for_reverse+rfunc,term_from)]
-        
-        
-        self.paths_for_termid={}      
-        self.sentence_for_termid={}  
-        self.top_relation_for_term = {}     ## termid --> (relation,topnode)
-        self.root_for_sentence = {}         ## sentenceid --> termid
-        
-        for term_obj in knaf_obj.get_terms():
-            termid = term_obj.get_id()
-            
-            #Calculating the sentence id for the term id
-            span_ids = term_obj.get_span().get_span_ids()
-            token_obj = knaf_obj.get_token(span_ids[0])
-            sentence = token_obj.get_sent()
-            
-            self.sentence_for_termid[termid] = sentence
-            ###########################################
-            
-            #paths = self.__propagate_node(termid,[])
-            #inversed = self.__reverse_propagate_node(termid)
-            
-            ## Due to the change on direction of dependencies...
-            inversed = self.__propagate_node(termid,[])
-            paths = self.__reverse_propagate_node(termid)
-
-            ##Calculate the top relation for the node, the relation with the main root of the tree
-            if len(inversed) != 0:
-                for ip in inversed:
-                    if len(ip)!=0:
-                        self.top_relation_for_term[termid] = ip[-1] ## ex. ('NMOD', 't2')
-                        root = ip[-1][1]
-                        if sentence not in self.root_for_sentence:
-                            self.root_for_sentence[sentence] = {}
-                        
-                        if root not in self.root_for_sentence[sentence]:
-                            self.root_for_sentence[sentence][root]=0
-                        else:
-                            self.root_for_sentence[sentence][root]+=1
-                        break
-                    
-            self.paths_for_termid[termid] = paths + inversed
-            '''
-            print termid
-            print 'DIRECT RELS'
-            for p in paths:
-                print ' ',p 
-                
-            print 'INDIRECT RELS'
-            for p in inversed:
-                print ' ',p 
-            '''
-        ####
-
-        for sent_id, distr in self.root_for_sentence.items():
-            ## get_max_distr_dict imported from VUA_pylib.common
-            most_freq,c = get_max_distr_dict(distr)
-            self.root_for_sentence[sent_id] = most_freq
-
-        
-        
-        
-    def __propagate_node(self,node,already_propagated=[]):
-        paths = []
-        
-        relations = self.relations_for_term.get(node)
-        if relations is None:   ##Case base
-            paths = [[]] 
-        elif node in already_propagated:
-            paths = [[]]
-
-        else:    
-            already_propagated.append(node)
-            for func, target_node in relations:
-                new_paths = self.__propagate_node(target_node, already_propagated)
-                for new_path in new_paths:
-                    new_path.insert(0,(func,target_node))
-                    paths.append(new_path)
-        return paths
-
-    def __reverse_propagate_node(self,node,already_propagated=[]):
-        paths = []
-        relations = self.reverse_relations_for_term.get(node)
-        if relations is None:   ##Case base
-            paths = [[]] 
-        elif node in already_propagated:
-            paths = [[]]
-        else:    
-            already_propagated.append(node)
-            for func, target_node in relations:
-                new_paths = self.__reverse_propagate_node(target_node,already_propagated)
-                for new_path in new_paths:
-                    new_path.insert(0,(func,target_node))
-                    paths.append(new_path)
-        return paths        
-
-
-    # Get the shortest path between 2 term ids
-    def get_shortest_path(self,term1,term2):
-        dep_path = None
-        if term1 == term2: dep_path = []
-        else:
-            paths1 = self.paths_for_termid[term1]
-            paths2 = self.paths_for_termid[term2]
-            
-            ##Check if term2 is on paths1 
-            hits = [] ## list of (common_id,idx1,idx2,numpath1,numpath2)
-            for num1, p1 in enumerate(paths1):
-                
-                ids1 = [ my_id for my_func, my_id in p1]
-                if term2 in ids1:
-                    idx1=ids1.index(term2)
-                    hits.append((term2,idx1+0,idx1,0,num1,None))
-                    
-            for num2,p2 in enumerate(paths2):
-                ids2 = [ my_id for my_func, my_id in p2]
-                if term1 in p2:
-                    idx2=ids2.index(term1)
-                    hits.append((term1,0+idx2,0,idx2,None,num2))
-            
-            #Pair by pair
-            for num1, p1 in enumerate(paths1):
-                #print 'Path1',term1, p1
-                ids1 = [ my_id for my_func, my_id in p1]
-                for num2, p2 in enumerate(paths2):
-                    #print '\t',term2,p2
-                    ids2 = [ my_id for my_func, my_id in p2]
-                    common_ids = set(ids1) & set(ids2)
-                    for common_id in common_ids:
-                        idx1 = ids1.index(common_id)
-                        idx2 = ids2.index(common_id)
-                        hits.append((common_id,idx1+idx2,idx1,idx2,num1,num2))
-
-                        
-            if len(hits) != 0:
-                dep_path = []
-                hits.sort(key=itemgetter(1))
-                best_hit = hits[0]
-                common_id, _, idx1, idx2, numpath1, numpath2 = best_hit
-                
-                if numpath2 is None:  #term2 is in one of the paths of t1
-                    path1 = paths1[numpath1]
-                    my_rels1 = path1[:idx1+1]
-                    ##complete_path = ''
-                    ##complete_path_ids = ''
-                    for func,node in my_rels1:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        ##complete_path_ids+=node+'#'
-                        
-                    #===========================================================
-                    # print 'CASE1',best_hit
-                    # print complete_path
-                    # print complete_path_ids
-                    #===========================================================
-                elif numpath1 is None: #term1 is in one of the paths of t2
-                    path2 = paths2[numpath2]
-                    my_rels2 = path2[:idx2+1]
-                    ##complete_path = ''
-                    ##complete_path_ids = ''
-                    for func,node in my_rels2:
-                        dep_path.append(func)
-                        #complete_path+=func+'#'
-                        #complete_path_ids+=node+'#'
-                        
-                    #===========================================================
-                    # print 'CASE2',best_hit
-                    # print complete_path
-                    # print complete_path_ids
-                    #===========================================================
-                else:   #There is a common node linking both
-                    path1 = paths1[numpath1]
-                    my_rels1 = path1[:idx1+1]
-                    
-                    path2 = paths2[numpath2]
-                    my_rels2 = path2[:idx2+1]
-                    
-                    ##complete_path = ''
-                    #complete_path_ids = ''
-                    for func,node in my_rels1:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
-                        
-                    for func,node in my_rels2[-1::-1]:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
-                    #===========================================================
-                    #    
-                    # print complete_path
-                    # print complete_path_ids
-                    # print path2
-                    # print my_rels1
-                    # print my_rels2
-                    # print 'CASE3',best_hit
-                    #===========================================================
-        return dep_path
-    
-    ## Get the shortest dependency path between 2 sets of spans
-    def get_shortest_path_spans(self,span1,span2):
-        shortest_path = None
-        
-        for term1 in span1:
-            for term2 in span2:
-                this_path = self.get_shortest_path(term1, term2)
-                if shortest_path is None or (this_path is not None and len(this_path)<len(shortest_path)):
-                    shortest_path = this_path
-        return shortest_path
-    
-    # Get the dependency path to the sentence root for a term id
-    def get_path_to_root(self,termid):
-        # Get the sentence for the term
-        root = None
-        sentence = self.sentence_for_termid.get(termid)
-        
-        if sentence is None:    #try with the top node
-            top_node = self.top_relation_for_term.get(termid)
-            if top_node is not None:
-                root = top_node[1]
-            else:
-                return None
-        else:
-            if sentence in self.root_for_sentence:
-                root = self.root_for_sentence[sentence]
-            else:
-                ##There is no root for this sentence
-                return None
-        # In this point top_node should be properly set
-        path = self.get_shortest_path(termid, root)
-        return path
-    
-    # Get the shortest dependency path to the sentence root for a span of ids
-    # extractor.get_shortest_path_to_root_span(['t444','t445','t446'])
-    def get_shortest_path_to_root_span(self,span):
-        shortest_path = None
-        for termid in span:
-            this_path = self.get_path_to_root(termid)
-            ## In case of , or . or whatever, the path to the root usually is None, there are no dependencies...
-            if shortest_path is None or (this_path is not None and len(this_path) < len(shortest_path)):
-                shortest_path = this_path
-        return shortest_path
-        
-
-
-                
-                           
-            
-       

data/core/site-packages/pre_build/KafNafParser/features_data.py

@@ -1,71 +0,0 @@
-from lxml import etree
-from lxml.objectify import dump
-from references_data import *
-
-
-
-class Cproperty:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('property')
-        else:
-            self.node = node
-    
-    def get_id(self):
-        if self.type == 'KAF':
-            return self.node.get('pid')
-        elif self.type == 'NAF':
-            return self.node.get('id')
-    
-    def get_type(self):
-        return self.node.get('lemma')
-    
-    def get_references(self):
-        for ref_node in self.node.findall('references'):
-            yield Creferences(ref_node)
-                             
-                        
-                
-class Cproperties:
-    def __init__(self,node=None,type='NAF'):
-        self.type=type
-        if node is None:
-            self.node = etree.Element('properties')
-        else:
-            self.node = node
-            
-    def __iter__(self):
-        for prop_node in self.node.findall('property'):
-            yield Cproperty(prop_node,self.type)
-                
-class Cfeatures:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('features')
-        else:
-            self.node = node
-        
-    def to_kaf(self):
-        if self.type == 'NAF':
-            ##convert all the properties
-            for node in self.node.findall('properties/property'):
-                node.set('pid',node.get('id'))
-                del node.attrib['id']
-
-    def to_naf(self):
-        if self.type == 'KAF':
-            ##convert all the properties
-            for node in self.node.findall('properties/property'):
-                node.set('id',node.get('pid'))
-                del node.attrib['pid']                
-            
-        
-    def get_properties(self):
-        node_prop = self.node.find('properties')
-        if node_prop is not None:
-            obj_properties = Cproperties(node_prop,self.type)
-            for prop in obj_properties:
-                yield prop
-                

data/core/site-packages/pre_build/KafNafParser/header_data.py

@@ -1,127 +0,0 @@
-# Modified to KAF / NAF
-
-from lxml import etree
-import time
-
-class CfileDesc:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('fileDesc')
-        else:
-            self.node = node
-    
-    #self.title=''    #self.author=''    #self.creationtime=''    #self.filename=''    #self.filetype=''    #self.pages=''
-    
- 
-class Cpublic:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('public')
-        else:
-            self.node = node
-            
-        #self.publicId = ''
-        #slf.uri = ''
-
-   
-class Clp:
-    def __init__(self,node=None,name="",version="",timestamp=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('lp')
-            self.set_name(name)
-            self.set_version(name)
-            self.set_timestamp(timestamp)
-        else:
-            self.node = node
-            
-    def set_name(self,name):
-        self.node.set('name',name)
-        
-    def set_version(self,version):
-        self.node.set('version',version)
-        
-    def set_timestamp(self,timestamp=None):
-        if timestamp is None:
-            import time
-            timestamp = time.strftime('%Y-%m-%dT%H:%M:%S%Z')
-        self.node.set('timestamp',timestamp)
-        
-
-    def get_node(self):
-        return self.node
-        
-    
-class ClinguisticProcessors:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('linguisticProcessors')
-        else:
-            self.node = node
-            
-    def get_layer(self):
-        return self.node.get('layer')
-    
-    def set_layer(self,layer):
-        self.node.set('layer',layer)
-    
-    def add_linguistic_processor(self,my_lp):
-        self.node.append(my_lp.get_node())
-        
-    def get_node(self):
-        return self.node
-
-    
-class CHeader:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            if self.type == 'NAF':
-                self.node = etree.Element('nafHeader')
-            elif self.type == 'KAF':
-                self.node = etree.Element('kafHeader')
-        else:
-            self.node = node
-    
-    def to_kaf(self):
-        if self.type == 'NAF':
-            self.node.tag = 'kafHeader'
-            self.type = 'KAF'
-        
-    def to_naf(self):
-        if self.type == 'KAF':
-            self.node.tag = 'nafHeader'
-            self.type = 'NAF'
-      
-    def add_linguistic_processors(self,linpro):
-        self.node.append(linpro.get_node())
-        
-    def remove_lp(self,layer):
-        for this_node in self.node.findall('linguisticProcessors'):
-            if this_node.get('layer') == layer:
-                self.node.remove(this_node)
-                break
-            
-        
-    def add_linguistic_processor(self, layer ,my_lp):
-        ## Locate the linguisticProcessor element for taht layer
-        found_lp_obj = None
-        for this_lp in self.node.findall('linguisticProcessors'):
-            lp_obj = ClinguisticProcessors(this_lp)
-            if lp_obj.get_layer() == layer:
-                found_lp_obj = lp_obj
-                break
-        
-        if found_lp_obj is None:    #Not found
-            found_lp_obj = ClinguisticProcessors()
-            found_lp_obj.set_layer(layer)
-            self.add_linguistic_processors(found_lp_obj)
-            
-        found_lp_obj.add_linguistic_processor(my_lp)
-        
-        
-            
-    

data/core/site-packages/pre_build/KafNafParser/opinion_data.py

@@ -1,200 +0,0 @@
-#Modified for KAF NAF ok
-
-from lxml import etree
-from lxml.objectify import dump
-from span_data import *
-
-
-
-class Cholder:
-    def __init__(self,node=None):
-        self.type = 'NAF/KAF'
-        if node is None:
-            self.node = etree.Element('opinion_holder')
-        else:
-            self.node = node    
-    
-    def set_span(self,my_span):
-        self.node.append(my_span.get_node())
-
-    def set_comment(self,c):
-        c = c.replace('--','- -')
-        self.node.insert(0,etree.Comment(c) )
-        
-    def get_span(self):
-        span_obj = self.node.find('span')
-        if span_obj is not None:
-            return Cspan(span_obj)
-        return None
-        
-    def __str__(self):
-        return dump(self.node)  
-    
-    def get_node(self):
-        return self.node
-        
-
-class Ctarget:
-    def __init__(self,node=None):
-        self.type = 'NAF/KAF'
-        if node is None:
-            self.node = etree.Element('opinion_target')
-        else:
-            self.node = node    
-    
-    def set_comment(self,c):
-        c = c.replace('--','- -')
-        self.node.insert(0,etree.Comment(c) )
-        
-    def get_comment(self):
-        return self.node_comment
-        
-    def set_span(self,my_span):
-        self.node.append(my_span.get_node())  
-    
-    def get_span(self):
-        span_obj = self.node.find('span')
-        if span_obj is not None:
-            return Cspan(span_obj)
-        return None
-    
-    def __str__(self):
-        return dump(self.node)  
-
-    def get_node(self):
-        return self.node
-        
-        
-class Cexpression:
-    def __init__(self,node=None):
-        self.type = 'NAF/KAF'
-        if node is None:
-            self.node = etree.Element('opinion_expression')
-        else:
-            self.node = node   
-
-    def set_comment(self,c):
-        c = c.replace('--','- -')
-        self.node.insert(0,etree.Comment(c)) 
-    
-    def set_polarity(self,pol):
-        self.node.set('polarity',pol)
-        
-    def get_polarity(self):
-        return self.node.get('polarity')
-        
-    def set_strength(self,st):
-        self.node.set('strength',st)
-        
-    def get_strength(self):
-        return self.node.get('strength')
-        
-    def set_span(self,my_span):
-        self.node.append(my_span.get_node())
-        
-    def get_span(self):
-        span_obj = self.node.find('span')
-        if span_obj is not None:
-            return Cspan(span_obj)
-        return None
-
-    def __str__(self):
-        return dump(self.node)  
-    
-    def get_node(self):
-        return self.node
-
-class Copinion:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('opinion')
-        else:
-            self.node = node  
-    
-    def set_id(self,my_id):
-        if self.type == 'NAF':
-            self.node.set('id',my_id)
-        elif self.type == 'KAF':
-            self.node.set('oid',my_id)
-            
-    def get_id(self):
-        if self.type == 'NAF':
-            self.node.het('id')
-        elif self.type == 'KAF':
-            self.node.get('oid')        
-        
-    def set_holder(self,hol):
-        self.node.append(hol.get_node())
-        
-    def get_holder(self):
-        node_hol = self.node.find('opinion_holder')
-        if node_hol is not None:
-            return Cholder(node_hol)
-        else:
-            return None
-
-    def set_target(self,tar):
-        self.node.append(tar.get_node())
-        
-    def get_target(self):
-        node_target = self.node.find('opinion_target')
-        if node_target is not None:
-            return Ctarget(node_target)
-        else:
-            return None
-        
-    def set_expression(self,exp):
-        self.node.append(exp.get_node())
-
-    def get_expression(self):
-        node_exp = self.node.find('opinion_expression')
-        if node_exp is not None:
-            return Cexpression(node_exp)
-        else:
-            return None
-        
-    def __str__(self):
-        return dump(self.node)  
-    
-    def get_node(self):
-        return self.node
-    
-    
-
-
-
-class Copinions:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('opinions')
-        else:
-            self.node = node
-            
-    def __get_opinion_nodes(self):
-        for node in self.node.findall('opinion'):
-            yield node
-            
-    def get_opinions(self):
-        for node in self.__get_opinion_nodes():
-            yield Copinion(node,self.type)
-            
-    def to_kaf(self):
-        if self.type == 'NAF':
-            for node in self.__get_opinion_nodes():
-                node.set('oid',node.get('id'))
-                del node.attrib['id']
-
-    def to_naf(self):
-        if self.type == 'KAF':
-            for node in self.__get_opinion_nodes():
-                node.set('id',node.get('oid'))
-                del node.attrib['oid']                
-            
-            
-    def add_opinion(self,opi_obj):
-        self.node.append(opi_obj.get_node())
-                         
-    def get_node(self):
-        return self.node