opener-opinion-detector-base 2.0.0

data/core/site-packages/pre_build/KafNafParser/__init__.py

@@ -0,0 +1,14 @@
+from KafNafParserMod import *
+from header_data import *
+from external_references_data import *
+from span_data import *
+from term_data import *
+from term_sentiment_data import *
+from text_data import *
+from entity_data import *
+from features_data import  *
+from opinion_data import *
+from dependency_data import *
+from constituency_data import *
+from references_data import *
+from coreference_data import *

data/core/site-packages/pre_build/KafNafParser/constituency_data.py

@@ -0,0 +1,125 @@
+from lxml import etree
+from lxml.objectify import dump
+from span_data import Cspan
+
+
+class Cnonterminal:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('nt')
+        else:
+            self.node = node
+            
+    def get_id(self):
+        return self.node.get('id')
+    
+    def get_label(self):
+        return self.node.get('label')
+
+    def __str__(self):
+        return dump(self.node)
+            
+
+
+class Cterminal:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('t')
+        else:
+            self.node = node
+            
+    def get_id(self):
+        return self.node.get('id')
+    
+    def get_span(self):
+        span_node = self.node.find('span')
+        return Cspan(span_node)
+    
+    def __str__(self):
+        return dump(self.node)
+            
+class Cedge:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('edge')
+        else:
+            self.node = node
+            
+    def __str__(self):
+        return dump(self.node)
+    
+    def get_from(self):
+        return self.node.get('from')
+    
+    def get_to(self):
+        return self.node.get('to')
+            
+
+
+class Ctree:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('tree')
+        else:
+            self.node = node
+            
+
+    def __str__(self):
+        return dump(self.node)
+    
+    ## Fore getting non terminals
+    def __get_nt_nodes(self):
+        for nt_node in self.node.findall('nt'):
+            yield nt_node
+
+    def get_non_terminals(self):
+        for nt_node in self.__get_nt_nodes():
+            yield Cnonterminal(nt_node)
+    ##################################
+            
+    ## Fore getting  terminals
+    def __get_t_nodes(self):
+        for t_node in self.node.findall('t'):
+            yield t_node
+
+    def get_terminals(self):
+        for t_node in self.__get_t_nodes():
+            yield Cterminal(t_node)
+    ##################################            
+            
+     ## Fore getting  edges
+    def __get_edge_nodes(self):
+        for t_node in self.node.findall('edge'):
+            yield t_node
+
+    def get_edges(self):
+        for edge_node in self.__get_edge_nodes():
+            yield Cedge(edge_node)
+    ##################################          
+            
+    
+
+class Cconstituency:
+    def __init__(self,node=None):
+        self.type = 'NAF/NAF'
+        if node is None:
+            self.node = etree.Element('constituency')
+        else:
+            self.node = node
+            
+    def to_kaf(self):
+        pass
+    
+    def to_naf(self):
+        pass
+    
+    def __get_tree_nodes(self):
+        for tree_node in self.node.findall('tree'):
+            yield tree_node
+    
+    def get_trees(self):
+        for tree_node in self.__get_tree_nodes():
+            yield Ctree(tree_node)
+            
+    def __str__(self):
+        return dump(self.node)

data/core/site-packages/pre_build/KafNafParser/coreference_data.py

@@ -0,0 +1,52 @@
+from lxml import etree
+from span_data import Cspan
+
+class Ccoreference:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('coref')
+        else:
+            self.node = node
+            
+    def get_id(self):
+        if self.type == 'NAF':
+            return self.node.get('id')
+        elif self.type == 'KAF':
+            return self.node.get('coid')
+    
+    def get_spans(self):
+        for node_span in self.node.findall('span'):
+            yield Cspan(node_span)
+    
+            
+
+class Ccoreferences:
+    def __init__(self,node=None, type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('coreferences')
+        else:
+            self.node = node
+            
+    def __get_corefs_nodes(self):
+        for coref_node in self.node.findall('coref'):
+            yield coref_node
+            
+    def get_corefs(self):
+        for coref_node in self.__get_corefs_nodes():
+            yield Ccoreference(coref_node,self.type)
+            
+    def to_kaf(self):
+        if self.type == 'NAF':
+            for node_coref in self.__get_corefs_nodes():
+                node_coref.set('coid',node_coref.get('id'))
+                del node_coref.attrib['id']
+        
+    def to_naf(self):
+        if self.type == 'KAF':
+            for node_coref in self.__get_corefs_nodes():
+                node_coref.set('id',node_coref.get('coid'))
+                del node_coref.attrib['coid']
+                    
+            

data/core/site-packages/pre_build/KafNafParser/dependency_data.py

@@ -0,0 +1,78 @@
+from lxml import etree
+#from lxml.objectify import dump
+
+
+class Cdependency:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('dep')
+        else:
+            self.node = node
+            
+    def get_node_comment(self):
+        return self.node_comment
+    
+    def get_node(self):
+        return self.node
+    
+    def get_from(self):
+        return self.node.get('from')
+    
+    def get_to(self):
+        return self.node.get('to')
+    
+    def get_function(self):
+        return self.node.get('rfunc')
+
+    def set_from(self, f):
+        self.node.set('from',f)
+    
+    def set_to(self,t):
+        self.node.set('to',t)
+    
+    def set_function(self,f):
+        self.node.set('rfunc',f)
+        
+
+    def set_comment(self,c):
+        c = c.replace('--','- -')
+        self.node.insert(0,etree.Comment(c) )
+            
+    
+    def __str__(self):
+        return dump(self.node)
+
+
+
+class Cdependencies:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('deps')
+        else:
+            self.node = node
+            
+    def get_node(self):
+        return self.node
+            
+    def to_kaf(self):
+        pass
+    
+    def to_naf(self):
+        pass
+            
+    def __str__(self):
+        return dump(self.node)
+
+
+    def __get_node_deps(self):
+        for node_dep in self.node.findall('dep'):
+            yield node_dep
+            
+    def get_dependencies(self):
+        for node in self.__get_node_deps():
+            yield Cdependency(node)
+            
+            
+    def add_dependency(self,my_dep):
+        self.node.append(my_dep.get_node())
+            

data/core/site-packages/pre_build/KafNafParser/entity_data.py

@@ -0,0 +1,59 @@
+## Modified for KAF NAF adaptation
+from lxml import etree
+from lxml.objectify import dump
+from references_data import *
+    
+    
+class Centity:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('entity')
+        else:
+            self.node = node
+            
+    def get_id(self):
+        if self.type == 'NAF':
+            return self.node.get('id')
+        elif self.type == 'KAF':
+            return self.node.get('eid')
+    
+    def get_type(self):
+        return self.node.get('type')
+    
+    def get_references(self):
+        for ref_node in self.node.findall('references'):
+            yield Creferences(ref_node)
+    
+class Centities:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        if node is None:
+            self.node = etree.Element('entities')
+        else:
+            self.node = node
+        
+        
+    def to_kaf(self):
+        if self.type == 'NAF':
+            for node in self.__get_entity_nodes():
+                node.set('eid',node.get('id'))
+                del node.attrib['id']
+
+    def to_naf(self):
+        if self.type == 'KAF':
+            for node in self.__get_entity_nodes():
+                node.set('id',node.get('eid'))
+                del node.attrib['eid']
+                
+    def __get_entity_nodes(self):
+        for ent_node in self.node.findall('entity'):
+            yield ent_node
+                
+    def __iter__(self):
+        for ent_node in self.__get_entity_nodes():
+            yield Centity(ent_node,self.type)
+        
+        
+    def __str__(self):
+        return dump(self.node)

data/core/site-packages/pre_build/KafNafParser/external_references_data.py

@@ -0,0 +1,41 @@
+# included modification for KAF/NAF
+from term_sentiment_data import Cterm_sentiment
+from lxml import etree
+
+class CexternalReference:
+    def __init__(self,node=None):
+        self.type= 'NAF/KAF'
+        #self.resource = self.reference = self.reftype = self.status = self.source = self.confidence = ''
+        if node is None:
+            self.node = etree.Element('externalRef')
+        else:
+            self.node = node
+        
+    def get_node(self):
+        return self.node
+        
+    def set_resource(self,resource):
+        self.node.set('resource',resource)
+    
+    def set_confidence(self,confidence):
+        self.node.set('confidence',confidence)
+    
+    def set_reference(self,reference):
+        self.node.set('reference',reference)
+        
+
+class CexternalReferences:
+    def __init__(self,node=None):
+        if node is None:
+            self.node = etree.Element('externalReferences')
+        else:
+            self.node = node
+                
+    def add_external_reference(self,ext_ref):
+        self.node.append(ext_ref.get_node())
+        
+    def get_node(self):
+        return self.node
+        
+ 
+    

data/core/site-packages/pre_build/KafNafParser/feature_extractor/__init__.py

@@ -0,0 +1,2 @@
+from dependency import *
+from constituency import *

data/core/site-packages/pre_build/KafNafParser/feature_extractor/constituency.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+
+from operator import itemgetter
+
+'''
+Extract information from the contituent layer from a NAF file
+'''
+
+class Cconstituency_extractor:
+    def __init__(self,knaf_obj):
+        self.naf = knaf_obj
+        #Extract terminals, non terminals and edges
+        ## Extracted directly from 
+        self.terminals = {}          #terminal id --> list term ids
+        self.terminal_for_term = {}  #term id --> terminal id
+        self.label_for_nonter = {}   # nonter --> label
+        self.reachable_from = {}     # node_from --> [nodeto1, nodeto2...]
+        
+        self.extract_info_from_naf(knaf_obj)
+        
+        #Extracting all posible paths from leave to root for each terminal id
+        self.paths_for_terminal= {}
+        for terminal_id in self.terminals.keys():
+            paths = self.__expand_node(terminal_id,False)
+            self.paths_for_terminal[terminal_id] = paths
+        #######################################
+        
+        ## Create, for each non terminal, which are the terminals subsumed
+        self.terms_subsumed_by_nonter = {}  ## ['nonter12'] = set('t1,'t2','t3','t4')
+        for terminal_id, span_terms in self.terminals.items():
+            for path in self.paths_for_terminal[terminal_id]:
+                for nonter in path:
+                    if nonter not in self.terms_subsumed_by_nonter:
+                        self.terms_subsumed_by_nonter[nonter] = set()
+                    for termid in span_terms:
+                        self.terms_subsumed_by_nonter[nonter].add(termid)
+        
+        ## To print the paths calculated
+#        for terminal in self.terminals.keys():
+#            print terminal
+#            for path in self.paths_for_terminal[terminal]:
+#                sep=' '
+#                for node in path:
+#                    print sep,node,self.label_for_nonter.get(node,'?')
+#                    sep+=' '
+#            print '#'*20
+
+               
+    def get_deepest_phrases(self):
+        all_nonter = set()
+        for terminal in self.terminals.keys():
+            for path in self.paths_for_terminal[terminal]:
+                first_non_ter_phrase = path[1]
+                all_nonter.add(first_non_ter_phrase)
+        
+        ter_for_nonter = {}
+        for nonter in all_nonter:
+            for terminal in self.terminals.keys():
+                for path in self.paths_for_terminal[terminal]:
+                    if nonter in path:
+                        if nonter in ter_for_nonter:
+                            ter_for_nonter[nonter].append(terminal)
+                        else:
+                            ter_for_nonter[nonter] = [terminal]
+        
+        visited = set()
+        for nonter, list_term in ter_for_nonter.items():
+            for ter in list_term:
+
+                visited.add(ter)
+    
+    
+    ### Returns the label of the deepest phrase for the term id (termid as in the term layer)
+    def get_deepest_phrase_for_termid(self,termid):
+        terminal_id = self.terminal_for_term.get(termid)
+        label = None
+        subsumed = []
+        if terminal_id is not None:
+            first_path = self.paths_for_terminal[terminal_id][0]
+            first_phrase_id = first_path[1]
+            label = self.label_for_nonter.get(first_phrase_id)
+            subsumed = self.terms_subsumed_by_nonter.get(first_phrase_id,[])
+        return label,sorted(list(subsumed))
+    
+    
+    def get_least_common_subsumer(self,from_tid,to_tid):
+        termid_from = self.terminal_for_term.get(from_tid)
+        termid_to = self.terminal_for_term.get(to_tid)
+        
+        path_from = self.paths_for_terminal[termid_from][0]
+        path_to = self.paths_for_terminal[termid_to][0]
+        common_nodes = set(path_from) & set(path_to)
+        if len(common_nodes) == 0:
+            return None
+        else:
+            indexes = []
+            for common_node in common_nodes:
+                index1 = path_from.index(common_node)
+                index2 = path_to.index(common_node)
+                indexes.append((common_node,index1+index2))
+            indexes.sort(key=itemgetter(1))
+            shortest_common = indexes[0][0]
+            return shortest_common
+        
+    
+    def get_path_from_to(self,from_tid, to_tid):
+        shortest_subsumer = self.get_least_common_subsumer(from_tid, to_tid)
+        
+        #print 'From:',self.naf.get_term(from_tid).get_lemma()
+        #print 'To:',self.naf.get_term(to_tid).get_lemma()
+        termid_from = self.terminal_for_term.get(from_tid)
+        termid_to = self.terminal_for_term.get(to_tid)
+        
+        path_from = self.paths_for_terminal[termid_from][0]
+        path_to = self.paths_for_terminal[termid_to][0]
+        
+        if shortest_subsumer is None:
+            return None
+        
+        complete_path = []
+        for node in path_from:
+            complete_path.append(node)
+            if node == shortest_subsumer: break
+        
+        begin=False
+        for node in path_to[-1::-1]:
+            if begin:
+                complete_path.append(node)
+             
+            if node==shortest_subsumer:
+                begin=True
+        labels = [self.label_for_nonter[nonter] for nonter in complete_path]
+        return labels
+                
+            
+    def get_path_for_termid(self,termid):
+        terminal_id = self.terminal_for_term.get(termid)
+        paths = self.paths_for_terminal[terminal_id]
+        labels = [self.label_for_nonter[nonter] for nonter in paths[0]]
+        return labels
+        
+    def extract_info_from_naf(self,knaf_obj):
+        ## Generated internally
+        # For each terminal node, a list of paths through all the edges
+        self.paths_for_terminal = {} 
+        for tree in knaf_obj.get_trees():
+            for terminal in tree.get_terminals():
+                ter_id = terminal.get_id()
+                span_ids = terminal.get_span().get_span_ids()
+                self.terminals[ter_id] = span_ids
+                for this_id in span_ids:
+                    self.terminal_for_term[this_id] = ter_id
+            
+            
+            for non_terminal in tree.get_non_terminals():
+                nonter_id = non_terminal.get_id()
+                label = non_terminal.get_label()
+                self.label_for_nonter[nonter_id] = label
+                
+                
+            for edge in tree.get_edges():
+                node_from = edge.get_from()
+                node_to = edge.get_to()
+                if node_from not in self.reachable_from:
+                    self.reachable_from[node_from] = [node_to]
+                else:
+                    self.reachable_from[node_from].append(node_to)
+        
+     
+        
+    ##Recursive function
+    ## Propagates the node through all the relations extracte from the edges information
+    ## It returns a list of lists, one for each path
+    ## Include_this_node is used for avoiding the first node 
+    def __expand_node(self,node,include_this_node=True):
+        paths = []
+        possible_nodes = self.reachable_from.get(node,[])
+        if len(possible_nodes) == 0:
+            return [[node]]
+        else:
+            for possible_node in possible_nodes:
+                new_paths = self.__expand_node(possible_node)
+                for path in new_paths:
+                    if include_this_node: 
+                        path.insert(0,node)
+                    paths.append(path)
+            return paths
+       
+    def get_chunks(self,chunk_type):
+        for nonter,this_type in self.label_for_nonter.items():
+            if this_type == chunk_type:
+                subsumed = self.terms_subsumed_by_nonter.get(nonter)
+                if subsumed is not None:
+                    yield sorted(list(subsumed))
+                    
+    def get_all_chunks_for_term(self,termid):
+        terminal_id = self.terminal_for_term.get(termid)
+        paths = self.paths_for_terminal[terminal_id]
+        for path in paths:
+            for node in path:
+                this_type = self.label_for_nonter[node]
+                subsumed =  self.terms_subsumed_by_nonter.get(node)
+                if subsumed is not None:
+                    yield this_type,sorted(list(subsumed))
+