opener-opinion-detector-base 2.0.1 → 2.1.2

data/core/site-packages/pre_build/KafNafParser/__init__.py

@@ -1,14 +0,0 @@
-from KafNafParserMod import *
-from header_data import *
-from external_references_data import *
-from span_data import *
-from term_data import *
-from term_sentiment_data import *
-from text_data import *
-from entity_data import *
-from features_data import  *
-from opinion_data import *
-from dependency_data import *
-from constituency_data import *
-from references_data import *
-from coreference_data import *

data/core/site-packages/pre_build/KafNafParser/constituency_data.py

@@ -1,125 +0,0 @@
-from lxml import etree
-from lxml.objectify import dump
-from span_data import Cspan
-
-
-class Cnonterminal:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('nt')
-        else:
-            self.node = node
-            
-    def get_id(self):
-        return self.node.get('id')
-    
-    def get_label(self):
-        return self.node.get('label')
-
-    def __str__(self):
-        return dump(self.node)
-            
-
-
-class Cterminal:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('t')
-        else:
-            self.node = node
-            
-    def get_id(self):
-        return self.node.get('id')
-    
-    def get_span(self):
-        span_node = self.node.find('span')
-        return Cspan(span_node)
-    
-    def __str__(self):
-        return dump(self.node)
-            
-class Cedge:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('edge')
-        else:
-            self.node = node
-            
-    def __str__(self):
-        return dump(self.node)
-    
-    def get_from(self):
-        return self.node.get('from')
-    
-    def get_to(self):
-        return self.node.get('to')
-            
-
-
-class Ctree:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('tree')
-        else:
-            self.node = node
-            
-
-    def __str__(self):
-        return dump(self.node)
-    
-    ## Fore getting non terminals
-    def __get_nt_nodes(self):
-        for nt_node in self.node.findall('nt'):
-            yield nt_node
-
-    def get_non_terminals(self):
-        for nt_node in self.__get_nt_nodes():
-            yield Cnonterminal(nt_node)
-    ##################################
-            
-    ## Fore getting  terminals
-    def __get_t_nodes(self):
-        for t_node in self.node.findall('t'):
-            yield t_node
-
-    def get_terminals(self):
-        for t_node in self.__get_t_nodes():
-            yield Cterminal(t_node)
-    ##################################            
-            
-     ## Fore getting  edges
-    def __get_edge_nodes(self):
-        for t_node in self.node.findall('edge'):
-            yield t_node
-
-    def get_edges(self):
-        for edge_node in self.__get_edge_nodes():
-            yield Cedge(edge_node)
-    ##################################          
-            
-    
-
-class Cconstituency:
-    def __init__(self,node=None):
-        self.type = 'NAF/NAF'
-        if node is None:
-            self.node = etree.Element('constituency')
-        else:
-            self.node = node
-            
-    def to_kaf(self):
-        pass
-    
-    def to_naf(self):
-        pass
-    
-    def __get_tree_nodes(self):
-        for tree_node in self.node.findall('tree'):
-            yield tree_node
-    
-    def get_trees(self):
-        for tree_node in self.__get_tree_nodes():
-            yield Ctree(tree_node)
-            
-    def __str__(self):
-        return dump(self.node)

data/core/site-packages/pre_build/KafNafParser/coreference_data.py

@@ -1,52 +0,0 @@
-from lxml import etree
-from span_data import Cspan
-
-class Ccoreference:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('coref')
-        else:
-            self.node = node
-            
-    def get_id(self):
-        if self.type == 'NAF':
-            return self.node.get('id')
-        elif self.type == 'KAF':
-            return self.node.get('coid')
-    
-    def get_spans(self):
-        for node_span in self.node.findall('span'):
-            yield Cspan(node_span)
-    
-            
-
-class Ccoreferences:
-    def __init__(self,node=None, type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('coreferences')
-        else:
-            self.node = node
-            
-    def __get_corefs_nodes(self):
-        for coref_node in self.node.findall('coref'):
-            yield coref_node
-            
-    def get_corefs(self):
-        for coref_node in self.__get_corefs_nodes():
-            yield Ccoreference(coref_node,self.type)
-            
-    def to_kaf(self):
-        if self.type == 'NAF':
-            for node_coref in self.__get_corefs_nodes():
-                node_coref.set('coid',node_coref.get('id'))
-                del node_coref.attrib['id']
-        
-    def to_naf(self):
-        if self.type == 'KAF':
-            for node_coref in self.__get_corefs_nodes():
-                node_coref.set('id',node_coref.get('coid'))
-                del node_coref.attrib['coid']
-                    
-            

data/core/site-packages/pre_build/KafNafParser/dependency_data.py

@@ -1,78 +0,0 @@
-from lxml import etree
-#from lxml.objectify import dump
-
-
-class Cdependency:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('dep')
-        else:
-            self.node = node
-            
-    def get_node_comment(self):
-        return self.node_comment
-    
-    def get_node(self):
-        return self.node
-    
-    def get_from(self):
-        return self.node.get('from')
-    
-    def get_to(self):
-        return self.node.get('to')
-    
-    def get_function(self):
-        return self.node.get('rfunc')
-
-    def set_from(self, f):
-        self.node.set('from',f)
-    
-    def set_to(self,t):
-        self.node.set('to',t)
-    
-    def set_function(self,f):
-        self.node.set('rfunc',f)
-        
-
-    def set_comment(self,c):
-        c = c.replace('--','- -')
-        self.node.insert(0,etree.Comment(c) )
-            
-    
-    def __str__(self):
-        return dump(self.node)
-
-
-
-class Cdependencies:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('deps')
-        else:
-            self.node = node
-            
-    def get_node(self):
-        return self.node
-            
-    def to_kaf(self):
-        pass
-    
-    def to_naf(self):
-        pass
-            
-    def __str__(self):
-        return dump(self.node)
-
-
-    def __get_node_deps(self):
-        for node_dep in self.node.findall('dep'):
-            yield node_dep
-            
-    def get_dependencies(self):
-        for node in self.__get_node_deps():
-            yield Cdependency(node)
-            
-            
-    def add_dependency(self,my_dep):
-        self.node.append(my_dep.get_node())
-            

data/core/site-packages/pre_build/KafNafParser/entity_data.py

@@ -1,59 +0,0 @@
-## Modified for KAF NAF adaptation
-from lxml import etree
-from lxml.objectify import dump
-from references_data import *
-    
-    
-class Centity:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('entity')
-        else:
-            self.node = node
-            
-    def get_id(self):
-        if self.type == 'NAF':
-            return self.node.get('id')
-        elif self.type == 'KAF':
-            return self.node.get('eid')
-    
-    def get_type(self):
-        return self.node.get('type')
-    
-    def get_references(self):
-        for ref_node in self.node.findall('references'):
-            yield Creferences(ref_node)
-    
-class Centities:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('entities')
-        else:
-            self.node = node
-        
-        
-    def to_kaf(self):
-        if self.type == 'NAF':
-            for node in self.__get_entity_nodes():
-                node.set('eid',node.get('id'))
-                del node.attrib['id']
-
-    def to_naf(self):
-        if self.type == 'KAF':
-            for node in self.__get_entity_nodes():
-                node.set('id',node.get('eid'))
-                del node.attrib['eid']
-                
-    def __get_entity_nodes(self):
-        for ent_node in self.node.findall('entity'):
-            yield ent_node
-                
-    def __iter__(self):
-        for ent_node in self.__get_entity_nodes():
-            yield Centity(ent_node,self.type)
-        
-        
-    def __str__(self):
-        return dump(self.node)

data/core/site-packages/pre_build/KafNafParser/external_references_data.py

@@ -1,41 +0,0 @@
-# included modification for KAF/NAF
-from term_sentiment_data import Cterm_sentiment
-from lxml import etree
-
-class CexternalReference:
-    def __init__(self,node=None):
-        self.type= 'NAF/KAF'
-        #self.resource = self.reference = self.reftype = self.status = self.source = self.confidence = ''
-        if node is None:
-            self.node = etree.Element('externalRef')
-        else:
-            self.node = node
-        
-    def get_node(self):
-        return self.node
-        
-    def set_resource(self,resource):
-        self.node.set('resource',resource)
-    
-    def set_confidence(self,confidence):
-        self.node.set('confidence',confidence)
-    
-    def set_reference(self,reference):
-        self.node.set('reference',reference)
-        
-
-class CexternalReferences:
-    def __init__(self,node=None):
-        if node is None:
-            self.node = etree.Element('externalReferences')
-        else:
-            self.node = node
-                
-    def add_external_reference(self,ext_ref):
-        self.node.append(ext_ref.get_node())
-        
-    def get_node(self):
-        return self.node
-        
- 
-    

data/core/site-packages/pre_build/KafNafParser/feature_extractor/__init__.py

@@ -1,2 +0,0 @@
-from dependency import *
-from constituency import *

data/core/site-packages/pre_build/KafNafParser/feature_extractor/constituency.py

@@ -1,205 +0,0 @@
-#!/usr/bin/env python
-
-from operator import itemgetter
-
-'''
-Extract information from the contituent layer from a NAF file
-'''
-
-class Cconstituency_extractor:
-    def __init__(self,knaf_obj):
-        self.naf = knaf_obj
-        #Extract terminals, non terminals and edges
-        ## Extracted directly from 
-        self.terminals = {}          #terminal id --> list term ids
-        self.terminal_for_term = {}  #term id --> terminal id
-        self.label_for_nonter = {}   # nonter --> label
-        self.reachable_from = {}     # node_from --> [nodeto1, nodeto2...]
-        
-        self.extract_info_from_naf(knaf_obj)
-        
-        #Extracting all posible paths from leave to root for each terminal id
-        self.paths_for_terminal= {}
-        for terminal_id in self.terminals.keys():
-            paths = self.__expand_node(terminal_id,False)
-            self.paths_for_terminal[terminal_id] = paths
-        #######################################
-        
-        ## Create, for each non terminal, which are the terminals subsumed
-        self.terms_subsumed_by_nonter = {}  ## ['nonter12'] = set('t1,'t2','t3','t4')
-        for terminal_id, span_terms in self.terminals.items():
-            for path in self.paths_for_terminal[terminal_id]:
-                for nonter in path:
-                    if nonter not in self.terms_subsumed_by_nonter:
-                        self.terms_subsumed_by_nonter[nonter] = set()
-                    for termid in span_terms:
-                        self.terms_subsumed_by_nonter[nonter].add(termid)
-        
-        ## To print the paths calculated
-#        for terminal in self.terminals.keys():
-#            print terminal
-#            for path in self.paths_for_terminal[terminal]:
-#                sep=' '
-#                for node in path:
-#                    print sep,node,self.label_for_nonter.get(node,'?')
-#                    sep+=' '
-#            print '#'*20
-
-               
-    def get_deepest_phrases(self):
-        all_nonter = set()
-        for terminal in self.terminals.keys():
-            for path in self.paths_for_terminal[terminal]:
-                first_non_ter_phrase = path[1]
-                all_nonter.add(first_non_ter_phrase)
-        
-        ter_for_nonter = {}
-        for nonter in all_nonter:
-            for terminal in self.terminals.keys():
-                for path in self.paths_for_terminal[terminal]:
-                    if nonter in path:
-                        if nonter in ter_for_nonter:
-                            ter_for_nonter[nonter].append(terminal)
-                        else:
-                            ter_for_nonter[nonter] = [terminal]
-        
-        visited = set()
-        for nonter, list_term in ter_for_nonter.items():
-            for ter in list_term:
-
-                visited.add(ter)
-    
-    
-    ### Returns the label of the deepest phrase for the term id (termid as in the term layer)
-    def get_deepest_phrase_for_termid(self,termid):
-        terminal_id = self.terminal_for_term.get(termid)
-        label = None
-        subsumed = []
-        if terminal_id is not None:
-            first_path = self.paths_for_terminal[terminal_id][0]
-            first_phrase_id = first_path[1]
-            label = self.label_for_nonter.get(first_phrase_id)
-            subsumed = self.terms_subsumed_by_nonter.get(first_phrase_id,[])
-        return label,sorted(list(subsumed))
-    
-    
-    def get_least_common_subsumer(self,from_tid,to_tid):
-        termid_from = self.terminal_for_term.get(from_tid)
-        termid_to = self.terminal_for_term.get(to_tid)
-        
-        path_from = self.paths_for_terminal[termid_from][0]
-        path_to = self.paths_for_terminal[termid_to][0]
-        common_nodes = set(path_from) & set(path_to)
-        if len(common_nodes) == 0:
-            return None
-        else:
-            indexes = []
-            for common_node in common_nodes:
-                index1 = path_from.index(common_node)
-                index2 = path_to.index(common_node)
-                indexes.append((common_node,index1+index2))
-            indexes.sort(key=itemgetter(1))
-            shortest_common = indexes[0][0]
-            return shortest_common
-        
-    
-    def get_path_from_to(self,from_tid, to_tid):
-        shortest_subsumer = self.get_least_common_subsumer(from_tid, to_tid)
-        
-        #print 'From:',self.naf.get_term(from_tid).get_lemma()
-        #print 'To:',self.naf.get_term(to_tid).get_lemma()
-        termid_from = self.terminal_for_term.get(from_tid)
-        termid_to = self.terminal_for_term.get(to_tid)
-        
-        path_from = self.paths_for_terminal[termid_from][0]
-        path_to = self.paths_for_terminal[termid_to][0]
-        
-        if shortest_subsumer is None:
-            return None
-        
-        complete_path = []
-        for node in path_from:
-            complete_path.append(node)
-            if node == shortest_subsumer: break
-        
-        begin=False
-        for node in path_to[-1::-1]:
-            if begin:
-                complete_path.append(node)
-             
-            if node==shortest_subsumer:
-                begin=True
-        labels = [self.label_for_nonter[nonter] for nonter in complete_path]
-        return labels
-                
-            
-    def get_path_for_termid(self,termid):
-        terminal_id = self.terminal_for_term.get(termid)
-        paths = self.paths_for_terminal[terminal_id]
-        labels = [self.label_for_nonter[nonter] for nonter in paths[0]]
-        return labels
-        
-    def extract_info_from_naf(self,knaf_obj):
-        ## Generated internally
-        # For each terminal node, a list of paths through all the edges
-        self.paths_for_terminal = {} 
-        for tree in knaf_obj.get_trees():
-            for terminal in tree.get_terminals():
-                ter_id = terminal.get_id()
-                span_ids = terminal.get_span().get_span_ids()
-                self.terminals[ter_id] = span_ids
-                for this_id in span_ids:
-                    self.terminal_for_term[this_id] = ter_id
-            
-            
-            for non_terminal in tree.get_non_terminals():
-                nonter_id = non_terminal.get_id()
-                label = non_terminal.get_label()
-                self.label_for_nonter[nonter_id] = label
-                
-                
-            for edge in tree.get_edges():
-                node_from = edge.get_from()
-                node_to = edge.get_to()
-                if node_from not in self.reachable_from:
-                    self.reachable_from[node_from] = [node_to]
-                else:
-                    self.reachable_from[node_from].append(node_to)
-        
-     
-        
-    ##Recursive function
-    ## Propagates the node through all the relations extracte from the edges information
-    ## It returns a list of lists, one for each path
-    ## Include_this_node is used for avoiding the first node 
-    def __expand_node(self,node,include_this_node=True):
-        paths = []
-        possible_nodes = self.reachable_from.get(node,[])
-        if len(possible_nodes) == 0:
-            return [[node]]
-        else:
-            for possible_node in possible_nodes:
-                new_paths = self.__expand_node(possible_node)
-                for path in new_paths:
-                    if include_this_node: 
-                        path.insert(0,node)
-                    paths.append(path)
-            return paths
-       
-    def get_chunks(self,chunk_type):
-        for nonter,this_type in self.label_for_nonter.items():
-            if this_type == chunk_type:
-                subsumed = self.terms_subsumed_by_nonter.get(nonter)
-                if subsumed is not None:
-                    yield sorted(list(subsumed))
-                    
-    def get_all_chunks_for_term(self,termid):
-        terminal_id = self.terminal_for_term.get(termid)
-        paths = self.paths_for_terminal[terminal_id]
-        for path in paths:
-            for node in path:
-                this_type = self.label_for_nonter[node]
-                subsumed =  self.terms_subsumed_by_nonter.get(node)
-                if subsumed is not None:
-                    yield this_type,sorted(list(subsumed))
-