RubyGems - opener-opinion-detector-base - Versions diffs - 2.0.1 → 2.1.2 - Mend

opener-opinion-detector-base 2.0.1 → 2.1.2

Files changed (67) hide show

data/core/site-packages/pre_build/KafNafParser/feature_extractor/dependency.py DELETED Viewed

@@ -1,309 +0,0 @@
-from operator import itemgetter
-from VUA_pylib.common import get_max_distr_dict
-import sys
-class Cdependency_extractor:
-    def __init__(self,knaf_obj):
-        self.naf = knaf_obj
-        self.relations_for_term = {}
-        self.reverse_relations_for_term = {}
-        self.prefix_for_reverse = ''
-        already_linked = {}
-        for dep in knaf_obj.get_dependencies():
-            term_from = dep.get_from()
-            term_to = dep.get_to()
-            rfunc = dep.get_function()
-            # Dependencies reversed are skipped...
-            #if rfunc.startswith('rhd/') or rfunc.startswith('whd/'):
-            #    continue
-            #  For detecting cycles like:
-            #       <!-- rhd/body(geef,wat) -->
-            #       <dep from="t19" to="t15" rfunc="rhd/body"/>
-            #       <!-- hd/su(wat,geef) -->
-            #      <dep from="t15" to="t19" rfunc="hd/su"/>
-            '''
-            if term_from in already_linked and term_to in already_linked[term_from]:
-                #There could be a cycle, skip this
-                print>>sys.stderr,'Skipped from',term_from,'to',term_to,'func',rfunc,' cycle detected'
-                continue
-            else:
-                #Include term_from as linked with term_to for future...
-                if term_to not in already_linked:
-                    already_linked[term_to] = set()
-                already_linked[term_to].add(term_from)
-            '''
-            if term_from in self.relations_for_term:
-                self.relations_for_term[term_from].append((rfunc,term_to))
-            else:
-                self.relations_for_term[term_from] = [(rfunc,term_to)]
-            if term_to in self.reverse_relations_for_term:
-                self.reverse_relations_for_term[term_to].append((self.prefix_for_reverse+rfunc,term_from))
-            else:
-                self.reverse_relations_for_term[term_to] = [(self.prefix_for_reverse+rfunc,term_from)]
-        self.paths_for_termid={}
-        self.sentence_for_termid={}
-        self.top_relation_for_term = {}     ## termid --> (relation,topnode)
-        self.root_for_sentence = {}         ## sentenceid --> termid
-        for term_obj in knaf_obj.get_terms():
-            termid = term_obj.get_id()
-            #Calculating the sentence id for the term id
-            span_ids = term_obj.get_span().get_span_ids()
-            token_obj = knaf_obj.get_token(span_ids[0])
-            if token_obj is None:
-                continue
-            sentence = token_obj.get_sent()
-            self.sentence_for_termid[termid] = sentence
-            ###########################################
-            #paths = self.__propagate_node(termid,[])
-            #inversed = self.__reverse_propagate_node(termid)
-            ## Due to the change on direction of dependencies...
-            inversed = self.__propagate_node(termid,already_propagated=[])
-            paths = self.__reverse_propagate_node(termid,already_propagated=[])
-            ##Calculate the top relation for the node, the relation with the main root of the tree
-            if len(inversed) != 0:
-                for ip in inversed:
-                    if len(ip)!=0:
-                        self.top_relation_for_term[termid] = ip[-1] ## ex. ('NMOD', 't2')
-                        root = ip[-1][1]
-                        if sentence not in self.root_for_sentence:
-                            self.root_for_sentence[sentence] = {}
-                        if root not in self.root_for_sentence[sentence]:
-                            self.root_for_sentence[sentence][root]=0
-                        else:
-                            self.root_for_sentence[sentence][root]+=1
-                        break
-            self.paths_for_termid[termid] = paths + inversed
-            '''
-            print termid
-            print 'DIRECT RELS'
-            for p in paths:
-                print ' ',p
-            print 'INDIRECT RELS'
-            for p in inversed:
-                print ' ',p
-            '''
-        ####
-        for sent_id, distr in self.root_for_sentence.items():
-            ## get_max_distr_dict imported from VUA_pylib.common
-            most_freq,c = get_max_distr_dict(distr)
-            self.root_for_sentence[sent_id] = most_freq
-    def __propagate_node(self,node,already_propagated=[]):
-        paths = []
-        relations = self.relations_for_term.get(node)
-        #print 'Propagate ',node,relations
-        if relations is None:   ##Case base
-            paths = [[]]
-        elif node in already_propagated:
-            paths = [[]]
-        else:
-            already_propagated.append(node)
-            for func, target_node in relations:
-                new_paths = self.__propagate_node(target_node, already_propagated)
-                for new_path in new_paths:
-                    new_path.insert(0,(func,target_node))
-                    paths.append(new_path)
-        return paths
-    def __reverse_propagate_node(self,node,already_propagated=[]):
-        paths = []
-        relations = self.reverse_relations_for_term.get(node)
-        #print 'Propagate reverse',node,relations,already_propagated
-        if relations is None:   ##Case base
-            paths = [[]]
-        elif node in already_propagated:
-            paths = [[]]
-        else:
-            already_propagated.append(node)
-            for func, target_node in relations:
-                new_paths = self.__reverse_propagate_node(target_node,already_propagated)
-                for new_path in new_paths:
-                    new_path.insert(0,(func,target_node))
-                    paths.append(new_path)
-        return paths
-    # Get the shortest path between 2 term ids
-    def get_shortest_path(self,term1,term2):
-        dep_path = None
-        if term1 == term2: dep_path = []
-        else:
-            paths1 = self.paths_for_termid[term1]
-            paths2 = self.paths_for_termid[term2]
-            ##Check if term2 is on paths1
-            hits = [] ## list of (common_id,idx1,idx2,numpath1,numpath2)
-            for num1, p1 in enumerate(paths1):
-                ids1 = [ my_id for my_func, my_id in p1]
-                if term2 in ids1:
-                    idx1 = ids1.index(term2)
-                    hits.append((term2,idx1+0,idx1,0,num1,None))
-            for num2,p2 in enumerate(paths2):
-                ids2 = [ my_id for my_func, my_id in p2]
-                if term1 in ids2:
-                    idx2=ids2.index(term1)
-                    hits.append((term1,0+idx2,0,idx2,None,num2))
-            #Pair by pair
-            for num1, p1 in enumerate(paths1):
-                #print 'Path1',term1, p1
-                ids1 = [ my_id for my_func, my_id in p1]
-                #print 'IDS1',ids1
-                for num2, p2 in enumerate(paths2):
-                    #print '\t',term2,p2
-                    ids2 = [ my_id for my_func, my_id in p2]
-                    #print '  IDS2',ids2
-                    common_ids = set(ids1) & set(ids2)
-                    #print '  cmmon',common_ids
-                    for common_id in common_ids:
-                        idx1 = ids1.index(common_id)
-                        idx2 = ids2.index(common_id)
-                        hits.append((common_id,idx1+idx2,idx1,idx2,num1,num2))
-            if len(hits) != 0:
-                dep_path = []
-                hits.sort(key=itemgetter(1))
-                best_hit = hits[0]
-                common_id, _, idx1, idx2, numpath1, numpath2 = best_hit
-                if numpath2 is None:  #term2 is in one of the paths of t1
-                    path1 = paths1[numpath1]
-                    my_rels1 = path1[:idx1+1]
-                    ##complete_path = ''
-                    ##complete_path_ids = ''
-                    for func,node in my_rels1:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        ##complete_path_ids+=node+'#'
-                    #===========================================================
-                    # print 'CASE1',best_hit
-                    # print complete_path
-                    # print complete_path_ids
-                    #===========================================================
-                elif numpath1 is None: #term1 is in one of the paths of t2
-                    path2 = paths2[numpath2]
-                    my_rels2 = path2[:idx2+1]
-                    ##complete_path = ''
-                    ##complete_path_ids = ''
-                    for func,node in my_rels2:
-                        dep_path.append(func)
-                        #complete_path+=func+'#'
-                        #complete_path_ids+=node+'#'
-                    #===========================================================
-                    # print 'CASE2',best_hit
-                    # print complete_path
-                    # print complete_path_ids
-                    #===========================================================
-                else:   #There is a common node linking both
-                    path1 = paths1[numpath1]
-                    my_rels1 = path1[:idx1+1]
-                    path2 = paths2[numpath2]
-                    my_rels2 = path2[:idx2+1]
-                    ##complete_path = ''
-                    #complete_path_ids = ''
-                    for func,node in my_rels1:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
-                    for func,node in my_rels2[-1::-1]:
-                        dep_path.append(func)
-                        ##complete_path+=func+'#'
-                        #complete_path_ids+=func+'->'+self.naf.get_term(node).get_lemma()+'->'
-                    #===========================================================
-                    #
-                    # print complete_path
-                    # print complete_path_ids
-                    # print path2
-                    # print my_rels1
-                    # print my_rels2
-                    # print 'CASE3',best_hit
-                    #===========================================================
-        return dep_path
-    ## Get the shortest dependency path between 2 sets of spans
-    def get_shortest_path_spans(self,span1,span2):
-        shortest_path = None
-        for term1 in span1:
-            for term2 in span2:
-                this_path = self.get_shortest_path(term1, term2)
-                #print term1,term2, this_path
-                if shortest_path is None or (this_path is not None and len(this_path)<len(shortest_path)):
-                    shortest_path = this_path
-        return shortest_path
-    # Get the dependency path to the sentence root for a term id
-    def get_path_to_root(self,termid):
-        # Get the sentence for the term
-        root = None
-        sentence = self.sentence_for_termid.get(termid)
-        if sentence is None:    #try with the top node
-            top_node = self.top_relation_for_term.get(termid)
-            if top_node is not None:
-                root = top_node[1]
-            else:
-                return None
-        else:
-            if sentence in self.root_for_sentence:
-                root = self.root_for_sentence[sentence]
-            else:
-                ##There is no root for this sentence
-                return None
-        # In this point top_node should be properly set
-        path = self.get_shortest_path(termid, root)
-        return path
-    # Get the shortest dependency path to the sentence root for a span of ids
-    # extractor.get_shortest_path_to_root_span(['t444','t445','t446'])
-    def get_shortest_path_to_root_span(self,span):
-        shortest_path = None
-        for termid in span:
-            this_path = self.get_path_to_root(termid)
-            ## In case of , or . or whatever, the path to the root usually is None, there are no dependencies...
-            if shortest_path is None or (this_path is not None and len(this_path) < len(shortest_path)):
-                shortest_path = this_path
-        return shortest_path

data/core/site-packages/pre_build/KafNafParser/features_data.py DELETED Viewed

@@ -1,131 +0,0 @@
-from lxml import etree
-from lxml.objectify import dump
-from references_data import *
-class Cproperty:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('property')
-        else:
-            self.node = node
-    def get_node(self):
-        return self.node
-    def get_id(self):
-        if self.type == 'KAF':
-            return self.node.get('pid')
-        elif self.type == 'NAF':
-            return self.node.get('id')
-    def set_id(self,pid):
-        if self.type == 'KAF':
-            return self.node.set('pid',pid)
-        elif self.type == 'NAF':
-            return self.node.set('id',pid)
-    def get_type(self):
-        return self.node.get('lemma')
-    def set_type(self,t):
-        return self.node.set('lemma',t)
-    def get_references(self):
-        for ref_node in self.node.findall('references'):
-            yield Creferences(ref_node)
-    def set_reference(self,ref):
-        self.node.append(ref.get_node())
-class Cproperties:
-    def __init__(self,node=None,type='NAF'):
-        self.type=type
-        if node is None:
-            self.node = etree.Element('properties')
-        else:
-            self.node = node
-    def get_node(self):
-        return self.node
-    def __iter__(self):
-        for prop_node in self.node.findall('property'):
-            yield Cproperty(prop_node,self.type)
-    def add_property(self,pid, label,term_span):
-        new_property = Cproperty(type=self.type)
-        self.node.append(new_property.get_node())
-        ##Set the id
-        if pid is None:
-            ##Generate a new pid
-            existing_pids = [property.get_id() for property in self]
-            n = 0
-            new_pid = ''
-            while True:
-                new_pid = 'p'+str(n)
-                if new_pid not in existing_pids: break
-                n += 1
-            pid = new_pid
-        new_property.set_id(pid)
-        new_property.set_type(label)
-        new_ref = Creferences()
-        new_ref.add_span(term_span)
-        new_property.set_reference(new_ref)
-class Cfeatures:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            self.node = etree.Element('features')
-        else:
-            self.node = node
-    def get_node(self):
-        return self.node
-    def to_kaf(self):
-        if self.type == 'NAF':
-            ##convert all the properties
-            for node in self.node.findall('properties/property'):
-                node.set('pid',node.get('id'))
-                del node.attrib['id']
-    def to_naf(self):
-        if self.type == 'KAF':
-            ##convert all the properties
-            for node in self.node.findall('properties/property'):
-                node.set('id',node.get('pid'))
-                del node.attrib['pid']
-    def add_property(self,pid, label,term_span):
-        node_prop = self.node.find('properties')
-        if node_prop is None:
-            properties = Cproperties(type=self.type)
-            self.node.append(properties.get_node())
-        else:
-            properties = Cproperties(node=node_prop,type=self.type)
-        properties.add_property(pid, label,term_span)
-    def get_properties(self):
-        node_prop = self.node.find('properties')
-        if node_prop is not None:
-            obj_properties = Cproperties(node_prop,self.type)
-            for prop in obj_properties:
-                yield prop
-    def remove_properties(self):
-        node_prop = self.node.find('properties')
-        if node_prop is not None:
-            self.node.remove(node_prop)

data/core/site-packages/pre_build/KafNafParser/header_data.py DELETED Viewed

@@ -1,127 +0,0 @@
-# Modified to KAF / NAF
-from lxml import etree
-import time
-class CfileDesc:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('fileDesc')
-        else:
-            self.node = node
-    #self.title=''    #self.author=''    #self.creationtime=''    #self.filename=''    #self.filetype=''    #self.pages=''
-class Cpublic:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('public')
-        else:
-            self.node = node
-        #self.publicId = ''
-        #slf.uri = ''
-class Clp:
-    def __init__(self,node=None,name="",version="",timestamp=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('lp')
-            self.set_name(name)
-            self.set_version(version)
-            self.set_timestamp(timestamp)
-        else:
-            self.node = node
-    def set_name(self,name):
-        self.node.set('name',name)
-    def set_version(self,version):
-        self.node.set('version',version)
-    def set_timestamp(self,timestamp=None):
-        if timestamp is None:
-            import time
-            timestamp = time.strftime('%Y-%m-%dT%H:%M:%S%Z')
-        self.node.set('timestamp',timestamp)
-    def get_node(self):
-        return self.node
-class ClinguisticProcessors:
-    def __init__(self,node=None):
-        self.type = 'KAF/NAF'
-        if node is None:
-            self.node = etree.Element('linguisticProcessors')
-        else:
-            self.node = node
-    def get_layer(self):
-        return self.node.get('layer')
-    def set_layer(self,layer):
-        self.node.set('layer',layer)
-    def add_linguistic_processor(self,my_lp):
-        self.node.append(my_lp.get_node())
-    def get_node(self):
-        return self.node
-class CHeader:
-    def __init__(self,node=None,type='NAF'):
-        self.type = type
-        if node is None:
-            if self.type == 'NAF':
-                self.node = etree.Element('nafHeader')
-            elif self.type == 'KAF':
-                self.node = etree.Element('kafHeader')
-        else:
-            self.node = node
-    def to_kaf(self):
-        if self.type == 'NAF':
-            self.node.tag = 'kafHeader'
-            self.type = 'KAF'
-    def to_naf(self):
-        if self.type == 'KAF':
-            self.node.tag = 'nafHeader'
-            self.type = 'NAF'
-    def add_linguistic_processors(self,linpro):
-        self.node.append(linpro.get_node())
-    def remove_lp(self,layer):
-        for this_node in self.node.findall('linguisticProcessors'):
-            if this_node.get('layer') == layer:
-                self.node.remove(this_node)
-                break
-    def add_linguistic_processor(self, layer ,my_lp):
-        ## Locate the linguisticProcessor element for taht layer
-        found_lp_obj = None
-        for this_lp in self.node.findall('linguisticProcessors'):
-            lp_obj = ClinguisticProcessors(this_lp)
-            if lp_obj.get_layer() == layer:
-                found_lp_obj = lp_obj
-                break
-        if found_lp_obj is None:    #Not found
-            found_lp_obj = ClinguisticProcessors()
-            found_lp_obj.set_layer(layer)
-            self.add_linguistic_processors(found_lp_obj)
-        found_lp_obj.add_linguistic_processor(my_lp)