opener-opinion-detector-base 2.0.0

data/core/python-scripts/scripts/crfutils.py

@@ -0,0 +1,215 @@
+import optparse
+import sys
+
+
+
+def feature_extractor(X,templates):
+    # Apply attribute templates to obtain features (in fact, attributes)
+    apply_templates(X, templates)
+    if X:
+        # Append BOS and EOS features manually
+        X[0]['F'].append('__BOS__')     # BOS feature
+        X[-1]['F'].append('__EOS__')    # EOS feature
+
+
+
+def extract_features_to_crf(inputfile,outputfile,fields,separator,templates,possible_classes=None):
+  fi = open(inputfile,'r')
+  fo = open(outputfile,'w')
+  #this_main(feature_extractor,templates=templates,fields=fields,sep=separator,fi=fi,fo=fo,possible_classes=possible_classes)
+  
+  F = fields.split(' ')
+  for X in readiter(fi, F, separator):
+      feature_extractor(X,templates)
+      output_features(fo, X, 'y',possible_classes)
+
+  fi.close()
+  fo.close()
+
+def apply_templates(X, templates):
+
+    """
+    Generate features for an item sequence by applying feature templates.
+    A feature template consists of a tuple of (name, offset) pairs,
+    where name and offset specify a field name and offset from which
+    the template extracts a feature value. Generated features are stored
+    in the 'F' field of each item in the sequence.
+
+    @type   X:      list of mapping objects
+    @param  X:      The item sequence.
+    @type   template:   tuple of (str, int)
+    @param  template:   The feature template.
+    """
+    name=''
+    for template in templates:
+        '''
+        name=''
+        for f, o in template:
+          if o<0: o=-1
+          elif o>0: o=1
+          name += f+'['+str(o)+']'+'|'
+        name = name[:-1]  
+        '''
+        name = '|'.join(['%s[%d]' % (f, o) for f, o in template])
+        for t in range(len(X)):
+            values = []
+            for field, offset in template:
+                p = t + offset
+                if p not in range(len(X)):
+                    values = []
+                    break
+                value = X[p][field]
+                if value != '' and value !='-':
+                  values.append(value)
+            if values and len(values)>0 :
+                X[t]['F'].append('%s=%s' % (name, '|'.join(values)))
+
+def readiter(fi, names, sep=' '):
+    """
+    Return an iterator for item sequences read from a file object.
+    This function reads a sequence from a file object L{fi}, and
+    yields the sequence as a list of mapping objects. Each line
+    (item) from the file object is split by the separator character
+    L{sep}. Separated values of the item are named by L{names},
+    and stored in a mapping object. Every item has a field 'F' that
+    is reserved for storing features.
+
+    @type   fi:     file
+    @param  fi:     The file object.
+    @type   names:  tuple
+    @param  names:  The list of field names.
+    @type   sep:    str
+    @param  sep:    The separator character.
+    @rtype          list of mapping objects
+    @return         An iterator for sequences.
+    """
+    X = []
+    for line in fi:
+        line = line.strip('\n')
+        if not line:
+            yield X
+            X = []
+        else:
+            fields = line.split(sep)
+            if len(fields) < len(names):
+                raise ValueError(
+                    'Too few fields (%d) for %r\n%s' % (len(fields), names, line))
+            item = {'F': []}    # 'F' is reserved for features.
+            for i in range(len(names)):
+                item[names[i]] = fields[i]
+            X.append(item)
+
+def escape(src):
+    """
+    Escape colon characters from feature names.
+
+    @type   src:    str
+    @param  src:    A feature name
+    @rtype          str
+    @return         The feature name escaped.
+    """
+    return src.replace(':', '__COLON__')
+
+def output_features(fo, X, field='', possible_classes=None):
+    """
+    Output features (and reference labels) of a sequence in CRFSuite
+    format. For each item in the sequence, this function writes a
+    reference label (if L{field} is a non-empty string) and features.
+
+    @type   fo:     file
+    @param  fo:     The file object.
+    @type   X:      list of mapping objects
+    @param  X:      The sequence.
+    @type   field:  str
+    @param  field:  The field name of reference labels.
+    """
+    for t in range(len(X)):
+        if field:
+            this_class = X[t][field]   ### B-positive
+            general_class = this_class[2:]
+            if possible_classes is not None and general_class not in possible_classes:
+                this_class = 'O'                
+            fo.write('%s' % this_class)
+        for a in X[t]['F']:
+            if isinstance(a, str):
+                fo.write('\t%s' % escape(a))
+            else:
+                fo.write('\t%s:%f' % (escape(a[0]), a[1]))
+        fo.write('\n')
+    fo.write('\n')
+
+def to_crfsuite(X):
+    """
+    Convert an item sequence into an object compatible with crfsuite
+    Python module.
+
+    @type   X:      list of mapping objects
+    @param  X:      The sequence.
+    @rtype          crfsuite.ItemSequence
+    @return        The same sequence in crfsuite.ItemSequence type.
+    """
+    import crfsuite
+    xseq = crfsuite.ItemSequence()
+    for x in X:
+        item = crfsuite.Item()
+        for f in x['F']:
+            if isinstance(f, str):
+                item.append(crfsuite.Attribute(escape(f)))
+            else:
+                item.append(crfsuite.Attribute(escape(f[0]), f[1]))
+        xseq.append(item)
+    return xseq
+
+'''
+def this_main(feature_extractor, templates,fields='w pos y', sep=' ',fi = sys.stdin,fo = sys.stdout,possible_classes=None):
+
+    # Parse the command-line arguments.
+    parser = optparse.OptionParser(usage="""usage: %prog [options]
+This utility reads a data set from STDIN, and outputs attributes to STDOUT.
+Each line of a data set must consist of field values separated by SEPARATOR
+characters. The names and order of field values can be specified by -f option.
+The separator character can be specified with -s option. Instead of outputting
+attributes, this utility tags the input data when a model file is specified by
+-t option (CRFsuite Python module must be installed)."""
+        )
+    parser.add_option(
+        '-t', dest='model',
+        help='tag the input using the model (requires "crfsuite" module)'
+        )
+    parser.add_option(
+        '-f', dest='fields', default=fields,
+        help='specify field names of input data [default: "%default"]'
+        )
+    parser.add_option(
+        '-s', dest='separator', default=sep,
+        help='specify the separator of columns of input data [default: "%default"]'
+        )
+    (options, args) = parser.parse_args()
+
+    # The fields of input: ('w', 'pos', 'y) by default.
+    F = options.fields.split(' ')
+
+    if not options.model:
+        # The generator function readiter() reads a sequence from a 
+        for X in readiter(fi, F, options.separator):
+            feature_extractor(X,templates)
+            output_features(fo, X, 'y',possible_classes)
+
+    else:
+        # Create a tagger with an existing model.
+        import crfsuite
+        tagger = crfsuite.Tagger()
+        tagger.open(options.model)
+
+        # For each sequence from STDIN.
+        for X in readiter(fi, F, options.separator):
+            # Obtain features.
+            feature_extractor(X,templates)
+            xseq = to_crfsuite(X)
+            yseq = tagger.tag(xseq)
+            for t in range(len(X)):
+                v = X[t]
+                fo.write('\t'.join([v[f] for f in F]))
+                fo.write('\t%s\n' % yseq[t])
+            fo.write('\n')
+'''

data/core/python-scripts/scripts/extract_feats_relations.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python
+
+import sys
+
+def write_to_output(my_class,feats, output):
+    my_str = my_class
+    for name, value in feats:
+        my_str += '\t'+name+'='+value
+    output.write(my_str.encode('utf-8')+'\n')
+    
+    
+    
+#########################################################################   
+# EXTRACTION OF FEATURES FOR TRAINING THE RELATION CLASSIFIER EXP --> TARGET
+#########################################################################  
+# This function extracts features for the relation between expression adn target
+# for the svm classifier
+def extract_feats_exp_tar(exp_ids,tar_ids,knaf_obj, use_lemmas=True, use_tokens=True, use_dependencies=True):
+    all_feats = []
+    
+    data_for_token = {}     # [token_id] -> (word, sentence_id)
+    for num_token, token_obj in enumerate(knaf_obj.get_tokens()):
+        word = token_obj.get_text()
+        s_id = token_obj.get_sent()
+        w_id = token_obj.get_id()
+        
+        data_for_token[w_id] = (word,s_id,num_token)
+    
+    # Loading data for terms
+    data_for_term = {}      # [term_id] -> (lemma, span_token_ids)
+    for term in knaf_obj.get_terms():
+        termid = term.get_id()
+        lemma = term.get_lemma()
+        span = term.get_span()
+        span_token_ids = []
+        if span is not None:
+            span_token_ids = span.get_span_ids()
+        data_for_term[termid] = (lemma,span_token_ids)
+
+    sentence_for_exp = None
+    avg_position_exp = 0
+    n_toks = 0
+    for my_id in exp_ids:
+        lemma, span_tok_ids = data_for_term[my_id]
+        if use_lemmas:
+            all_feats.append(('lemmaExp',lemma))
+                
+                
+        for tok_id in span_tok_ids:
+            token,sent_id,num_token = data_for_token[tok_id]
+            avg_position_exp += num_token
+            n_toks += 1
+            if use_tokens:
+                all_feats.append(('tokenExp',token))
+            
+            if sentence_for_exp is None:
+                sentence_for_exp = sent_id
+                
+    avg_position_exp = avg_position_exp * 1.0 / n_toks  
+
+    #Lemmas for target    
+    sentence_for_tar = None
+    avg_position_tar = 0
+    n_toks = 0
+    for my_id in tar_ids:
+        lemma, span_tok_ids = data_for_term[my_id]
+        if use_lemmas:
+            all_feats.append(('lemmaTar',lemma))
+                
+        for tok_id in span_tok_ids:
+            token,sent_id,num_token = data_for_token[tok_id]
+            avg_position_tar += num_token
+            n_toks += 1
+            if use_tokens:
+                all_feats.append(('tokenTar',token))
+
+            if sentence_for_tar is None:
+                sentence_for_tar = sent_id
+    
+    avg_position_tar = avg_position_tar * 1.0 / n_toks  
+    
+    if use_dependencies:
+        dependency_extractor = knaf_obj.get_dependency_extractor()
+        if dependency_extractor is not None:
+            deps = dependency_extractor.get_shortest_path_spans(exp_ids,tar_ids)
+            if deps is not None:
+                all_feats.append(('deps-exp-tar','#'.join(deps)))
+    
+  
+    if sentence_for_exp is not None and sentence_for_tar is not None and sentence_for_exp == sentence_for_tar:
+        all_feats.append(('same_sentence','yes'))
+    else:
+        all_feats.append(('same_sentence','no'))
+        
+    ##Distance
+    dist = abs(avg_position_exp - avg_position_tar)
+    if dist <= 10:
+        my_dist = 'veryclose'
+    elif dist <=20:
+        my_dist  = 'close'
+    elif dist <=25:
+        my_dist = 'far'
+    else:
+        my_dist = 'veryfar'
+    all_feats.append(('distExpTar',my_dist))
+  
+    return all_feats
+
+    
+      
+  
+        
+    
+def create_rel_exp_tar_training(knaf_obj, output=sys.stdout, valid_opinions=None,use_dependencies=True,use_tokens=True, use_lemmas=True):
+    # Obtain pairs of features for Expression and Target
+    pairs = [] # [(Exp,Tar), (E,T), (E,T)....]
+    for opinion in knaf_obj.get_opinions():
+        opi_id = opinion.get_id()
+        opi_exp = opinion.get_expression()
+        exp_type = ''
+        exp_ids = []
+        if opi_exp is not None:
+            exp_type = opi_exp.get_polarity()
+            span = opi_exp.get_span()
+            if span is not None:
+                exp_ids = span.get_span_ids()
+
+        opi_tar = opinion.get_target()
+        tar_ids = []
+        if opi_tar is not None:
+            span = opi_tar.get_span()
+            if span is not None:
+                tar_ids = span.get_span_ids()
+        
+       
+        if valid_opinions is not None:
+            if exp_type not in valid_opinions:
+                continue    ## This opinions will not be used
+        
+        
+        if len(tar_ids) != 0 and len(exp_ids) != 0:
+            pairs.append((exp_ids,tar_ids))
+
+            
+    #extract_feats_exp_tar(exp_ids,tar_ids,knaf_obj, use_lemmas=True, use_tokens=True, use_dependencies=True)
+    for idx1, (exp1, tar1) in enumerate(pairs):
+        feats_positive = extract_feats_exp_tar(exp1,tar1,knaf_obj,use_dependencies=use_dependencies, use_tokens=use_tokens,use_lemmas=use_lemmas)
+        write_to_output('+1', feats_positive, output)
+        for idx2, (exp2, tar2) in enumerate(pairs):
+            if idx1 != idx2:
+                feats_negative = extract_feats_exp_tar(exp1,tar2,knaf_obj,use_dependencies=use_dependencies, use_tokens=use_tokens,use_lemmas=use_lemmas)
+                write_to_output('-1', feats_negative, output)
+                      
+    
+
+
+
+
+def extract_feats_exp_hol(exp_ids,hol_ids,knaf_obj, use_lemmas=True, use_tokens=True, use_dependencies=True):
+    all_feats = []
+    
+    data_for_token = {}     # [token_id] -> (word, sentence_id)
+    for num_token, token_obj in enumerate(knaf_obj.get_tokens()):
+        word = token_obj.get_text()
+        s_id = token_obj.get_sent()
+        w_id = token_obj.get_id()
+        
+        data_for_token[w_id] = (word,s_id,num_token)
+    
+    # Loading data for terms
+    data_for_term = {}      # [term_id] -> (lemma, span_token_ids)
+    for term in knaf_obj.get_terms():
+        termid = term.get_id()
+        lemma = term.get_lemma()
+        span = term.get_span()
+        span_token_ids = []
+        if span is not None:
+            span_token_ids = span.get_span_ids()
+        data_for_term[termid] = (lemma,span_token_ids)
+
+    sentence_for_exp = None
+    avg_position_exp = 0
+    n_toks = 0
+    for my_id in exp_ids:
+        lemma, span_tok_ids = data_for_term[my_id]
+        if use_lemmas:
+            all_feats.append(('lemmaExp',lemma))
+                
+                
+        for tok_id in span_tok_ids:
+            token,sent_id,num_token = data_for_token[tok_id]
+            avg_position_exp += num_token
+            n_toks += 1
+            if use_tokens:
+                all_feats.append(('tokenExp',token))
+            
+            if sentence_for_exp is None:
+                sentence_for_exp = sent_id
+                
+    avg_position_exp = avg_position_exp * 1.0 / n_toks  
+
+    #Lemmas for HOLDER    
+    sentence_for_hol = None
+    avg_position_hol = 0
+    n_toks = 0
+    for my_id in hol_ids:
+        lemma, span_tok_ids = data_for_term[my_id]
+        if use_lemmas:
+            all_feats.append(('lemmaHol',lemma))
+                
+        for tok_id in span_tok_ids:
+            token,sent_id,num_token = data_for_token[tok_id]
+            avg_position_hol += num_token
+            n_toks += 1
+            if use_tokens:
+                all_feats.append(('tokenHol',token))
+
+            if sentence_for_hol is None:
+                sentence_for_hol = sent_id
+                
+    avg_position_hol = avg_position_hol * 1.0 / n_toks
+    
+    if use_dependencies:
+        dependency_extractor = knaf_obj.get_dependency_extractor()
+        if dependency_extractor is not None:
+            deps = dependency_extractor.get_shortest_path_spans(exp_ids,hol_ids)
+            if deps is not None:
+                all_feats.append(('deps-exp-hol','#'.join(deps)))
+    
+  
+    if sentence_for_exp is not None and sentence_for_hol is not None and sentence_for_exp == sentence_for_hol:
+        all_feats.append(('same_sentence','yes'))
+    else:
+        all_feats.append(('same_sentence','no'))
+        
+    ##Distance
+    dist = abs(avg_position_exp - avg_position_hol)
+    if dist <= 10:
+        my_dist = 'veryclose'
+    elif dist <=20:
+        my_dist  = 'close'
+    elif dist <=25:
+        my_dist = 'far'
+    else:
+        my_dist = 'veryfar'
+    all_feats.append(('distExpHol',my_dist))
+    #all_feats.append(('absDist',str(dist))) 
+  
+    return all_feats
+  
+                
+    
+def create_rel_exp_hol_training(knaf_obj, output=sys.stdout, valid_opinions=None,use_dependencies=True,use_tokens=True,use_lemmas=True):
+       
+    # Obtain pairs of features for Expression and Holder
+    pairs = [] # [(Exp,Hol), (E,H), (E,H)....]
+    for opinion in knaf_obj.get_opinions():
+        opi_exp = opinion.get_expression()
+        exp_type = ''
+        exp_ids = []
+        if opi_exp is not None:
+            exp_type = opi_exp.get_polarity()
+            span = opi_exp.get_span()
+            if span is not None:
+                exp_ids = span.get_span_ids()
+
+        opi_hol = opinion.get_holder()
+        hol_ids = []
+        if opi_hol is not None:
+            span = opi_hol.get_span()
+            if span is not None:
+                hol_ids = span.get_span_ids()
+                
+        
+        if valid_opinions is not None:
+            if exp_type not in valid_opinions:
+                continue    ## This opinions will not be used
+        
+        
+        if len(exp_ids) != 0 and len(hol_ids) != 0:
+            pairs.append((exp_ids,hol_ids))
+            
+            
+    #for feat_exp, feat_tar
+    for idx1, (expids1, tarids1) in enumerate(pairs):
+        
+        feats_positive = extract_feats_exp_hol(expids1,tarids1,knaf_obj, use_dependencies=use_dependencies,use_tokens=use_tokens,use_lemmas=use_lemmas)
+        write_to_output('+1', feats_positive,output)
+        
+        for idx2, (expids2, tarids2) in enumerate(pairs):
+            if idx1 != idx2:
+                feats_negative = extract_feats_exp_hol(expids1,tarids2,knaf_obj, use_dependencies=use_dependencies,use_tokens=use_tokens,use_lemmas=use_lemmas)
+                write_to_output('-1', feats_negative ,output)
+    
+