opener-kaf-naf-parser 1.0.0

data/core/site-packages/pre_build/KafNafParser/text_data.py

@@ -0,0 +1,90 @@
+# included code for NAF/KAF
+
+from lxml import etree
+
+
+class Cwf:
+    def __init__(self,node=None,type='NAF'):
+        self.type = type
+        ##self.id = ''    self.sent = ''      self.para = ''      self.page = ''      self.offset = ''      self.lenght = '' s
+        if node is None:
+            self.node = etree.Element('wf')
+        else:
+            self.node = node
+
+    def get_node(self):
+        return self.node
+    
+    def set_id(self,this_id):
+        if self.type == 'NAF':
+            return self.node.set('id',this_id)
+        elif self.type == 'KAF':
+            return self.node.set('wid',this_id)
+                
+    def get_id(self):
+        if self.type == 'NAF':
+            return self.node.get('id')
+        elif self.type == 'KAF':
+            return self.node.get('wid')
+    
+    def set_text(self,this_text):
+        self.node.text = this_text
+        
+    def get_text(self):
+        return self.node.text
+    
+    def set_sent(self,this_sent):
+        self.node.set('sent',this_sent)
+        
+    def get_sent(self):
+        return self.node.get('sent')
+    
+    
+class Ctext:
+    def __init__(self,node=None,type='NAF'):
+        self.idx = {}
+        self.type = type
+        if node is None:
+            self.node = etree.Element('text')
+        else:
+            self.node = node
+            for wf_node in self.__get_wf_nodes():
+                if self.type == 'NAF': label_id = 'id'
+                elif self.type == 'KAF': label_id = 'wid'
+                self.idx[wf_node.get(label_id)] = wf_node
+                
+    def get_node(self):
+        return self.node
+    
+    def to_kaf(self):
+        if self.type == 'NAF':
+            self.type = 'KAF'
+            for node in self.__get_wf_nodes():
+                node.set('wid',node.get('id'))
+                del node.attrib['id']
+
+    def to_naf(self):
+        if self.type == 'KAF':
+            self.type = 'NAF'
+            for node in self.__get_wf_nodes():
+                node.set('id',node.get('wid'))
+                del node.attrib['wid']
+
+    def __get_wf_nodes(self):
+        for wf_node in self.node.findall('wf'):
+            yield wf_node
+            
+    def __iter__(self):
+        for wf_node in self.__get_wf_nodes():
+            yield Cwf(node=wf_node,type=self.type)
+            
+    def get_wf(self,token_id):
+        wf_node = self.idx.get(token_id)
+        if wf_node is not None:
+            return Cwf(node=wf_node,type=self.type)
+        else:
+            return None
+    
+    def add_wf(self,wf_obj):
+        self.node.append(wf_obj.get_node())
+        

data/core/site-packages/pre_build/VUA_pylib-1.3-py2.7.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: VUA-pylib
+Version: 1.3
+Summary: Various KAF / NAF python helpers
+Home-page: https://github.com/cltl/VUA_pylib
+Author: Ruben Izquierdo
+Author-email: r.izquierdobevia@vu.nl
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

data/core/site-packages/pre_build/VUA_pylib-1.3-py2.7.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README
+VUA_pylib/__init__.py
+VUA_pylib.egg-info/PKG-INFO
+VUA_pylib.egg-info/SOURCES.txt
+VUA_pylib.egg-info/dependency_links.txt
+VUA_pylib.egg-info/top_level.txt
+VUA_pylib/common/__init__.py
+VUA_pylib/common/common.py
+VUA_pylib/corpus_reader/__init__.py
+VUA_pylib/corpus_reader/google_web_nl.py
+VUA_pylib/io_utils/__init__.py
+VUA_pylib/io_utils/feature_file.py
+VUA_pylib/lexicon/__init__.py
+VUA_pylib/lexicon/lexicon.py

data/core/site-packages/pre_build/VUA_pylib-1.3-py2.7.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

data/core/site-packages/pre_build/VUA_pylib-1.3-py2.7.egg-info/installed-files.txt

@@ -0,0 +1,23 @@
+../VUA_pylib/__init__.py
+../VUA_pylib/lexicon/lexicon.py
+../VUA_pylib/lexicon/__init__.py
+../VUA_pylib/common/common.py
+../VUA_pylib/common/__init__.py
+../VUA_pylib/io_utils/feature_file.py
+../VUA_pylib/io_utils/__init__.py
+../VUA_pylib/corpus_reader/google_web_nl.py
+../VUA_pylib/corpus_reader/__init__.py
+../VUA_pylib/__init__.pyc
+../VUA_pylib/lexicon/lexicon.pyc
+../VUA_pylib/lexicon/__init__.pyc
+../VUA_pylib/common/common.pyc
+../VUA_pylib/common/__init__.pyc
+../VUA_pylib/io_utils/feature_file.pyc
+../VUA_pylib/io_utils/__init__.pyc
+../VUA_pylib/corpus_reader/google_web_nl.pyc
+../VUA_pylib/corpus_reader/__init__.pyc
+./
+SOURCES.txt
+dependency_links.txt
+top_level.txt
+PKG-INFO

data/core/site-packages/pre_build/VUA_pylib-1.3-py2.7.egg-info/top_level.txt

@@ -0,0 +1 @@
+VUA_pylib

data/core/site-packages/pre_build/VUA_pylib/__init__.py

@@ -0,0 +1 @@
+pass

data/core/site-packages/pre_build/VUA_pylib/common/__init__.py

@@ -0,0 +1 @@
+from common import *

data/core/site-packages/pre_build/VUA_pylib/common/common.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+from operator import itemgetter
+
+# Get the max (key,count) from a dict like  my_dict = {'a':20,'b':1,'c':50}
+# It will return --> (c,50)
+def get_max_distr_dict(my_dict):
+    vect = my_dict.items()
+    if len(vect) !=0:
+        vect.sort(key=itemgetter(1),reverse=True)
+        return vect[0]
+    return None
+
+def normalize_pos(pos):
+    pos = pos.lower()
+    new_pos = pos
+    if pos in ['adj','a'] or pos[0:2]=='jj':
+        new_pos = 'a'
+    elif pos in ['adverb','r'] or pos[0:2]=='rb':
+        new_pos = 'r'
+    elif pos in ['anypos']:
+        new_pos = '*'
+    elif pos in ['noun','n'] or pos[0:2]=='nn' or pos[0:2]=='np':
+        new_pos = 'n'
+    elif pos in ['verb','v'] or pos[0]=='v':
+        new_pos = 'v'
+    return new_pos
+

data/core/site-packages/pre_build/VUA_pylib/corpus_reader/__init__.py

@@ -0,0 +1 @@
+from google_web_nl import *

data/core/site-packages/pre_build/VUA_pylib/corpus_reader/google_web_nl.py

@@ -0,0 +1,156 @@
+import urllib2
+import urllib
+import sys
+import time
+
+try:
+    from lxml import etree
+except:
+    import xml.etree.cElementTree as etree
+
+class Citem:
+    def __init__(self,item=None):
+        self.hits = None
+        self.word = None
+        self.tokens = None
+        if item is not None:
+            if isinstance(item,str):
+                self.load_from_string(item)
+            else:
+                self.load_from_item_node(item)
+            
+    def load_from_string(self,line):
+        ## Example line: 22865,"de server van"
+        line = line.strip()
+        pos = line.find(',')
+        self.hits = int(line[:pos])
+        self.word = line[pos+2:-1]
+        self.tokens = self.word.split(' ')
+        
+    def load_from_item_node(self,item_node):
+        hits_node = item_node.find('hits')
+        if hits_node is not None:
+            self.hits = int(hits_node.text)
+        
+        word_node = item_node.find('word')
+        if word_node is not None:
+            self.word = str(word_node.text)
+            self.tokens = self.word.split(' ')
+            
+    def __str__(self):
+        if self.word is not None and self.hits is not None:
+            s = str(self.tokens)+' ->'+str(self.hits)+' hits'
+        else:
+            s = 'None'
+        return s
+    
+    def __repr__(self):
+        return self.__str__()
+    
+    def get_hits(self):
+        return self.hits
+    
+    def get_word(self):
+        return self.word
+    
+    def get_tokens(self):
+        return self.tokens
+    
+
+class Cgoogle_web_nl:
+    def __init__(self):
+        self.url='http://www.let.rug.nl/gosse/bin/Web1T5_freq.perl'
+        self.sleep_this_time = 5	#First time to sleep in case of error
+        self.max_trials = 20        
+        self.limit = 1000
+        self.min_freq = 100
+        self.items = []
+            
+    
+    def set_limit(self,l):
+        if not isinstance(l, int):
+            print>>sys.stderr,'Parameter for set_min_freq must be an integer and not ',type(m)
+            sys.exit(-1)
+        self.limit = l
+        
+    def set_min_freq(self,m):
+        if not isinstance(m, int):
+            print>>sys.stderr,'Parameter for set_min_freq must be an integer and not ',type(m)
+            sys.exit(-1)
+        self.min_freq = m
+    
+    def query(self,this_query,fixed='shown'):
+        #http://www.let.rug.nl/gosse/bin/Web1T5_freq.perl?
+        #query=interessante%20*&
+        #mode=XML&limit=10000&
+        #threshold=40&optimize=on&wildcards=listed+normally
+        #&fixed=shown&.cgifields=debug&.cgifields=optimize
+        dict_params = {}
+        dict_params['query'] = this_query
+        dict_params['mode']='XML'
+        #dict_params['mode']='csv'
+        dict_params['limit']=self.limit
+        dict_params['threshold']=self.min_freq
+        dict_params['optimize']='on'
+        dict_params['wildcards']='listed normally'
+        dict_params['fixed']=fixed
+        dict_params['.cgifields']='debug'
+        dict_params['.cgifields']='optimize'
+        params = urllib.urlencode(dict_params)
+        #print>>sys.stderr,self.url+'?%s' % params
+        
+        
+        done = False
+        this_url = None
+        trials = 0
+        while not done:
+            try:
+                this_url = urllib2.urlopen(self.url+'?%s' % params)   
+                code = this_url.getcode()
+            except Exception as e:
+                code = -1
+                print>>sys.stderr,str(e)
+
+            if code == 200:                
+                done = True
+            else:
+                print>>sys.stderr,'Got an error (code '+str(code)+') querying google web nl, with "'+this_query+'", retrying...'
+                print>>sys.stderr,'Trial ',trials,' waiting ',self.sleep_this_time,'seconds'
+                time.sleep(self.sleep_this_time)
+                trials += 1
+                self.sleep_this_time += 1
+                if trials == self.max_trials:
+                    print>>sys.stderr,'Maximum number of trials reached. Giving up...'
+                    done = True
+                    this_url = None
+        
+        if this_url is not None:
+            if dict_params['mode'] == 'XML':
+                xml_obj = etree.parse(this_url)
+                this_url.close()
+
+                for item_node in xml_obj.findall('item'):
+                    self.items.append(Citem(item_node))
+                del xml_obj
+            else: #CSV
+                first_line = True
+                ## The first line is frequency,"N-gram"
+                for line in this_url:
+                    if not first_line:
+                        self.items.append(Citem(line))
+                    first_line = False                
+                
+    
+    
+    def get_items(self):
+        for item in self.items:
+            yield item
+            
+    def get_all_items(self):
+        return self.items
+    
+    def len(self):
+        return len(self.items)
+    def __iter__(self):
+        for item in self.items:
+            yield item

data/core/site-packages/pre_build/VUA_pylib/io_utils/__init__.py

@@ -0,0 +1 @@
+from feature_file import *

data/core/site-packages/pre_build/VUA_pylib/io_utils/feature_file.py

@@ -0,0 +1,121 @@
+from operator import itemgetter
+import sys
+import cPickle
+
+
+
+class Cexample:
+    def __init__(self,str_line=None):
+        self.label = ''
+        self.features = []
+        if str_line is not None:
+            self.load_from_line(str_line)
+        
+    def load_from_line(self,str_line):
+        fields = str_line.strip().split('\t')
+        self.label = fields[0]
+        for feat in fields[1:]:
+            first_equal = feat.find('=')
+            if first_equal != -1:
+                name = feat[:first_equal]
+                value = feat[first_equal+1:]
+                self.features.append((name,value))
+
+    def __str__(self):
+        s  = 'Label: '+self.label+'\n'
+        s += 'Feats: '+str(self.features) 
+        return s
+    
+    def get_label(self):
+        return self.label
+    
+    def get_features(self):
+        for name,value in self.features:
+            yield name,value
+            
+    def get_all_features(self):
+        return self.features
+    
+    
+class Cfeature_index:
+    def __init__(self):
+        self.idx = {}
+    
+    def get_number_feat(self,feat):
+        return self.idx.get(feat,None)
+    
+    def add_feat(self,feat):
+        num_feat = len(self.idx)+1
+        self.idx[feat] = num_feat
+        return num_feat
+        
+        
+    def compose_feat(self,name,value):
+        return name+'###'+value
+    
+    
+    def __encode_features(self,feats,modify_index=True):
+        feats_for_example = {}
+        clean_feats = ''
+        for name, value in feats:
+            my_feat = self.compose_feat(name, value)
+            clean_feats+=my_feat+' '
+            num_feat = self.get_number_feat(my_feat)
+            if num_feat is None:
+                if modify_index:
+                    num_feat = self.add_feat(my_feat)
+            
+            if num_feat is not None:
+                if num_feat in feats_for_example:
+                    feats_for_example[num_feat] += 1
+                else:
+                    feats_for_example[num_feat] = 1
+        return sorted(feats_for_example.items(),key=itemgetter(0)),clean_feats
+         
+            
+    def encode_feature_file_to_svm(self,feat_file_obj,out_fic=sys.stdout):
+        for example in feat_file_obj:
+            class_label = example.get_label()
+            out_fic.write(class_label)
+            feats_for_example, clean_feats =self.__encode_features(example.get_all_features())
+                                                                   
+            for feat,freq_feat in feats_for_example:
+                value = freq_feat
+                out_fic.write(' %d:%d' % (feat,value))
+            out_fic.write(' #'+clean_feats+'\n')
+            
+    def encode_example_for_classification(self, feats,out_fic,my_class='0'):
+        feats_for_example, clean_feats =self.__encode_features(feats,modify_index=False)
+        out_fic.write(my_class)
+        for feat,freq_feat in feats_for_example:
+                value = freq_feat
+                out_fic.write(' %d:%d' % (feat,value))
+        out_fic.write(' #'+clean_feats.encode('utf-8')+'\n')        
+    
+    def save_to_file(self,filename):
+        fic = open(filename,'wb')
+        cPickle.dump(self.idx, fic, protocol=0)
+        fic.close()
+        
+    def load_from_file(self,filename):
+        fic = open(filename,'rb')
+        self.idx = cPickle.load(fic)
+        fic.close()
+
+                
+    
+class Cfeature_file:
+    def __init__(self,filename=None):
+        self.filename = filename
+
+    def __iter__(self):
+        if self.filename is not None:
+            fic = open(self.filename,'r')
+            for line in fic:
+                if line[0] != '#':
+                    yield Cexample(line)
+            fic.close()
+    
+   
+              
+