bee_python 0.0.1

data/tools/common/common/dbf.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""This is a DBF reader which reads Visual Fox Pro DBF format with Memo field
+
+Usage:
+
+>>> rec = readDbf('test.dbf')
+>>> for line in rec:
+>>>     print line['name']
+
+
+:date: 13/07/2007
+
+http://www.physics.ox.ac.uk/users/santoso/Software.Repository.html
+page says code is "available as is without any warranty or support".
+"""
+
+import struct
+import os, os.path
+import sys
+import csv
+import tempfile
+import ConfigParser
+
+class Dbase:
+    def __init__(self):
+        self.fdb = None
+        self.fmemo = None
+        self.db_data = None
+        self.memo_data = None
+        self.fields = None
+        self.num_records = 0
+        self.header = None
+        self.memo_file = ''
+        self.memo_header = None
+        self.memo_block_size = 0
+        self.memo_header_len = 0
+
+    def _drop_after_NULL(self, txt):
+        for i in range(0, len(txt)):
+            if ord(struct.unpack('c', txt[i])[0])==0:
+                return txt[:i]
+        return txt
+
+    def _reverse_endian(self, num):
+        if not len(num):
+            return 0
+        val = struct.unpack('<L', num)
+        val = struct.pack('>L', val[0])
+        val = struct.unpack('>L', val)
+        return val[0]
+
+    def _assign_ids(self, lst, ids):
+        result = {}
+        idx = 0
+        for item in lst:
+            id = ids[idx]
+            result[id] = item
+            idx += 1
+        return result
+
+    def open(self, db_name):
+        filesize = os.path.getsize(db_name)
+        if filesize <= 68:
+            raise IOError, 'The file is not large enough to be a dbf file'
+
+        self.fdb = open(db_name, 'rb')
+
+        self.memo_file = ''
+        if os.path.isfile(db_name[0:-1] + 't'):
+            self.memo_file = db_name[0:-1] + 't'
+        elif os.path.isfile(db_name[0:-3] + 'fpt'):
+            self.memo_file = db_name[0:-3] + 'fpt'
+
+        if self.memo_file:
+            #Read memo file
+            self.fmemo = open(self.memo_file, 'rb')
+            self.memo_data = self.fmemo.read()
+            self.memo_header = self._assign_ids(struct.unpack('>6x1H', self.memo_data[:8]), ['Block size'])
+            block_size = self.memo_header['Block size']
+            if not block_size:
+                block_size = 512
+            self.memo_block_size = block_size
+            self.memo_header_len = block_size
+            memo_size = os.path.getsize(self.memo_file)
+
+        #Start reading data file
+        data = self.fdb.read(32)
+        self.header = self._assign_ids(struct.unpack('<B 3B L 2H 20x', data), ['id', 'Year', 'Month', 'Day', '# of Records', 'Header Size', 'Record Size'])
+        self.header['id'] = hex(self.header['id'])
+
+        self.num_records = self.header['# of Records']
+        data = self.fdb.read(self.header['Header Size']-34)
+        self.fields = {}
+        x = 0
+        header_pattern = '<11s c 4x B B 14x'
+        ids = ['Field Name', 'Field Type', 'Field Length', 'Field Precision']
+        pattern_len = 32
+        for offset in range(0, len(data), 32):
+            if ord(data[offset])==0x0d:
+                break
+            x += 1
+            data_subset = data[offset: offset+pattern_len]
+            if len(data_subset) < pattern_len:
+                data_subset += ' '*(pattern_len-len(data_subset))
+            self.fields[x] = self._assign_ids(struct.unpack(header_pattern, data_subset), ids)
+            self.fields[x]['Field Name'] = self._drop_after_NULL(self.fields[x]['Field Name'])
+
+        self.fdb.read(3)
+        if self.header['# of Records']:
+            data_size = (self.header['# of Records'] * self.header['Record Size']) - 1
+            self.db_data = self.fdb.read(data_size)
+        else:
+            self.db_data = ''
+        self.row_format = '<'
+        self.row_ids = []
+        self.row_len = 0
+        for key in self.fields:
+            field = self.fields[key]
+            self.row_format += '%ds ' % (field['Field Length'])
+            self.row_ids.append(field['Field Name'])
+            self.row_len += field['Field Length']
+
+    def close(self):
+        if self.fdb:
+            self.fdb.close()
+        if self.fmemo:
+            self.fmemo.close()
+
+    def get_numrecords(self):
+        return self.num_records
+
+    def get_record_with_names(self, rec_no):
+        """
+        This function accept record number from 0 to N-1
+        """
+        if rec_no < 0 or rec_no > self.num_records:
+            raise Exception, 'Unable to extract data outside the range'
+
+        offset = self.header['Record Size'] * rec_no
+        data = self.db_data[offset:offset+self.row_len]
+        record = self._assign_ids(struct.unpack(self.row_format, data), self.row_ids)
+
+        if self.memo_file:
+            for key in self.fields:
+                field = self.fields[key]
+                f_type = field['Field Type']
+                f_name = field['Field Name']
+                c_data = record[f_name]
+
+                if f_type=='M' or f_type=='G' or f_type=='B' or f_type=='P':
+                    c_data = self._reverse_endian(c_data)
+                    if c_data:
+                        record[f_name] = self.read_memo(c_data-1).strip()
+                else:
+                    record[f_name] = c_data.strip()
+        return record
+
+    def read_memo_record(self, num, in_length):
+        """
+        Read the record of given number. The second parameter is the length of
+        the record to read. It can be undefined, meaning read the whole record,
+        and it can be negative, meaning at most the length
+        """
+        if in_length < 0:
+            in_length = -self.memo_block_size
+
+        offset = self.memo_header_len + num * self.memo_block_size
+        self.fmemo.seek(offset)
+        if in_length<0:
+            in_length = -in_length
+        if in_length==0:
+            return ''
+        return self.fmemo.read(in_length)
+
+    def read_memo(self, num):
+        result = ''
+        buffer = self.read_memo_record(num, -1)
+        if len(buffer)<=0:
+            return ''
+        length = struct.unpack('>L', buffer[4:4+4])[0] + 8
+
+        block_size = self.memo_block_size
+        if length < block_size:
+            return buffer[8:length]
+        rest_length = length - block_size
+        rest_data = self.read_memo_record(num+1, rest_length)
+        if len(rest_data)<=0:
+            return ''
+        return buffer[8:] + rest_data
+
+def readDbf(filename):
+    """
+    Read the DBF file specified by the filename and
+    return the records as a list of dictionary.
+
+    :param: filename File name of the DBF
+    :return: List of rows
+    """
+    db = Dbase()
+    db.open(filename)
+    num = db.get_numrecords()
+    rec = []
+    for i in range(0, num):
+        record = db.get_record_with_names(i)
+        rec.append(record)
+    db.close()
+    return  rec
+
+if __name__=='__main__':
+    rec = readDbf('dbf/sptable.dbf')
+    for line in rec:
+        print '%s %s' % (line['GENUS'].strip(), line['SPECIES'].strip())

data/tools/common/common/debugger.py

@@ -0,0 +1,208 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Customized version of pdb's default debugger.
+
+- sets up a history file
+- uses ipython if available to colorize lines of code
+- overrides list command to search for current block instead
+  of using 5 lines of context
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+try:
+    import readline
+except ImportError:
+    readline = None
+import os
+import os.path as osp
+import sys
+from pdb import Pdb
+from cStringIO import StringIO
+import inspect
+
+try:
+    from IPython import PyColorize
+except ImportError:
+    def colorize(source, *args):
+        """fallback colorize function"""
+        return source
+    def colorize_source(source, *args):
+        return source
+else:
+    def colorize(source, start_lineno, curlineno):
+        """colorize and annotate source with linenos
+        (as in pdb's list command)
+        """
+        parser = PyColorize.Parser()
+        output = StringIO()
+        parser.format(source, output)
+        annotated = []
+        for index, line in enumerate(output.getvalue().splitlines()):
+            lineno = index + start_lineno
+            if lineno == curlineno:
+                annotated.append('%4s\t->\t%s' % (lineno, line))
+            else:
+                annotated.append('%4s\t\t%s' % (lineno, line))
+        return '\n'.join(annotated)
+
+    def colorize_source(source):
+        """colorize given source"""
+        parser = PyColorize.Parser()
+        output = StringIO()
+        parser.format(source, output)
+        return output.getvalue()
+
+
+def getsource(obj):
+    """Return the text of the source code for an object.
+
+    The argument may be a module, class, method, function, traceback, frame,
+    or code object.  The source code is returned as a single string.  An
+    IOError is raised if the source code cannot be retrieved."""
+    lines, lnum = inspect.getsourcelines(obj)
+    return ''.join(lines), lnum
+
+
+################################################################
+class Debugger(Pdb):
+    """custom debugger
+
+    - sets up a history file
+    - uses ipython if available to colorize lines of code
+    - overrides list command to search for current block instead
+      of using 5 lines of context
+    """
+    def __init__(self, tcbk=None):
+        Pdb.__init__(self)
+        self.reset()
+        if tcbk:
+            while tcbk.tb_next is not None:
+                tcbk = tcbk.tb_next
+        self._tcbk = tcbk
+        self._histfile = osp.join(os.environ["HOME"], ".pdbhist")
+
+    def setup_history_file(self):
+        """if readline is available, read pdb history file
+        """
+        if readline is not None:
+            try:
+                readline.read_history_file(self._histfile)
+            except IOError:
+                pass
+
+    def start(self):
+        """starts the interactive mode"""
+        self.interaction(self._tcbk.tb_frame, self._tcbk)
+
+    def setup(self, frame, tcbk):
+        """setup hook: set up history file"""
+        self.setup_history_file()
+        Pdb.setup(self, frame, tcbk)
+
+    def set_quit(self):
+        """quit hook: save commands in the history file"""
+        if readline is not None:
+            readline.write_history_file(self._histfile)
+        Pdb.set_quit(self)
+
+    def complete_p(self, text, line, begin_idx, end_idx):
+        """provide variable names completion for the ``p`` command"""
+        namespace = dict(self.curframe.f_globals)
+        namespace.update(self.curframe.f_locals)
+        if '.' in text:
+            return self.attr_matches(text, namespace)
+        return [varname for varname in namespace if varname.startswith(text)]
+
+
+    def attr_matches(self, text, namespace):
+        """implementation coming from rlcompleter.Completer.attr_matches
+        Compute matches when text contains a dot.
+
+        Assuming the text is of the form NAME.NAME....[NAME], and is
+        evaluatable in self.namespace, it will be evaluated and its attributes
+        (as revealed by dir()) are used as possible completions.  (For class
+        instances, class members are also considered.)
+
+        WARNING: this can still invoke arbitrary C code, if an object
+        with a __getattr__ hook is evaluated.
+
+        """
+        import re
+        m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text)
+        if not m:
+            return
+        expr, attr = m.group(1, 3)
+        object = eval(expr, namespace)
+        words = dir(object)
+        if hasattr(object,'__class__'):
+            words.append('__class__')
+            words = words + self.get_class_members(object.__class__)
+        matches = []
+        n = len(attr)
+        for word in words:
+            if word[:n] == attr and word != "__builtins__":
+                matches.append("%s.%s" % (expr, word))
+        return matches
+
+    def get_class_members(self, klass):
+        """implementation coming from rlcompleter.get_class_members"""
+        ret = dir(klass)
+        if hasattr(klass,'__bases__'):
+            for base in klass.__bases__:
+                ret = ret + self.get_class_members(base)
+        return ret
+
+    ## specific / overridden commands
+    def do_list(self, arg):
+        """overrides default list command to display the surrounding block
+        instead of 5 lines of context
+        """
+        self.lastcmd = 'list'
+        if not arg:
+            try:
+                source, start_lineno = getsource(self.curframe)
+                print colorize(''.join(source), start_lineno,
+                               self.curframe.f_lineno)
+            except KeyboardInterrupt:
+                pass
+            except IOError:
+                Pdb.do_list(self, arg)
+        else:
+            Pdb.do_list(self, arg)
+    do_l = do_list
+
+    def do_open(self, arg):
+        """opens source file corresponding to the current stack level"""
+        filename = self.curframe.f_code.co_filename
+        lineno = self.curframe.f_lineno
+        cmd = 'emacsclient --no-wait +%s %s' % (lineno, filename)
+        os.system(cmd)
+
+    do_o = do_open
+
+def pm():
+    """use our custom debugger"""
+    dbg = Debugger(sys.last_traceback)
+    dbg.start()
+
+def set_trace():
+    Debugger().set_trace(sys._getframe().f_back)

data/tools/common/common/decorators.py

@@ -0,0 +1,190 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""A few useful function/method decorators.
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+from types import MethodType
+from time import clock, time
+import sys, re
+
+# XXX rewrite so we can use the decorator syntax when keyarg has to be specified
+
+def cached(callableobj, keyarg=None):
+    """Simple decorator to cache result of method call."""
+    if callableobj.func_code.co_argcount == 1 or keyarg == 0:
+
+        def cache_wrapper1(self, *args):
+            cache = '_%s_cache_' % callableobj.__name__
+            #print 'cache1?', cache
+            try:
+                return self.__dict__[cache]
+            except KeyError:
+                #print 'miss'
+                value = callableobj(self, *args)
+                setattr(self, cache, value)
+                return value
+        cache_wrapper1.__doc__ = callableobj.__doc__
+        return cache_wrapper1
+
+    elif keyarg:
+
+        def cache_wrapper2(self, *args, **kwargs):
+            cache = '_%s_cache_' % callableobj.__name__
+            key = args[keyarg-1]
+            #print 'cache2?', cache, self, key
+            try:
+                _cache = self.__dict__[cache]
+            except KeyError:
+                #print 'init'
+                _cache = {}
+                setattr(self, cache, _cache)
+            try:
+                return _cache[key]
+            except KeyError:
+                #print 'miss', self, cache, key
+                _cache[key] = callableobj(self, *args, **kwargs)
+            return _cache[key]
+        cache_wrapper2.__doc__ = callableobj.__doc__
+        return cache_wrapper2
+
+    def cache_wrapper3(self, *args):
+        cache = '_%s_cache_' % callableobj.__name__
+        #print 'cache3?', cache, self, args
+        try:
+            _cache = self.__dict__[cache]
+        except KeyError:
+            #print 'init'
+            _cache = {}
+            setattr(self, cache, _cache)
+        try:
+            return _cache[args]
+        except KeyError:
+            #print 'miss'
+            _cache[args] = callableobj(self, *args)
+        return _cache[args]
+    cache_wrapper3.__doc__ = callableobj.__doc__
+    return cache_wrapper3
+
+def clear_cache(obj, funcname):
+    """Function to clear a cache handled by the cached decorator."""
+    try:
+        del obj.__dict__['_%s_cache_' % funcname]
+    except KeyError:
+        pass
+
+def copy_cache(obj, funcname, cacheobj):
+    """Copy cache for <funcname> from cacheobj to obj."""
+    cache = '_%s_cache_' % funcname
+    try:
+        setattr(obj, cache, cacheobj.__dict__[cache])
+    except KeyError:
+        pass
+
+class wproperty(object):
+    """Simple descriptor expecting to take a modifier function as first argument
+    and looking for a _<function name> to retrieve the attribute.
+    """
+    def __init__(self, setfunc):
+        self.setfunc = setfunc
+        self.attrname = '_%s' % setfunc.__name__
+
+    def __set__(self, obj, value):
+        self.setfunc(obj, value)
+
+    def __get__(self, obj, cls):
+        assert obj is not None
+        return getattr(obj, self.attrname)
+
+
+class classproperty(object):
+    """this is a simple property-like class but for class attributes.
+    """
+    def __init__(self, get):
+        self.get = get
+    def __get__(self, inst, cls):
+        return self.get(cls)
+
+
+class iclassmethod(object):
+    '''Descriptor for method which should be available as class method if called
+    on the class or instance method if called on an instance.
+    '''
+    def __init__(self, func):
+        self.func = func
+    def __get__(self, instance, objtype):
+        if instance is None:
+            return MethodType(self.func, objtype, objtype.__class__)
+        return MethodType(self.func, instance, objtype)
+    def __set__(self, instance, value):
+        raise AttributeError("can't set attribute")
+
+
+def timed(f):
+    def wrap(*args, **kwargs):
+        t = time()
+        c = clock()
+        res = f(*args, **kwargs)
+        print '%s clock: %.9f / time: %.9f' % (f.__name__,
+                                               clock() - c, time() - t)
+        return res
+    return wrap
+
+
+def locked(acquire, release):
+    """Decorator taking two methods to acquire/release a lock as argument,
+    returning a decorator function which will call the inner method after
+    having called acquire(self) et will call release(self) afterwards.
+    """
+    def decorator(f):
+        def wrapper(self, *args, **kwargs):
+            acquire(self)
+            try:
+                return f(self, *args, **kwargs)
+            finally:
+                release(self)
+        return wrapper
+    return decorator
+
+
+def monkeypatch(klass, methodname=None):
+    """Decorator extending class with the decorated function
+    >>> class A:
+    ...     pass
+    >>> @monkeypatch(A)
+    ... def meth(self):
+    ...     return 12
+    ...
+    >>> a = A()
+    >>> a.meth()
+    12
+    >>> @monkeypatch(A, 'foo')
+    ... def meth(self):
+    ...     return 12
+    ...
+    >>> a.foo()
+    12
+    """
+    def decorator(func):
+        setattr(klass, methodname or func.__name__, func)
+        return func
+    return decorator