bee_python 0.0.1

data/tools/common/common/shellutils.py

@@ -0,0 +1,383 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""shell/term utilities, useful to write some python scripts instead of shell
+scripts.
+"""
+__docformat__ = "restructuredtext en"
+
+import os
+import glob
+import shutil
+import stat
+import sys
+import tempfile
+import time
+import fnmatch
+import errno
+from os.path import exists, isdir, islink, basename, join, walk
+
+from logilab.common import STD_BLACKLIST
+try:
+    from logilab.common.proc import ProcInfo, NoSuchProcess
+except ImportError:
+    # windows platform
+    class NoSuchProcess(Exception): pass
+
+    def ProcInfo(pid):
+        raise NoSuchProcess()
+
+
+def chown(path, login=None, group=None):
+    """Same as `os.chown` function but accepting user login or group name as
+    argument. If login or group is omitted, it's left unchanged.
+
+    Note: you must own the file to chown it (or be root). Otherwise OSError is raised.
+    """
+    if login is None:
+        uid = -1
+    else:
+        try:
+            uid = int(login)
+        except ValueError:
+            import pwd # Platforms: Unix
+            uid = pwd.getpwnam(login).pw_uid
+    if group is None:
+        gid = -1
+    else:
+        try:
+            gid = int(group)
+        except ValueError:
+            import grp
+            gid = grp.getgrnam(group).gr_gid
+    os.chown(path, uid, gid)
+
+def mv(source, destination, _action=shutil.move):
+    """A shell-like mv, supporting wildcards.
+    """
+    sources = glob.glob(source)
+    if len(sources) > 1:
+        assert isdir(destination)
+        for filename in sources:
+            _action(filename, join(destination, basename(filename)))
+    else:
+        try:
+            source = sources[0]
+        except IndexError:
+            raise OSError('No file matching %s' % source)
+        if isdir(destination) and exists(destination):
+            destination = join(destination, basename(source))
+        try:
+            _action(source, destination)
+        except OSError, ex:
+            raise OSError('Unable to move %r to %r (%s)' % (
+                source, destination, ex))
+
+def rm(*files):
+    """A shell-like rm, supporting wildcards.
+    """
+    for wfile in files:
+        for filename in glob.glob(wfile):
+            if islink(filename):
+                os.remove(filename)
+            elif isdir(filename):
+                shutil.rmtree(filename)
+            else:
+                os.remove(filename)
+
+def cp(source, destination):
+    """A shell-like cp, supporting wildcards.
+    """
+    mv(source, destination, _action=shutil.copy)
+
+def find(directory, exts, exclude=False, blacklist=STD_BLACKLIST):
+    """Recursively find files ending with the given extensions from the directory.
+
+    :type directory: str
+    :param directory:
+      directory where the search should start
+
+    :type exts: basestring or list or tuple
+    :param exts:
+      extensions or lists or extensions to search
+
+    :type exclude: boolean
+    :param exts:
+      if this argument is True, returning files NOT ending with the given
+      extensions
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: list
+    :return:
+      the list of all matching files
+    """
+    if isinstance(exts, basestring):
+        exts = (exts,)
+    if exclude:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return False
+            return True
+    else:
+        def match(filename, exts):
+            for ext in exts:
+                if filename.endswith(ext):
+                    return True
+            return False
+    def func(files, directory, fnames):
+        """walk handler"""
+        # remove files/directories in the black list
+        for norecurs in blacklist:
+            try:
+                fnames.remove(norecurs)
+            except ValueError:
+                continue
+        for filename in fnames:
+            src = join(directory, filename)
+            if isdir(src):
+                continue
+            if match(filename, exts):
+                files.append(src)
+    files = []
+    walk(directory, func, files)
+    return files
+
+
+def globfind(directory, pattern, blacklist=STD_BLACKLIST):
+    """Recursively finds files matching glob `pattern` under `directory`.
+
+    This is an alternative to `logilab.common.shellutils.find`.
+
+    :type directory: str
+    :param directory:
+      directory where the search should start
+
+    :type pattern: basestring
+    :param pattern:
+      the glob pattern (e.g *.py, foo*.py, etc.)
+
+    :type blacklist: list or tuple
+    :param blacklist:
+      optional list of files or directory to ignore, default to the value of
+      `logilab.common.STD_BLACKLIST`
+
+    :rtype: iterator
+    :return:
+      iterator over the list of all matching files
+    """
+    for curdir, dirnames, filenames in os.walk(directory):
+        for fname in fnmatch.filter(filenames, pattern):
+            yield join(curdir, fname)
+        for skipped in blacklist:
+            if skipped in dirnames:
+                dirnames.remove(skipped)
+
+def unzip(archive, destdir):
+    import zipfile
+    if not exists(destdir):
+        os.mkdir(destdir)
+    zfobj = zipfile.ZipFile(archive)
+    for name in zfobj.namelist():
+        if name.endswith('/'):
+            os.mkdir(join(destdir, name))
+        else:
+            outfile = open(join(destdir, name), 'wb')
+            outfile.write(zfobj.read(name))
+            outfile.close()
+
+class Execute:
+    """This is a deadlock safe version of popen2 (no stdin), that returns
+    an object with errorlevel, out and err.
+    """
+
+    def __init__(self, command):
+        outfile = tempfile.mktemp()
+        errfile = tempfile.mktemp()
+        self.status = os.system("( %s ) >%s 2>%s" %
+                                (command, outfile, errfile)) >> 8
+        self.out = open(outfile,"r").read()
+        self.err = open(errfile,"r").read()
+        os.remove(outfile)
+        os.remove(errfile)
+
+def acquire_lock(lock_file, max_try=10, delay=10, max_delay=3600):
+    """Acquire a lock represented by a file on the file system
+
+    If the process written in lock file doesn't exist anymore, we remove the
+    lock file immediately
+    If age of the lock_file is greater than max_delay, then we raise a UserWarning
+    """
+    count = abs(max_try)
+    while count:
+        try:
+            fd = os.open(lock_file, os.O_EXCL | os.O_RDWR | os.O_CREAT)
+            os.write(fd, str(os.getpid()))
+            os.close(fd)
+            return True
+        except OSError, e:
+            if e.errno == errno.EEXIST:
+                try:
+                    fd = open(lock_file, "r")
+                    pid = int(fd.readline())
+                    pi = ProcInfo(pid)
+                    age = (time.time() - os.stat(lock_file)[stat.ST_MTIME])
+                    if age / max_delay > 1 :
+                        raise UserWarning("Command '%s' (pid %s) has locked the "
+                                          "file '%s' for %s minutes"
+                                          % (pi.name(), pid, lock_file, age/60))
+                except UserWarning:
+                    raise
+                except NoSuchProcess:
+                    os.remove(lock_file)
+                except Exception:
+                    # The try block is not essential. can be skipped.
+                    # Note: ProcInfo object is only available for linux
+                    # process information are not accessible...
+                    # or lock_file is no more present...
+                    pass
+            else:
+                raise
+            count -= 1
+            time.sleep(delay)
+    else:
+        raise Exception('Unable to acquire %s' % lock_file)
+
+def release_lock(lock_file):
+    """Release a lock represented by a file on the file system."""
+    os.remove(lock_file)
+
+
+class ProgressBar(object):
+    """A simple text progression bar."""
+
+    def __init__(self, nbops, size=20, stream=sys.stdout, title=''):
+        if title:
+            self._fstr = '\r%s [%%-%ss]' % (title, int(size))
+        else:
+            self._fstr = '\r[%%-%ss]' % int(size)
+        self._stream = stream
+        self._total = nbops
+        self._size = size
+        self._current = 0
+        self._progress = 0
+        self._current_text = None
+        self._last_text_write_size = 0
+
+    def _get_text(self):
+        return self._current_text
+
+    def _set_text(self, text=None):
+        self._current_text = text
+        self.refresh()
+
+    def _del_text(self):
+        self.text = None
+
+    text = property(_get_text, _set_text, _del_text)
+
+    def update(self):
+        """Update the progression bar."""
+        self._current += 1
+        progress = int((float(self._current)/float(self._total))*self._size)
+        if progress > self._progress:
+            self._progress = progress
+            self.refresh()
+
+    def refresh(self):
+        """Refresh the progression bar display."""
+        self._stream.write(self._fstr % ('.' * min(self._progress, self._size)) )
+        if self._last_text_write_size or self._current_text:
+            template = ' %%-%is' % (self._last_text_write_size)
+            text = self._current_text
+            if text is None:
+                text = ''
+            self._stream.write(template % text)
+            self._last_text_write_size = len(text.rstrip())
+        self._stream.flush()
+
+from logilab.common.deprecation import deprecated
+
+@deprecated('confirm() is deprecated, use RawInput.confirm() instead')
+def confirm(question, default_is_yes=True):
+    """ask for confirmation and return true on positive answer"""
+    return RawInput().confirm(question, default_is_yes)
+
+
+class RawInput(object):
+
+    def __init__(self, input=None, printer=None):
+        self._input = input or raw_input
+        self._print = printer
+
+    def ask(self, question, options, default):
+        assert default in options
+        choices = []
+        for option in options:
+            if option == default:
+                label = option[0].upper()
+            else:
+                label = option[0].lower()
+            if len(option) > 1:
+                label += '(%s)' % option[1:].lower()
+            choices.append((option, label))
+        prompt = "%s [%s]: " % (question,
+                                '/'.join(opt[1] for opt in choices))
+        tries = 3
+        while tries > 0:
+            answer = self._input(prompt).strip().lower()
+            if not answer:
+                return default
+            possible = [option for option, label in choices
+                        if option.lower().startswith(answer)]
+            if len(possible) == 1:
+                return possible[0]
+            elif len(possible) == 0:
+                msg = '%s is not an option.' % answer
+            else:
+                msg = ('%s is an ambiguous answer, do you mean %s ?' % (
+                        answer, ' or '.join(possible)))
+            if self._print:
+                self._print(msg)
+            else:
+                print msg
+            tries -= 1
+        raise Exception('unable to get a sensible answer')
+
+    def confirm(self, question, default_is_yes=True):
+        default = default_is_yes and 'y' or 'n'
+        answer = self.ask(question, ('y','n'), default)
+        return answer == 'y'
+
+ASK = RawInput()
+
+
+def getlogin():
+    """avoid using os.getlogin() because of strange tty / stdin problems
+    (man 3 getlogin)
+    Another solution would be to use $LOGNAME, $USER or $USERNAME
+    """
+    if sys.platform != 'win32':
+        import pwd # Platforms: Unix
+        return pwd.getpwuid(os.getuid())[0]
+    else:
+        return os.environ['USERNAME']
+

data/tools/common/common/sphinx_ext.py

@@ -0,0 +1,87 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+from logilab.common.decorators import monkeypatch
+
+from sphinx.ext import autodoc
+
+class DocstringOnlyModuleDocumenter(autodoc.ModuleDocumenter):
+    objtype = 'docstring'
+    def format_signature(self):
+        pass
+    def add_directive_header(self, sig):
+        pass
+    def document_members(self, all_members=False):
+        pass
+
+    def resolve_name(self, modname, parents, path, base):
+        if modname is not None:
+            return modname, parents + [base]
+        return (path or '') + base, []
+
+
+#autodoc.add_documenter(DocstringOnlyModuleDocumenter)
+
+def setup(app):
+    app.add_autodocumenter(DocstringOnlyModuleDocumenter)
+
+
+
+from sphinx.ext.autodoc import (ViewList, Options, AutodocReporter, nodes,
+                                assemble_option_dict, nested_parse_with_titles)
+
+@monkeypatch(autodoc.AutoDirective)
+def run(self):
+    self.filename_set = set()  # a set of dependent filenames
+    self.reporter = self.state.document.reporter
+    self.env = self.state.document.settings.env
+    self.warnings = []
+    self.result = ViewList()
+
+    # find out what documenter to call
+    objtype = self.name[4:]
+    doc_class = self._registry[objtype]
+    # process the options with the selected documenter's option_spec
+    self.genopt = Options(assemble_option_dict(
+        self.options.items(), doc_class.option_spec))
+    # generate the output
+    documenter = doc_class(self, self.arguments[0])
+    documenter.generate(more_content=self.content)
+    if not self.result:
+        return self.warnings
+
+    # record all filenames as dependencies -- this will at least
+    # partially make automatic invalidation possible
+    for fn in self.filename_set:
+        self.env.note_dependency(fn)
+
+    # use a custom reporter that correctly assigns lines to source
+    # filename/description and lineno
+    old_reporter = self.state.memo.reporter
+    self.state.memo.reporter = AutodocReporter(self.result,
+                                               self.state.memo.reporter)
+    if self.name in ('automodule', 'autodocstring'):
+        node = nodes.section()
+        # necessary so that the child nodes get the right source/line set
+        node.document = self.state.document
+        nested_parse_with_titles(self.state, self.result, node)
+    else:
+        node = nodes.paragraph()
+        node.document = self.state.document
+        self.state.nested_parse(self.result, 0, node)
+    self.state.memo.reporter = old_reporter
+    return self.warnings + node.children

data/tools/common/common/sphinxutils.py

@@ -0,0 +1,122 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Sphinx utils
+
+ModuleGenerator: Generate a file that lists all the modules of a list of
+packages in order to pull all the docstring.
+This should not be used in a makefile to systematically generate sphinx
+documentation!
+
+Typical usage:
+
+>>> from logilab.common.sphinxutils import ModuleGenerator
+>>> mgen = ModuleGenerator('logilab common', '/home/adim/src/logilab/common')
+>>> mgen.generate('api_logilab_common.rst', exclude_dirs=('test',))
+"""
+
+import os, sys
+import os.path as osp
+import inspect
+
+from logilab.common import STD_BLACKLIST
+from logilab.common.shellutils import globfind
+from logilab.common.modutils import load_module_from_file, modpath_from_file
+
+def module_members(module):
+    members = []
+    for name, value in inspect.getmembers(module):
+        if getattr(value, '__module__', None) == module.__name__:
+            members.append( (name, value) )
+    return sorted(members)
+
+
+def class_members(klass):
+    return sorted([name for name in vars(klass)
+                   if name not in ('__doc__', '__module__',
+                                   '__dict__', '__weakref__')])
+
+class ModuleGenerator:
+    file_header = """.. -*- coding: utf-8 -*-\n\n%s\n"""
+    module_def = """
+:mod:`%s`
+=======%s
+
+.. automodule:: %s
+   :members: %s
+"""
+    class_def = """
+
+.. autoclass:: %s
+   :members: %s
+
+"""
+
+    def __init__(self, project_title, code_dir):
+        self.title = project_title
+        self.code_dir = osp.abspath(code_dir)
+
+    def generate(self, dest_file, exclude_dirs=STD_BLACKLIST):
+        """make the module file"""
+        self.fn = open(dest_file, 'w')
+        num = len(self.title) + 6
+        title = "=" * num + "\n %s API\n" % self.title + "=" * num
+        self.fn.write(self.file_header % title)
+        self.gen_modules(exclude_dirs=exclude_dirs)
+        self.fn.close()
+
+    def gen_modules(self, exclude_dirs):
+        """generate all modules"""
+        for module in self.find_modules(exclude_dirs):
+            modname = module.__name__
+            classes = []
+            modmembers = []
+            for objname, obj in module_members(module):
+                if inspect.isclass(obj):
+                    classmembers = class_members(obj)
+                    classes.append( (objname, classmembers) )
+                else:
+                    modmembers.append(objname)
+            self.fn.write(self.module_def % (modname, '=' * len(modname),
+                                             modname,
+                                             ', '.join(modmembers)))
+            for klass, members in classes:
+                self.fn.write(self.class_def % (klass, ', '.join(members)))
+
+    def find_modules(self, exclude_dirs):
+        basepath = osp.dirname(self.code_dir)
+        basedir = osp.basename(basepath) + osp.sep
+        if basedir not in sys.path:
+            sys.path.insert(1, basedir)
+        for filepath in globfind(self.code_dir, '*.py', exclude_dirs):
+            if osp.basename(filepath) in ('setup.py', '__pkginfo__.py'):
+                continue
+            try:
+                module = load_module_from_file(filepath)
+            except: # module might be broken or magic
+                dotted_path = modpath_from_file(filepath)
+                module = type('.'.join(dotted_path), (), {}) # mock it
+            yield module
+
+
+if __name__ == '__main__':
+    # example :
+    title, code_dir, outfile = sys.argv[1:]
+    generator = ModuleGenerator(title, code_dir)
+    # XXX modnames = ['logilab']
+    generator.generate(outfile, ('test', 'tests', 'examples',
+                             'data', 'doc', '.hg', 'migration'))

data/tools/common/common/sqlgen.py

@@ -0,0 +1,31 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Help to generate SQL strings usable by the Python DB-API.
+
+
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+
+from warnings import warn
+warn('this module is deprecated, use logilab.database instead',
+     DeprecationWarning, stacklevel=1)
+from logilab.database.sqlgen import *