bee_python 0.0.1

data/tools/common/common/proc.py

@@ -0,0 +1,276 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""module providing:
+* process information (linux specific: rely on /proc)
+* a class for resource control (memory / time / cpu time)
+
+This module doesn't work on windows platforms (only tested on linux)
+
+:organization: Logilab
+
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+import os
+import stat
+from resource import getrlimit, setrlimit, RLIMIT_CPU, RLIMIT_AS
+from signal import signal, SIGXCPU, SIGKILL, SIGUSR2, SIGUSR1
+from threading import Timer, currentThread, Thread, Event
+from time import time
+
+from logilab.common.tree import Node
+
+class NoSuchProcess(Exception): pass
+
+def proc_exists(pid):
+    """check the a pid is registered in /proc
+    raise NoSuchProcess exception if not
+    """
+    if not os.path.exists('/proc/%s' % pid):
+        raise NoSuchProcess()
+
+PPID = 3
+UTIME = 13
+STIME = 14
+CUTIME = 15
+CSTIME = 16
+VSIZE = 22
+
+class ProcInfo(Node):
+    """provide access to process information found in /proc"""
+
+    def __init__(self, pid):
+        self.pid = int(pid)
+        Node.__init__(self, self.pid)
+        proc_exists(self.pid)
+        self.file = '/proc/%s/stat' % self.pid
+        self.ppid = int(self.status()[PPID])
+
+    def memory_usage(self):
+        """return the memory usage of the process in Ko"""
+        try :
+            return int(self.status()[VSIZE])
+        except IOError:
+            return 0
+
+    def lineage_memory_usage(self):
+        return self.memory_usage() + sum(child.lineage_memory_usage() for child in self.children)
+
+    def time(self, children=0):
+        """return the number of jiffies that this process has been scheduled
+        in user and kernel mode"""
+        status = self.status()
+        time = int(status[UTIME]) + int(status[STIME])
+        if children:
+            time += int(status[CUTIME]) + int(status[CSTIME])
+        return time
+
+    def status(self):
+        """return the list of fields found in /proc/<pid>/stat"""
+        return open(self.file).read().split()
+
+    def name(self):
+        """return the process name found in /proc/<pid>/stat
+        """
+        return self.status()[1].strip('()')
+
+    def age(self):
+        """return the age of the process
+        """
+        return os.stat(self.file)[stat.ST_MTIME]
+
+class ProcInfoLoader:
+    """manage process information"""
+
+    def __init__(self):
+        self._loaded = {}
+
+    def list_pids(self):
+        """return a list of existent process ids"""
+        for subdir in os.listdir('/proc'):
+            if subdir.isdigit():
+                yield int(subdir)
+
+    def load(self, pid):
+        """get a ProcInfo object for a given pid"""
+        pid = int(pid)
+        try:
+            return self._loaded[pid]
+        except KeyError:
+            procinfo = ProcInfo(pid)
+            procinfo.manager = self
+            self._loaded[pid] = procinfo
+            return procinfo
+
+
+    def load_all(self):
+        """load all processes information"""
+        for pid in self.list_pids():
+            try:
+                procinfo = self.load(pid)
+                if procinfo.parent is None and procinfo.ppid:
+                    pprocinfo = self.load(procinfo.ppid)
+                    pprocinfo.append(procinfo)
+            except NoSuchProcess:
+                pass
+
+
+try:
+    class ResourceError(BaseException):
+        """Error raise when resource limit is reached"""
+        limit = "Unknown Resource Limit"
+except NameError:
+    class ResourceError(Exception):
+        """Error raise when resource limit is reached"""
+        limit = "Unknown Resource Limit"
+
+
+class XCPUError(ResourceError):
+    """Error raised when CPU Time limit is reached"""
+    limit = "CPU Time"
+
+class LineageMemoryError(ResourceError):
+    """Error raised when the total amount of memory used by a process and
+    it's child is reached"""
+    limit = "Lineage total Memory"
+
+class TimeoutError(ResourceError):
+    """Error raised when the process is running for to much time"""
+    limit = "Real Time"
+
+# Can't use subclass because the StandardError MemoryError raised
+RESOURCE_LIMIT_EXCEPTION = (ResourceError, MemoryError)
+
+
+class MemorySentinel(Thread):
+    """A class checking a process don't use too much memory in a separated
+    daemonic thread
+    """
+    def __init__(self, interval, memory_limit, gpid=os.getpid()):
+        Thread.__init__(self, target=self._run, name="Test.Sentinel")
+        self.memory_limit = memory_limit
+        self._stop = Event()
+        self.interval = interval
+        self.setDaemon(True)
+        self.gpid = gpid
+
+    def stop(self):
+        """stop ap"""
+        self._stop.set()
+
+    def _run(self):
+        pil = ProcInfoLoader()
+        while not self._stop.isSet():
+            if self.memory_limit <= pil.load(self.gpid).lineage_memory_usage():
+                os.killpg(self.gpid, SIGUSR1)
+            self._stop.wait(self.interval)
+
+
+class ResourceController:
+
+    def __init__(self, max_cpu_time=None, max_time=None, max_memory=None,
+                 max_reprieve=60):
+        if SIGXCPU == -1:
+            raise RuntimeError("Unsupported platform")
+        self.max_time = max_time
+        self.max_memory = max_memory
+        self.max_cpu_time = max_cpu_time
+        self._reprieve = max_reprieve
+        self._timer = None
+        self._msentinel = None
+        self._old_max_memory = None
+        self._old_usr1_hdlr = None
+        self._old_max_cpu_time = None
+        self._old_usr2_hdlr = None
+        self._old_sigxcpu_hdlr = None
+        self._limit_set = 0
+        self._abort_try = 0
+        self._start_time = None
+        self._elapse_time = 0
+
+    def _hangle_sig_timeout(self, sig, frame):
+        raise TimeoutError()
+
+    def _hangle_sig_memory(self, sig, frame):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            raise LineageMemoryError("Memory limit reached")
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def _handle_sigxcpu(self, sig, frame):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            raise XCPUError("Soft CPU time limit reached")
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def _time_out(self):
+        if self._abort_try < self._reprieve:
+            self._abort_try += 1
+            os.killpg(os.getpid(), SIGUSR2)
+            if self._limit_set > 0:
+                self._timer = Timer(1, self._time_out)
+                self._timer.start()
+        else:
+            os.killpg(os.getpid(), SIGKILL)
+
+    def setup_limit(self):
+        """set up the process limit"""
+        assert currentThread().getName() == 'MainThread'
+        os.setpgrp()
+        if self._limit_set <= 0:
+            if self.max_time is not None:
+                self._old_usr2_hdlr = signal(SIGUSR2, self._hangle_sig_timeout)
+                self._timer = Timer(max(1, int(self.max_time) - self._elapse_time),
+                                    self._time_out)
+                self._start_time = int(time())
+                self._timer.start()
+            if self.max_cpu_time is not None:
+                self._old_max_cpu_time = getrlimit(RLIMIT_CPU)
+                cpu_limit = (int(self.max_cpu_time), self._old_max_cpu_time[1])
+                self._old_sigxcpu_hdlr = signal(SIGXCPU, self._handle_sigxcpu)
+                setrlimit(RLIMIT_CPU, cpu_limit)
+            if self.max_memory is not None:
+                self._msentinel = MemorySentinel(1, int(self.max_memory) )
+                self._old_max_memory = getrlimit(RLIMIT_AS)
+                self._old_usr1_hdlr = signal(SIGUSR1, self._hangle_sig_memory)
+                as_limit = (int(self.max_memory), self._old_max_memory[1])
+                setrlimit(RLIMIT_AS, as_limit)
+                self._msentinel.start()
+        self._limit_set += 1
+
+    def clean_limit(self):
+        """reinstall the old process limit"""
+        if self._limit_set > 0:
+            if self.max_time is not None:
+                self._timer.cancel()
+                self._elapse_time += int(time())-self._start_time
+                self._timer = None
+                signal(SIGUSR2, self._old_usr2_hdlr)
+            if self.max_cpu_time is not None:
+                setrlimit(RLIMIT_CPU, self._old_max_cpu_time)
+                signal(SIGXCPU, self._old_sigxcpu_hdlr)
+            if self.max_memory is not None:
+                self._msentinel.stop()
+                self._msentinel = None
+                setrlimit(RLIMIT_AS, self._old_max_memory)
+                signal(SIGUSR1, self._old_usr1_hdlr)
+        self._limit_set -= 1

data/tools/common/common/pyro_ext.py

@@ -0,0 +1,146 @@
+# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Python Remote Object utilities
+
+Main functions available:
+
+* `register_object` to expose arbitrary object through pyro using delegation
+  approach and register it in the nameserver.
+* `ns_unregister` unregister an object identifier from the nameserver.
+* `ns_get_proxy` get a pyro proxy from a nameserver object identifier.
+
+:organization: Logilab
+
+
+"""
+__docformat__ = "restructuredtext en"
+
+import logging
+import tempfile
+
+from Pyro import core, naming, errors, util, config
+
+_LOGGER = logging.getLogger('pyro')
+_MARKER = object()
+
+config.PYRO_STORAGE = tempfile.gettempdir()
+
+def ns_group_and_id(idstr, defaultnsgroup=_MARKER):
+    try:
+        nsgroup, nsid = idstr.rsplit('.', 1)
+    except ValueError:
+        if defaultnsgroup is _MARKER:
+            nsgroup = config.PYRO_NS_DEFAULTGROUP
+        else:
+            nsgroup = defaultnsgroup
+        nsid = idstr
+    if nsgroup is not None and not nsgroup.startswith(':'):
+        nsgroup = ':' + nsgroup
+    return nsgroup, nsid
+
+def host_and_port(hoststr):
+    if not hoststr:
+        return None, None
+    try:
+        hoststr, port = hoststr.split(':')
+    except ValueError:
+        port = None
+    else:
+        port = int(port)
+    return hoststr, port
+
+_DAEMONS = {}
+def _get_daemon(daemonhost, start=True):
+    if not daemonhost in _DAEMONS:
+        if not start:
+            raise Exception('no daemon for %s' % daemonhost)
+        if not _DAEMONS:
+            core.initServer(banner=0)
+        host, port = host_and_port(daemonhost)
+        daemon = core.Daemon(host=host, port=port)
+        _DAEMONS[daemonhost] = daemon
+    return _DAEMONS[daemonhost]
+
+
+def locate_ns(nshost):
+    """locate and return the pyro name server to the daemon"""
+    core.initClient(banner=False)
+    return naming.NameServerLocator().getNS(*host_and_port(nshost))
+
+
+def register_object(object, nsid, defaultnsgroup=_MARKER,
+                    daemonhost=None, nshost=None):
+    """expose the object as a pyro object and register it in the name-server
+
+    return the pyro daemon object
+    """
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    daemon = _get_daemon(daemonhost)
+    nsd = locate_ns(nshost)
+    # make sure our namespace group exists
+    try:
+        nsd.createGroup(nsgroup)
+    except errors.NamingError:
+        pass
+    daemon.useNameServer(nsd)
+    # use Delegation approach
+    impl = core.ObjBase()
+    impl.delegateTo(object)
+    daemon.connect(impl, '%s.%s' % (nsgroup, nsid))
+    _LOGGER.info('registered %s a pyro object using group %s and id %s',
+                 object, nsgroup, nsid)
+    return daemon
+
+
+def ns_unregister(nsid, defaultnsgroup=_MARKER, nshost=None):
+    """unregister the object with the given nsid from the pyro name server"""
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    try:
+        nsd = locate_ns(nshost)
+    except errors.PyroError, ex:
+        # name server not responding
+        _LOGGER.error('can\'t locate pyro name server: %s', ex)
+    else:
+        try:
+            nsd.unregister('%s.%s' % (nsgroup, nsid))
+            _LOGGER.info('%s unregistered from pyro name server', nsid)
+        except errors.NamingError:
+            _LOGGER.warning('%s not registered in pyro name server', nsid)
+
+
+def ns_get_proxy(nsid, defaultnsgroup=_MARKER, nshost=None):
+    nsgroup, nsid = ns_group_and_id(nsid, defaultnsgroup)
+    # resolve the Pyro object
+    try:
+        nsd = locate_ns(nshost)
+        pyrouri = nsd.resolve('%s.%s' % (nsgroup, nsid))
+    except errors.ProtocolError, ex:
+        raise errors.PyroError(
+            'Could not connect to the Pyro name server (host: %s)' % nshost)
+    except errors.NamingError:
+        raise errors.PyroError(
+            'Could not get proxy for %s (not registered in Pyro), '
+            'you may have to restart your server-side application' % nsid)
+    return core.getProxyForURI(pyrouri)
+
+
+def set_pyro_log_threshold(level):
+    pyrologger = logging.getLogger('Pyro.%s' % str(id(util.Log)))
+    # remove handlers so only the root handler is used
+    pyrologger.handlers = []
+    pyrologger.setLevel(level)