codetac 0.1.0 → 0.2.0

package/src/python/codetac_py/detail.py

@@ -0,0 +1,333 @@
+"""Detail on request (Phase 4 of the Node version, spec 3.8).
+
+The panel writes .codetac/<recording>/detalhe.json: functions (file, line,
+name) and whole files. For those functions only, and from their next call:
+
+- the arguments, read from the frame when the call starts;
+- the value returned, or the type of the exception;
+- the lines that ran: LINE events are switched on for those code objects
+  only (sys.monitoring.set_local_events), so the rest of the app pays
+  nothing.
+
+The `detail` event has the format of the Node runtime (src/runtime.mjs), so
+the panel does not change. The file is read every 400 ms by the writer's
+thread (no thread of its own: see writer.py on daemon threads at exit).
+
+preview() never runs code of the app: no __repr__, __str__, properties or
+__getattr__ of any type outside the standard library. Types are tested with
+issubclass on the real type (isinstance may read an overridden __class__),
+built-in containers are read through the methods of their base type, and
+objects of the app through the C descriptor of their __dict__ only.
+"""
+import datetime
+import decimal
+import enum
+import json
+import math
+import os
+import sys
+import types
+import uuid
+from pathlib import PurePath
+
+from .redact import MARKER, is_sensitive_name
+
+MONITORING = sys.monitoring
+LINE = MONITORING.events.LINE
+CO_VARARGS = 0x0004
+CO_VARKEYWORDS = 0x0008
+LIMITS = {'depth': 4, 'items': 20, 'keys': 30, 'nodes': 300, 'text': 2000}
+
+
+# preview ----------------------------------------------------------------------------
+
+_TYPE_DICT = type.__dict__['__dict__']        # a class's own namespace, without its metaclass
+_TYPE_QUALNAME = type.__dict__['__qualname__']
+_TYPE_MODULE = type.__dict__['__module__']
+_TYPE_MRO = type.__dict__['__mro__']
+_GETSET = types.GetSetDescriptorType
+
+
+def _class_name(cls):
+    try:
+        return _TYPE_QUALNAME.__get__(cls)
+    except Exception:
+        return '?'
+
+
+def _class_module(cls):
+    try:
+        module = _TYPE_MODULE.__get__(cls)
+        return module if isinstance(module, str) else ''
+    except Exception:
+        return ''
+
+
+def _instance_dict(item):
+    """The instance's __dict__, through the C descriptor of the class that has one; None when
+    the class defines __dict__ itself (a property: app code) or has none (slots, C types)."""
+    for cls in _TYPE_MRO.__get__(type(item)):
+        found = _TYPE_DICT.__get__(cls).get('__dict__')
+        if found is None:
+            continue
+        if type(found) is _GETSET:
+            try:
+                value = found.__get__(item)
+            except Exception:
+                return None
+            return value if type(value) is dict else None
+        return None
+    return None
+
+
+def _query_keys(query):
+    keys = []
+    for part in query.split('&'):
+        key = part.split('=', 1)[0]
+        if key and key not in keys:
+            keys.append(key)
+    return keys
+
+
+def _path(path, query):
+    keys = _query_keys(query) if query else []
+    return path + ('?' + '&'.join('%s=…' % key for key in keys) if keys else '')
+
+
+def _http_summary(item, cls, fields):
+    """Requests and responses of Starlette, werkzeug and Flask, from their own fields:
+    method, path without query values, header names; never header values or bodies."""
+    module = _class_module(cls)
+    if not module.startswith(('starlette.', 'fastapi.', 'werkzeug.', 'flask.')):
+        return None
+    name = _class_name(cls)
+    scope = fields.get('scope')
+    if type(scope) is dict and scope.get('type') in ('http', 'websocket'):
+        query = scope.get('query_string') or b''
+        headers = [key.decode('latin-1') for key, _ in list(scope.get('headers') or [])[:40] if type(key) is bytes]
+        return {'$classe': name, 'metodo': scope.get('method'), 'caminho': _path(str(scope.get('path', '')), query.decode('latin-1')),
+                'cabecalhos': headers}
+    environ = fields.get('environ')
+    if type(environ) is dict and 'REQUEST_METHOD' in environ:
+        headers = [key[5:].replace('_', '-').lower() for key in environ if type(key) is str and key.startswith('HTTP_')][:40]
+        return {'$classe': name, 'metodo': environ.get('REQUEST_METHOD'), 'caminho': _path(str(environ.get('PATH_INFO', '')),
+                                                                                       str(environ.get('QUERY_STRING', ''))),
+                'cabecalhos': headers}
+    if 'status_code' in fields and ('raw_headers' in fields or 'body' in fields):
+        raw = fields.get('raw_headers') or []
+        return {'$classe': name, 'estado': fields.get('status_code') if type(fields.get('status_code')) is int else None,
+                'cabecalhos': [key.decode('latin-1') for key, _ in list(raw)[:40] if type(key) is bytes]}
+    if '_status_code' in fields or '_status' in fields:
+        code = fields.get('_status_code')
+        return {'$classe': name, 'estado': code if type(code) is int else None}
+    return {'$classe': name, '$resumido': True}
+
+
+def preview(value):
+    """A readable copy for the detail view: plain JSON, bounded in depth, length and size.
+    Redaction happens when the event is emitted."""
+    nodes = [0]
+    seen = set()
+
+    def text(item):
+        return item if len(item) <= LIMITS['text'] else '%s[TRUNCATED %d]' % (item[:LIMITS['text']], len(item))
+
+    def walk(item, depth):
+        nodes[0] += 1
+        if nodes[0] > LIMITS['nodes']:
+            return '[…]'
+        cls = type(item)
+        if item is None or cls is bool:
+            return item
+        if cls is int:
+            return item if -2 ** 53 < item < 2 ** 53 else int.__repr__(item)
+        if cls is float:
+            return item if math.isfinite(item) else float.__repr__(item)
+        if cls is str:
+            return text(item)
+        if issubclass(cls, (bytes, bytearray, memoryview)):
+            return {'$tipo': 'bytes', 'bytes': len(item) if cls in (bytes, bytearray) else memoryview.__len__(item)}
+        if cls in (types.FunctionType, types.BuiltinFunctionType, types.MethodType):
+            return {'$tipo': 'função', 'nome': getattr(item, '__qualname__', None) if cls is not types.MethodType else '(método)'}
+        if issubclass(cls, type):
+            return {'$tipo': 'classe', 'nome': _class_name(item)}
+        if cls is types.ModuleType:
+            return {'$tipo': 'módulo'}
+        if cls in (types.GeneratorType, types.CoroutineType, types.AsyncGeneratorType):
+            return {'$tipo': {types.GeneratorType: 'gerador', types.CoroutineType: 'corrotina'}.get(cls, 'gerador assíncrono')}
+        if issubclass(cls, BaseException):
+            # The type only: the message comes from __str__, which may be the app's.
+            return {'$tipo': 'erro', 'nome': _class_name(cls)}
+        if cls in (datetime.datetime, datetime.date, datetime.time):
+            return {'$tipo': 'data', 'valor': cls.isoformat(item)}
+        if cls is decimal.Decimal:
+            return {'$tipo': 'decimal', 'valor': decimal.Decimal.__str__(item)}
+        if cls is uuid.UUID:
+            return {'$tipo': 'uuid', 'valor': uuid.UUID.__str__(item)}
+        if issubclass(cls, PurePath) and _class_module(cls).startswith('pathlib'):
+            return {'$tipo': 'caminho', 'valor': PurePath.__str__(item)}
+        key = id(item)
+        if key in seen:
+            return {'$tipo': 'circular'}
+        name = _class_name(cls)
+        if depth >= LIMITS['depth']:
+            return {'$tipo': name, '$resumido': True}
+        seen.add(key)
+        try:
+            if issubclass(cls, enum.Enum):
+                fields = _instance_dict(item) or {}
+                return {'$classe': name, 'nome': fields.get('_name_') if type(fields.get('_name_')) is str else None}
+            if issubclass(cls, dict):
+                out = {}
+                entries = list(dict.items(item))
+                for entry_key, entry in entries[:LIMITS['keys']]:
+                    label = entry_key if type(entry_key) is str else walk(entry_key, depth + 1) if type(entry_key) in (int, float, bool) or entry_key is None else '[%s]' % _class_name(type(entry_key))
+                    out[str(label)] = walk(entry, depth + 1)
+                if len(entries) > LIMITS['keys']:
+                    out['$mais'] = len(entries) - LIMITS['keys']
+                if cls is not dict:
+                    out = {'$classe': name, **out}
+                return out
+            if issubclass(cls, (list, tuple)):
+                base = list if issubclass(cls, list) else tuple
+                size = base.__len__(item)
+                out = [walk(base.__getitem__(item, index), depth + 1) for index in range(min(size, LIMITS['items']))]
+                if size > LIMITS['items']:
+                    out.append({'$mais': size - LIMITS['items']})
+                return out
+            if issubclass(cls, (set, frozenset)):
+                base = set if issubclass(cls, set) else frozenset
+                values = []
+                for entry in base.__iter__(item):
+                    if len(values) >= LIMITS['items']:
+                        break
+                    values.append(walk(entry, depth + 1))
+                return {'$tipo': 'conjunto', 'tamanho': base.__len__(item), 'valores': values}
+            fields = _instance_dict(item)
+            if fields is None:
+                # Slots, C types (connections, sockets, locks…): the class only.
+                return {'$classe': name, '$resumido': True}
+            summary = _http_summary(item, cls, fields)
+            if summary is not None:
+                return summary
+            out = {'$classe': name}
+            # Fields starting with "_" are internals (frameworks, ORMs), not the app's data.
+            keys = [field for field in fields if type(field) is str and not field.startswith('_')]
+            for field in keys[:LIMITS['keys']]:
+                out[field] = walk(fields[field], depth + 1)
+            if len(keys) > LIMITS['keys']:
+                out['$mais'] = len(keys) - LIMITS['keys']
+            return out
+        except Exception:
+            return {'$tipo': 'ilegível'}
+        finally:
+            seen.discard(key)
+
+    return walk(value, 0)
+
+
+# Detail on request ------------------------------------------------------------------
+
+def parameter_names(code):
+    """In the order of the signature: positional, *args, keyword-only, **kwargs
+    (co_varnames keeps *args after the keyword-only ones)."""
+    names = code.co_varnames
+    positional, keyword = code.co_argcount, code.co_kwonlyargcount
+    varargs = (names[positional + keyword],) if code.co_flags & CO_VARARGS else ()
+    extra = positional + keyword + len(varargs)
+    varkw = (names[extra],) if code.co_flags & CO_VARKEYWORDS else ()
+    return names[:positional] + varargs + names[positional:positional + keyword] + varkw
+
+
+def template_line(table, line):
+    """The template line of a generated line (Template.get_corresponding_lineno)."""
+    for template, generated in reversed(table):
+        if generated <= line:
+            return template
+    return 1
+
+
+class Detail(object):
+    def __init__(self, capture, writer, directory):
+        self.capture = capture
+        self.writer = writer
+        self.file = os.path.join(directory, 'detalhe.json')
+        self.stamp = None
+        self.functions = set()   # "file:line:name"
+        self.files = set()
+        self.codes = {}          # code object -> parameter names, for the functions with detail
+        self.calls = {}          # id(frame) -> [args, lines, node, request literal, code]
+
+    # detalhe.json -----------------------------------------------------------------
+    def poll(self):
+        try:
+            stat = os.stat(self.file)
+            stamp = (stat.st_mtime_ns, stat.st_size)
+        except OSError:
+            stamp = None
+        if stamp == self.stamp:
+            return
+        self.stamp = stamp
+        spec = {}
+        if stamp is not None:
+            try:
+                with open(self.file, 'rb') as file:
+                    spec = json.loads(file.read().decode('utf-8'))
+            except (OSError, ValueError):
+                spec = {}
+        functions = spec.get('functions') if isinstance(spec, dict) else None
+        files = spec.get('files') if isinstance(spec, dict) else None
+        self.functions = set('%s:%s:%s' % (item.get('file'), item.get('line'), item.get('function') or '')
+                             for item in functions or [] if isinstance(item, dict))
+        self.files = set(item for item in files or [] if isinstance(item, str))
+        for code, place in list(self.capture.places_of.items()):
+            self.consider(code, place)
+
+    def wanted(self, place):
+        file, line, name = place
+        return file in self.files or '%s:%s:%s' % (file, line, name) in self.functions or '%s:%s:' % (file, line) in self.functions
+
+    def consider(self, code, place):
+        """Switches detail on or off for one code object of the project."""
+        wanted = self.wanted(place)
+        tool = self.capture.tool
+        if wanted and code not in self.codes:
+            self.codes[code] = parameter_names(code)
+            MONITORING.set_local_events(tool, code, LINE)
+        elif not wanted and code in self.codes:
+            del self.codes[code]
+            MONITORING.set_local_events(tool, code, 0)
+
+    # Calls ------------------------------------------------------------------------
+    def start(self, code, frame, node, request):
+        try:
+            names = self.codes.get(code)
+            if names is None:
+                return
+            local = frame.f_locals
+            args = [{'name': name, 'value': MARKER if is_sensitive_name(name) else preview(local.get(name))} for name in names]
+        except Exception:
+            args = []
+        self.calls[id(frame)] = [args, set(), node, request, code]
+
+    def line(self, frame, code, line):
+        call = self.calls.get(id(frame))
+        if call is not None and call[4] is code:
+            table = self.capture.template_tables.get(code)
+            call[1].add(template_line(table, line) if table is not None else line)
+
+    def finish(self, frame, code, returned=None, exception=None):
+        call = self.calls.pop(id(frame), None)
+        if call is None or call[4] is not code:
+            return
+        args, lines, node, request, _ = call
+        from .context import request_of
+        request_id, _ = request_of(request)
+        event = {'type': 'detail', 'id': '%s:%d' % (self.writer.process, node), 'requestId': request_id,
+                 'args': args, 'lines': sorted(lines)}
+        if exception is not None and type(exception) is not GeneratorExit:
+            event['threw'] = preview(exception)
+        else:
+            name = self.capture.places_of.get(code, (None, None, ''))[2]
+            event['returned'] = MARKER if is_sensitive_name(name.rsplit('.', 1)[-1]) else preview(returned)
+        self.writer.emit(event)

package/src/python/codetac_py/files.py

@@ -0,0 +1,135 @@
+"""File boundaries (stage 7), as installFiles in src/boundaries.mjs: only
+operations made directly by the project's code during a request.
+
+  boundary  {kind: 'ficheiros', provider: 'sistema de ficheiros', library: 'builtins' | 'os',
+             function, operation, path}
+
+- open() in a mode that writes (w, a, x, +): 'escrita'. pathlib's
+  write_text, write_bytes and open go through io.open;
+- os.remove/unlink/rmdir ('remoção'), os.rename/replace ('mudança de
+  nome'), os.mkdir ('criação de pasta'; os.makedirs and Path.mkdir use it).
+
+The first caller frame outside the captor and the standard library decides:
+a library (site-packages, a virtual environment) is not the project. Reads
+are not recorded (M66). The captor's own files and CodeTAC's data folder are
+never boundaries; the captor writes with os.open/os.write, which are not
+observed.
+
+Keep it importable on old Pythons (3.8+): the minimal mode records boundaries.
+"""
+import builtins
+import functools
+import io
+import os
+import sys
+import sysconfig
+
+from .boundaries import _inside, start_boundary
+from .context import NULL, current
+from .project import CAPTOR, ProjectFiles
+from .servers import safe_path
+
+WRITES = set('wax+')
+OPERATIONS = {'remove': 'remoção', 'unlink': 'remoção', 'rmdir': 'remoção', 'rename': 'mudança de nome',
+              'replace': 'mudança de nome', 'mkdir': 'criação de pasta'}
+DEPTH = 12
+# The captor's code, by its real path and by the path it was imported from.
+CODE = tuple(set(folder + os.sep for folder in (CAPTOR, os.path.dirname(os.path.abspath(__file__)))))
+
+
+def _standard_library():
+    paths = sysconfig.get_paths()
+    folders = set()
+    for key in ('stdlib', 'platstdlib'):
+        if paths.get(key):
+            folders.add(os.path.realpath(paths[key]) + os.sep)
+            folders.add(os.path.abspath(paths[key]) + os.sep)
+    return tuple(folders)
+
+
+class Files(object):
+    def __init__(self, root, data):
+        self.root = os.path.realpath(root)
+        self.project_file = ProjectFiles(self.root)
+        self.own = tuple(os.path.realpath(folder) + os.sep for folder in (CAPTOR, data))
+        self.stdlib = _standard_library()
+
+    def caller_is_project(self):
+        frame = sys._getframe(3)
+        for _ in range(DEPTH):
+            if frame is None:
+                return False
+            filename = frame.f_code.co_filename
+            if 'site-packages' in filename or 'dist-packages' in filename:
+                return False
+            if filename.startswith(CODE) or filename.startswith('<frozen') or filename.startswith(self.stdlib):
+                frame = frame.f_back
+                continue
+            return self.project_file(filename) is not None
+        return False
+
+    def describe(self, target):
+        if isinstance(target, int):
+            return '(descritor)'
+        path = os.fsdecode(os.fspath(target))
+        real = os.path.realpath(path)
+        if (real + os.sep).startswith(self.own):
+            return None
+        within = os.path.relpath(real, self.root)
+        if within != os.curdir and not within.startswith(os.pardir) and not os.path.isabs(within):
+            path = within
+        return safe_path(path)
+
+    def start(self, function, operation, target):
+        """end(extra), or None when this operation is not a boundary."""
+        if current.get()[1] == NULL or _inside.get() or not self.caller_is_project():
+            return None
+        try:
+            path = self.describe(target)
+        except (TypeError, ValueError, OSError):
+            return None
+        if path is None:
+            return None
+        return start_boundary({'kind': 'ficheiros', 'provider': 'sistema de ficheiros',
+                               'library': 'builtins' if function == 'open' else 'os',
+                               'function': function, 'operation': operation, 'path': path})
+
+
+def _observe(files, function, operation, original):
+    @functools.wraps(original)
+    def wrapper(target, *args, **kwargs):
+        end = files.start(function, operation, target)
+        if end is None:
+            return original(target, *args, **kwargs)
+        try:
+            result = original(target, *args, **kwargs)
+        except BaseException:
+            end({'error': True})
+            raise
+        end()
+        return result
+    wrapper.__codetac__ = True
+    return wrapper
+
+
+def install(root, data):
+    if getattr(builtins.open, '__codetac__', False):
+        return
+    files = Files(root, data)
+    original_open = builtins.open
+    writing = _observe(files, 'open', 'escrita', original_open)
+
+    @functools.wraps(original_open)
+    def codetac_open(file, mode='r', *args, **kwargs):
+        if isinstance(mode, str) and WRITES.intersection(mode):
+            return writing(file, mode, *args, **kwargs)
+        return original_open(file, mode, *args, **kwargs)
+
+    codetac_open.__codetac__ = True
+    builtins.open = codetac_open
+    if io.open is original_open:
+        io.open = codetac_open
+    for name, operation in OPERATIONS.items():
+        original = getattr(os, name, None)
+        if original is not None and not getattr(original, '__codetac__', False):
+            setattr(os, name, _observe(files, name, operation, original))

package/src/python/codetac_py/frameworks.py

@@ -0,0 +1,72 @@
+"""What only a framework knows, when the dossier would otherwise have a gap.
+
+- FastAPI: the body of a request is validated by pydantic-core, compiled
+  (Rust), where no Python function runs. The dossier shows an opaque step,
+  "validação do pedido (não observável)", around it: its time, whether it
+  failed (422), and any validators of the project as its children. The point
+  is fastapi.dependencies.utils.request_body_to_args, which solve_dependencies
+  looks up in its module at each call.
+
+Keep it importable on old Pythons (3.8+); in the minimal mode, without
+functions, nothing is recorded here.
+"""
+import functools
+import inspect
+
+from .hooks import register
+
+VALIDATION = 'validação do pedido (não observável)'
+
+
+def _monitor():
+    import codetac_py
+    return codetac_py.monitor if codetac_py.role == 'app' else None
+
+
+def _patch_fastapi_dependencies(module):
+    original = module.request_body_to_args
+    if getattr(original, '__codetac_wrapped__', False):
+        return
+
+    def failed(result):
+        # (values, errors): errors is the list of validation errors.
+        try:
+            return bool(result[1])
+        except (TypeError, IndexError, KeyError):
+            return False
+
+    if inspect.iscoroutinefunction(original):
+        @functools.wraps(original)
+        async def request_body_to_args(*args, **kwargs):
+            monitor = _monitor()
+            if monitor is None:
+                return await original(*args, **kwargs)
+            end = monitor.opaque(VALIDATION)
+            error = True
+            try:
+                result = await original(*args, **kwargs)
+                error = failed(result)
+                return result
+            finally:
+                end(error)
+    else:
+        @functools.wraps(original)
+        def request_body_to_args(*args, **kwargs):
+            monitor = _monitor()
+            if monitor is None:
+                return original(*args, **kwargs)
+            end = monitor.opaque(VALIDATION)
+            error = True
+            try:
+                result = original(*args, **kwargs)
+                error = failed(result)
+                return result
+            finally:
+                end(error)
+
+    request_body_to_args.__codetac_wrapped__ = True
+    module.request_body_to_args = request_body_to_args
+
+
+def install():
+    register('fastapi.dependencies.utils', _patch_fastapi_dependencies)

package/src/python/codetac_py/hooks.py

@@ -0,0 +1,72 @@
+"""Post-import hooks: a module of a library is patched right after it is
+imported, before the code that imported it can bind its names. Used by
+servers.py (servers) and boundaries.py (databases, HTTP...).
+
+Keep it importable on old Pythons (3.8+).
+"""
+import sys
+
+HOOKS = {}
+
+
+def register(name, hook):
+    """Runs hook(module) when `name` is imported, or now if it already was."""
+    HOOKS.setdefault(name, []).append(hook)
+    module = sys.modules.get(name)
+    if module is not None:  # imported before the capture started
+        _apply(hook, module)
+    _install()
+
+
+def _apply(hook, module):
+    try:
+        hook(module)
+    except Exception as error:  # a version we do not know: it runs unobserved
+        sys.stderr.write('codeTAC: %s não foi preparado (%s: %s)\n' % (module.__name__, type(error).__name__, error))
+
+
+class _Loader(object):
+    """Runs the real loader, then the hooks. Everything else is the real loader's."""
+
+    def __init__(self, loader, hooks):
+        self._loader = loader
+        self._hooks = hooks
+
+    def __getattr__(self, name):
+        return getattr(self._loader, name)
+
+    def create_module(self, spec):
+        return self._loader.create_module(spec)
+
+    def exec_module(self, module):
+        # The module keeps the real loader: nothing else sees this wrapper.
+        module.__loader__ = self._loader
+        if getattr(module, '__spec__', None) is not None:
+            module.__spec__.loader = self._loader
+        self._loader.exec_module(module)
+        for hook in self._hooks:
+            _apply(hook, module)
+
+
+class _Finder(object):
+    def find_spec(self, name, path=None, target=None):
+        hooks = HOOKS.get(name)
+        if not hooks:
+            return None
+        for finder in sys.meta_path:
+            if finder is self or not hasattr(finder, 'find_spec'):
+                continue
+            spec = finder.find_spec(name, path, target)
+            if spec is not None:
+                if spec.loader is not None and hasattr(spec.loader, 'exec_module'):
+                    spec.loader = _Loader(spec.loader, hooks)
+                return spec
+        return None
+
+    def invalidate_caches(self):
+        pass
+
+
+def _install():
+    if not any(isinstance(finder, _Finder) for finder in sys.meta_path):
+        sys.meta_path.insert(0, _Finder())