flowent 0.0.6 → 0.0.10

package/backend/src/flowent/logging.py

@@ -1,155 +1,196 @@
 from __future__ import annotations
 
 import logging
+import os
+import re
 import sys
-import time
-from collections.abc import Sequence
 from datetime import datetime
 from pathlib import Path
-from typing import TYPE_CHECKING, Literal
 
-from loguru import logger
+from flowent.paths import data_directory
+
+TRACE_LEVEL = 5
+DEFAULT_LOG_RETENTION = 5
+LITELLM_LOGGER_NAMES = ("LiteLLM", "LiteLLM Router", "LiteLLM Proxy")
+_configured_log_file: Path | None = None
+_configured_log_process_id: int | None = None
+_SECRET_PATTERNS = (
+    re.compile(r"(?i)\b(bearer)\s+([^\s,}]+)"),
+    re.compile(
+        r"(?i)(api[_-]?key|authorization|secret[_-]?reference|access[_-]?key)([\"'\s:=]+)([^\"'\s,}]+)"
+    ),
+)
 
-if TYPE_CHECKING:
-    from loguru import Record
 
-from flowent.config import Config
+def install_trace_level() -> None:
+    if logging.getLevelName(TRACE_LEVEL) != "TRACE":
+        logging.addLevelName(TRACE_LEVEL, "TRACE")
 
-RuntimeMode = Literal["dev", "release"]
 
-CONSOLE_FORMAT = (
-    "<green>{time:HH:mm:ss.SSS}</green> | "
-    "<level>{level: <8}</level> | "
-    "<cyan>{extra[source]}</cyan>{extra[agent_suffix]} | "
-    "<level>{message}</level>{exception}"
-)
+def redact_log_value(value: object) -> str:
+    text = str(value)
+    for pattern in _SECRET_PATTERNS:
+        text = pattern.sub(
+            lambda match: f"{match.group(1)}{match.group(2)}[REDACTED]", text
+        )
+    return text
 
-FILE_FORMAT = (
-    "{time:YYYY-MM-DD HH:mm:ss.SSS} | "
-    "{level: <8} | "
-    "{extra[source]}{extra[agent_suffix]} | "
-    "{message}{exception}"
-)
 
+def log_directory(directory: Path | None = None) -> Path:
+    return (directory or data_directory()) / "logs"
 
-class HealthcheckAccessFilter(logging.Filter):
-    def filter(self, record: logging.LogRecord) -> bool:
-        args = record.args
-        if not isinstance(args, tuple) or len(args) < 3:
-            return True
-        full_path = args[2]
-        if not isinstance(full_path, str):
-            return True
-        return full_path.split("?", 1)[0] != "/health"
-
-
-def detect_runtime_mode(
-    argv: Sequence[str] | None = None,
-    config: Config | None = None,
-) -> RuntimeMode:
-    args = list(sys.argv if argv is None else argv)
-    executable = Path(args[0]).name.lower() if args else ""
-    command = args[1] if len(args) > 1 else ""
-
-    if "fastapi" in executable and command == "dev":
-        return "dev"
-    if "fastapi" in executable and command == "run":
-        return "release"
-    if command == "dev":
-        return "dev"
-    if command == "run":
-        return "release"
-    if "--reload" in args:
-        return "dev"
-    if config is not None and config.DEBUG:
-        return "dev"
-    return "release"
-
-
-def _patch_record(record: Record) -> None:
-    agent_id = record["extra"].get("agent_id")
-    record["extra"]["agent_suffix"] = f" | agent:{agent_id}" if agent_id else ""
-    record["extra"]["source"] = (
-        f"{record['name']}:{record['function']}:{record['line']}"
-    )
 
+def parse_log_level(value: str | None, default: int) -> int:
+    if not value:
+        return default
+    normalized = value.strip().upper()
+    if normalized == "TRACE":
+        return TRACE_LEVEL
+    if normalized.isdigit():
+        return int(normalized)
+    level = logging.getLevelName(normalized)
+    return level if isinstance(level, int) else default
 
-def _get_log_dir(runtime_dir: Path | None = None) -> Path:
-    base_dir = Path.cwd() if runtime_dir is None else runtime_dir
-    return base_dir / "logs"
 
+def development_mode() -> bool:
+    return os.environ.get("DEBUG", "").strip().lower() in {"1", "true", "yes", "on"}
 
-def _allocate_log_file(log_dir: Path) -> Path:
-    timestamp = time.time_ns() // 1_000_000
-    prefix = datetime.fromtimestamp(timestamp / 1000).strftime("%Y-%m-%d_%H%M%S")
-    log_path = log_dir / f"{prefix}_{timestamp}.log"
-    while log_path.exists():
-        timestamp += 1
-        prefix = datetime.fromtimestamp(timestamp / 1000).strftime("%Y-%m-%d_%H%M%S")
-        log_path = log_dir / f"{prefix}_{timestamp}.log"
-    return log_path
 
+def console_log_level() -> int:
+    default = logging.DEBUG if development_mode() else logging.INFO
+    return parse_log_level(os.environ.get("LOG_LEVEL"), default)
 
-def _sort_key(path: Path) -> tuple[int, int, str]:
+
+def litellm_handler_level_name(value: str | None) -> str:
+    if not value:
+        return "INFO"
+    normalized = value.strip().upper()
+    if normalized.isdigit():
+        numeric_level = int(normalized)
+        if numeric_level <= logging.INFO:
+            return "INFO"
+        level_name = logging.getLevelName(numeric_level)
+        return level_name if isinstance(level_name, str) else "INFO"
+    level = parse_log_level(normalized, logging.INFO)
+    if level < logging.INFO:
+        return "INFO"
+    if normalized == "WARN":
+        return "WARNING"
+    if isinstance(logging.getLevelName(normalized), int):
+        return normalized
+    return "INFO"
+
+
+def configure_litellm_logging() -> None:
+    os.environ["LITELLM_LOG"] = litellm_handler_level_name(
+        os.environ.get("LITELLM_LOG")
+    )
+    handlers_to_close: set[logging.Handler] = set()
+    for logger_name in LITELLM_LOGGER_NAMES:
+        litellm_logger = logging.getLogger(logger_name)
+        for handler in list(litellm_logger.handlers):
+            litellm_logger.removeHandler(handler)
+            handlers_to_close.add(handler)
+        litellm_logger.propagate = True
+    for handler in handlers_to_close:
+        handler.close()
+
+
+def log_retention() -> int:
+    raw_retention = os.environ.get("FLOWENT_LOG_RETENTION")
+    if not raw_retention:
+        return DEFAULT_LOG_RETENTION
     try:
-        timestamp = int(path.stem.rsplit("_", 1)[-1])
+        return max(1, int(raw_retention))
     except ValueError:
+        return DEFAULT_LOG_RETENTION
+
+
+def new_log_file_path(directory: Path | None = None) -> Path:
+    logs = log_directory(directory)
+    timestamp = datetime.now().strftime("%Y%m%d-%H%M%S")
+    return logs / f"flowent-{timestamp}-{os.getpid()}.log"
+
+
+def prune_old_logs(logs: Path, *, keep: int = DEFAULT_LOG_RETENTION) -> None:
+    files = sorted(logs.glob("flowent-*.log"), key=lambda item: item.name)
+    for old_log in files[:-keep]:
         try:
-            timestamp = int(path.stem)
-        except ValueError:
-            timestamp = -1
-    return (timestamp, path.stat().st_mtime_ns, path.name)
-
-
-def prune_old_logs(log_dir: Path, keep: int = 10) -> None:
-    log_files = [path for path in log_dir.iterdir() if path.is_file()]
-    log_files.sort(key=_sort_key, reverse=True)
-    for path in log_files[keep:]:
-        path.unlink(missing_ok=True)
-
-
-def configure_uvicorn_access_logging() -> None:
-    access_logger = logging.getLogger("uvicorn.access")
-    if any(isinstance(f, HealthcheckAccessFilter) for f in access_logger.filters):
-        return
-    access_logger.addFilter(HealthcheckAccessFilter())
-
-
-def setup_logging(
-    config: Config | None = None,
-    *,
-    argv: Sequence[str] | None = None,
-    runtime_dir: Path | None = None,
-) -> None:
-    if config is None:
-        config = Config()
-
-    mode = detect_runtime_mode(argv=argv, config=config)
-    console_level = "DEBUG" if mode == "dev" else "INFO"
-
-    logger.remove()
-    logger.configure(patcher=_patch_record)
-    configure_uvicorn_access_logging()
-
-    log_dir = _get_log_dir(runtime_dir)
-    log_dir.mkdir(parents=True, exist_ok=True)
-    log_path = _allocate_log_file(log_dir)
-    log_path.touch()
-    prune_old_logs(log_dir)
-
-    logger.add(
-        sys.stderr,
-        format=CONSOLE_FORMAT,
-        level=console_level,
-        colorize=True,
-        backtrace=True,
-        diagnose=mode == "dev",
+            old_log.unlink()
+        except OSError as error:
+            logging.getLogger("flowent.logging").warning(
+                "Could not remove old log file %s: %s", old_log, error
+            )
+
+
+class RedactingFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        rendered = super().format(record)
+        return redact_log_value(rendered)
+
+
+class ConsoleNoiseFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        return record.levelno > logging.DEBUG or record.name.startswith("flowent")
+
+
+def configure_logging(*, directory: Path | None = None) -> Path:
+    global _configured_log_file, _configured_log_process_id
+
+    install_trace_level()
+    configure_litellm_logging()
+
+    log_file = new_log_file_path(directory)
+    log_file.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+
+    root_logger = logging.getLogger()
+    for handler in list(root_logger.handlers):
+        root_logger.removeHandler(handler)
+        handler.close()
+
+    root_logger.setLevel(TRACE_LEVEL)
+
+    file_handler = logging.FileHandler(log_file, encoding="utf-8")
+    file_handler.setLevel(TRACE_LEVEL)
+    file_handler.setFormatter(
+        RedactingFormatter(
+            "%(asctime)s %(levelname)s [%(process)d] %(name)s: %(message)s"
+        )
     )
 
-    logger.add(
-        log_path,
-        format=FILE_FORMAT,
-        level="TRACE",
-        backtrace=True,
-        diagnose=mode == "dev",
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setLevel(console_log_level())
+    console_handler.setFormatter(
+        RedactingFormatter("%(levelname)s %(name)s: %(message)s")
     )
+    console_handler.addFilter(ConsoleNoiseFilter())
+
+    root_logger.addHandler(file_handler)
+    root_logger.addHandler(console_handler)
+    configure_litellm_logging()
+
+    prune_old_logs(log_file.parent, keep=log_retention())
+
+    logger = logging.getLogger("flowent")
+    logger.info("Flowent logging initialized")
+    logger.info("Data directory: %s", directory or data_directory())
+    logger.info("Log file: %s", log_file)
+    logger.debug("Console log level: %s", logging.getLevelName(console_log_level()))
+    logger.log(TRACE_LEVEL, "File log level: TRACE")
+
+    _configured_log_file = log_file
+    _configured_log_process_id = os.getpid()
+
+    return log_file
+
+
+def ensure_logging_configured(*, directory: Path | None = None) -> Path:
+    target_log_directory = log_directory(directory).resolve(strict=False)
+    if (
+        _configured_log_file is not None
+        and _configured_log_process_id == os.getpid()
+        and _configured_log_file.parent.resolve(strict=False) == target_log_directory
+    ):
+        return _configured_log_file
+
+    return configure_logging(directory=directory)