@ngocsangairvds/vsaf 3.2.15 → 4.0.0

package/tools/vds-scripts/telegram_bridge/daemon/__init__.py

@@ -1 +0,0 @@
-"""telegram_bridge daemon — long-running aiogram bot + per-agent runner sidecars."""

package/tools/vds-scripts/telegram_bridge/daemon/__main__.py

@@ -1,351 +0,0 @@
-"""Daemon entrypoint — `python -m telegram_bridge.daemon`.
-
-Lifecycle:
-  1. Load settings from ~/.vds/.env (mounted at /etc/vds/.env in container).
-  2. Open Postgres pool; assert Alembic head (MigrationHeadMismatch → SystemExit).
-  3. Acquire daemon PID-lock at /run/vds-telegram/daemon.pid (FR-8.3 — one-bot-one-container).
-  4. Register signal handlers (SIGTERM, SIGINT) for clean shutdown drain (FR-9.7).
-  5. Build Bot + Dispatcher; install allow-list outer-middleware.
-  6. Start PG outage watchdog (TSK-7.7); alarms general topic on sustained outage.
-  7. Register command routers (P1-F — Wave 3).
-  8. Run preflight `getChatMember` permission check if supergroup is registered.
-  9. Start long-polling via `Dispatcher.start_polling(bot)`.
- 10. On signal: `dispatcher.stop_polling()` + drain pending throttler edits + close pool.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import signal
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-import structlog
-
-from telegram_bridge.daemon.alembic_head_check import (
-    MigrationHeadMismatch,
-    assert_migrations_at_head,
-)
-from telegram_bridge.daemon.audit import (
-    AuditWriter,
-    BlobInterceptor,
-    PassthroughInterceptor,
-    PatternRedactionInterceptor,
-)
-from telegram_bridge.daemon.bot import build_bot, build_dispatcher
-from telegram_bridge.daemon.bot_gateway import AiogramStreamingGateway, AiogramTopicGateway
-from telegram_bridge.daemon.bus import SessionBus
-from telegram_bridge.daemon.commands import build_router
-from telegram_bridge.daemon.config import load_settings
-from telegram_bridge.daemon.dispatcher import SessionDispatcher
-from telegram_bridge.daemon.health_server import HealthServer
-from telegram_bridge.daemon.job_scheduler import NightlyJobScheduler
-from telegram_bridge.daemon.lifecycle import LifecycleTicker
-from telegram_bridge.daemon.pg import PgPool
-from telegram_bridge.daemon.pg_outage_watchdog import PgOutageWatchdog
-from telegram_bridge.daemon.polling_state import load_last_offset
-from telegram_bridge.daemon.preflight import PreflightError, check_bot_admin_permissions
-from telegram_bridge.daemon.registry_loader import load_registry
-from telegram_bridge.daemon.runner_orchestrator import RunnerOrchestrator
-from telegram_bridge.daemon.session_registry import SessionRegistry
-from telegram_bridge.daemon.shim_server import ShimServer
-from telegram_bridge.daemon.throttler import ChatTokenBucket, StreamingThrottler
-from telegram_bridge.daemon.topic_promoter import TopicPromoter
-from telegram_bridge.daemon.uploads import build_uploads_router
-
-if TYPE_CHECKING:
-    from aiogram import Bot, Dispatcher
-
-log = structlog.get_logger(__name__)
-
-
-class _DaemonState:
-    """Container for objects that need clean shutdown."""
-
-    dispatcher: Dispatcher | None
-    bot: Bot | None
-    pg: PgPool | None
-    orchestrator: RunnerOrchestrator | None
-    lifecycle: LifecycleTicker | None
-    session_dispatcher: SessionDispatcher | None
-    shim_server: ShimServer | None
-    health_server: HealthServer | None
-    pid_lock_fd: int | None
-    watchdog: PgOutageWatchdog | None
-    scheduler: NightlyJobScheduler | None
-
-    def __init__(self) -> None:
-        self.dispatcher = None
-        self.bot = None
-        self.pg = None
-        self.orchestrator = None
-        self.lifecycle = None
-        self.session_dispatcher = None
-        self.shim_server = None
-        self.health_server = None
-        self.pid_lock_fd = None
-        self.watchdog = None
-        self.scheduler = None
-
-    async def shutdown(self) -> None:
-        log.info("daemon.shutdown.begin")
-        if self.watchdog is not None:
-            await self.watchdog.stop()
-        # Per FR-9.7: stop_polling + drain pending throttler edits before exit.
-        dispatcher = self.dispatcher
-        if dispatcher is not None:
-            await dispatcher.stop_polling()
-        if self.session_dispatcher is not None:
-            await self.session_dispatcher.detach_all()
-        if self.shim_server is not None:
-            await self.shim_server.stop()
-        if self.health_server is not None:
-            await self.health_server.stop()
-        if self.scheduler is not None:
-            await self.scheduler.stop()
-        if self.lifecycle is not None:
-            await self.lifecycle.stop()
-        if self.orchestrator is not None:
-            await self.orchestrator.shutdown_all()
-        bot = self.bot
-        if bot is not None:
-            await bot.session.close()
-        pg = self.pg
-        if pg is not None:
-            await pg.close()
-        if self.pid_lock_fd is not None:
-            import os
-
-            os.close(self.pid_lock_fd)
-        log.info("daemon.shutdown.done")
-
-
-def _acquire_pid_lock(runner_dir: Path) -> int:
-    """flock-style exclusive lock on /run/vds-telegram/daemon.pid (FR-8.3).
-
-    Boot fails fast if another daemon process already holds the lock — enforces
-    the "one bot = one container, ever" invariant from `reference_compose_project_identity_anchor.md`.
-    """
-    import fcntl
-    import os
-
-    runner_dir.mkdir(parents=True, exist_ok=True)
-    pid_path = runner_dir / "daemon.pid"
-    fd = os.open(pid_path, os.O_CREAT | os.O_RDWR, 0o600)
-    try:
-        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
-    except BlockingIOError as e:
-        os.close(fd)
-        raise SystemExit(
-            f"daemon PID lock at {pid_path} is held by another process. "
-            "Refusing to start — only one daemon per Compose project is allowed."
-        ) from e
-    os.write(fd, f"{os.getpid()}\n".encode())
-    os.fsync(fd)
-    return fd
-
-
-async def _run() -> None:
-    settings = load_settings()
-    log.info("daemon.boot", supergroup_id=settings.VDS_TELEGRAM_SUPERGROUP_ID)
-
-    state = _DaemonState()
-    state.pid_lock_fd = _acquire_pid_lock(settings.VDS_TELEGRAM_RUNNER_DIR)
-
-    pg = PgPool(dsn=settings.VDS_TELEGRAM_DB_DSN)
-    await pg.open()
-    try:
-        await assert_migrations_at_head(pg)
-    except MigrationHeadMismatch as exc:
-        raise SystemExit(str(exc)) from exc
-    if not await pg.health_check():
-        raise SystemExit("Postgres unreachable at boot; refusing to start")
-    state.pg = pg
-
-    # Compose the daemon graph: audit + bus + registry + orchestrator + lifecycle.
-    audit_interceptor: BlobInterceptor
-    if settings.VDS_TELEGRAM_AUDIT_REDACT_ENABLED:
-        from telegram_bridge.runners._base.secret_patterns import (
-            DEFAULT_PATTERNS,
-            OPTIONAL_PATTERNS,
-            parse_extra_patterns,
-        )
-
-        active_patterns: tuple = DEFAULT_PATTERNS
-        if settings.VDS_TELEGRAM_AUDIT_REDACT_AWS_SECRET:
-            active_patterns = (*active_patterns, OPTIONAL_PATTERNS["AWS_SECRET_KEY"])
-        extra = parse_extra_patterns(settings.VDS_TELEGRAM_AUDIT_REDACT_PATTERNS)
-        audit_interceptor = PatternRedactionInterceptor(patterns=active_patterns + extra)
-        log.info(
-            "audit.redaction.enabled",
-            default_patterns=len(DEFAULT_PATTERNS),
-            optional_patterns=len(active_patterns) - len(DEFAULT_PATTERNS),
-            extra_patterns=len(extra),
-        )
-    else:
-        audit_interceptor = PassthroughInterceptor()
-        log.warning("audit.redaction.disabled")
-    audit = AuditWriter(pg=pg, interceptor=audit_interceptor)
-    bus = SessionBus()
-    registry = SessionRegistry(bus=bus, audit=audit)
-    orchestrator = RunnerOrchestrator(
-        registry=registry,
-        audit=audit,
-        bus=bus,
-        runner_dir=settings.VDS_TELEGRAM_RUNNER_DIR,
-        db_dsn=settings.VDS_TELEGRAM_DB_DSN,
-    )
-    state.orchestrator = orchestrator
-
-    # TSK-7.7: PG outage watchdog — 6-line wiring block.
-    async def _pg_alarm(text: str) -> None:
-        if state.bot and settings.VDS_TELEGRAM_SUPERGROUP_ID:
-            await state.bot.send_message(settings.VDS_TELEGRAM_SUPERGROUP_ID, text, message_thread_id=1)
-        else:
-            log.warning(
-                "pg_outage.alarm_dropped_bot_not_ready",
-                reason="alarm fired during boot before state.bot was constructed",
-                supergroup_id=settings.VDS_TELEGRAM_SUPERGROUP_ID,
-            )
-
-    watchdog = PgOutageWatchdog(registry=registry, orchestrator=orchestrator, alarm_callback=_pg_alarm)
-    state.watchdog = watchdog
-    audit.set_watchdog(watchdog)
-    await watchdog.start()
-
-    lifecycle = LifecycleTicker(registry=registry, audit=audit)
-    await lifecycle.start()
-    state.lifecycle = lifecycle
-
-    # Load agent registry; refuses to boot on validation errors.
-    agents = load_registry(settings.VDS_TELEGRAM_AGENTS_REGISTRY)
-    log.info("daemon.agents_loaded", count=len(agents))
-
-    state.bot = build_bot(settings)
-    state.dispatcher = build_dispatcher(settings, pg=pg)
-
-    # Build aiogram-backed gateway implementations for promoter + throttler.
-    topic_gateway = AiogramTopicGateway(state.bot)
-    streaming_gateway = AiogramStreamingGateway(
-        state.bot,
-        supergroup_id=settings.VDS_TELEGRAM_SUPERGROUP_ID or 0,
-    )
-
-    promoter = TopicPromoter(
-        bus=bus,
-        gateway=topic_gateway,
-        supergroup_id=settings.VDS_TELEGRAM_SUPERGROUP_ID or 0,
-        subscriptions=set(),  # populated from vds_telegram_topic_subscriptions on attach
-        promotion_threshold_s=float(settings.VDS_TELEGRAM_TOPIC_PROMOTION_THRESHOLD_S),
-    )
-    throttler = StreamingThrottler(
-        gateway=streaming_gateway,
-        bucket=ChatTokenBucket(),
-    )
-    session_dispatcher = SessionDispatcher(bus=bus, throttler=throttler, promoter=promoter)
-    state.session_dispatcher = session_dispatcher
-
-    # Wire commands router + uploads router (commands first; uploads catches
-    # Document/Photo/Audio/Video that fall through commands).
-    state.dispatcher.include_router(
-        build_router(
-            settings=settings,
-            pg=pg,
-            registry=registry,
-            orchestrator=orchestrator,
-            agents=agents,
-        )
-    )
-    state.dispatcher.include_router(build_uploads_router(pg=pg, orchestrator=orchestrator))
-
-    # Bring up the host-shim server (TSK-9.2-9.4).
-    shim = ShimServer(
-        socket_path=settings.VDS_TELEGRAM_RUNNER_DIR / "shim.sock",
-        settings=settings,
-        pg=pg,
-        registry=registry,
-        orchestrator=orchestrator,
-        bus=bus,
-        agents=agents,
-    )
-    await shim.start()
-    state.shim_server = shim
-
-    # Bring up the health UDS endpoint (TSK-7.8).
-    health = HealthServer(
-        socket_path=settings.VDS_TELEGRAM_RUNNER_DIR / "health.sock",
-        pg=pg,
-        registry=registry,
-        throttler=throttler,
-        supergroup_id=settings.VDS_TELEGRAM_SUPERGROUP_ID,
-    )
-    await health.start()
-    state.health_server = health
-
-    # getChatMember preflight if supergroup already registered.
-    if settings.VDS_TELEGRAM_SUPERGROUP_ID is not None:
-        try:
-            await check_bot_admin_permissions(state.bot, settings.VDS_TELEGRAM_SUPERGROUP_ID)
-        except PreflightError as e:
-            raise SystemExit(str(e)) from e
-
-    # TSK-7.4 / TSK-7.5 / FR-2.14: nightly archive + rotate + soft-cap scheduler.
-    scheduler = NightlyJobScheduler(
-        pg=pg,
-        get_bot=lambda: state.bot,
-        parquet_root=settings.VDS_TELEGRAM_PARQUET_ROOT,
-        soft_cap=settings.VDS_TELEGRAM_ACTIVE_TOPIC_SOFT_CAP,
-        hour_utc=settings.VDS_TELEGRAM_NIGHTLY_HOUR_UTC,
-    )
-    await scheduler.start()
-    state.scheduler = scheduler
-
-    # Signal handlers for clean drain.
-    loop = asyncio.get_running_loop()
-    stop_event = asyncio.Event()
-    for sig in (signal.SIGTERM, signal.SIGINT):
-        loop.add_signal_handler(sig, stop_event.set)
-
-    if state.dispatcher is None or state.bot is None:
-        raise RuntimeError("daemon graph incomplete: dispatcher or bot is None at polling start")
-
-    # FR-9.10: resume from persisted last_update_offset. aiogram 3.27's
-    # start_polling has no offset kwarg — instead we issue a single
-    # getUpdates(offset=N+1) at boot to acknowledge prior updates
-    # server-side. Telegram then drops them from the polling stream so the
-    # subsequent dispatch loop only sees fresh updates.
-    last_offset = await load_last_offset(pg)
-    if last_offset is not None and last_offset > 0:
-        try:
-            await state.bot.get_updates(offset=last_offset + 1, limit=1, timeout=0)
-            log.info("daemon.poll.resume", offset=last_offset)
-        except Exception:
-            log.exception("daemon.poll.resume.error", offset=last_offset)
-    poll_task = asyncio.create_task(state.dispatcher.start_polling(state.bot))
-    stop_task = asyncio.create_task(stop_event.wait())
-
-    _done, pending = await asyncio.wait(
-        {poll_task, stop_task},
-        return_when=asyncio.FIRST_COMPLETED,
-    )
-
-    for t in pending:
-        t.cancel()
-    await state.shutdown()
-
-
-def main() -> None:
-    """Synchronous entrypoint — wraps async run loop."""
-    structlog.configure(
-        processors=[
-            structlog.processors.TimeStamper(fmt="iso"),
-            structlog.processors.add_log_level,
-            structlog.processors.JSONRenderer(),
-        ],
-        wrapper_class=structlog.make_filtering_bound_logger(20),  # INFO
-    )
-    asyncio.run(_run())
-
-
-if __name__ == "__main__":
-    main()

package/tools/vds-scripts/telegram_bridge/daemon/alembic_head_check.py

@@ -1,116 +0,0 @@
-"""Daemon-boot Alembic head check (TSK-7 / Task #7).
-
-The daemon never auto-applies migrations — that is the job of the one-shot
-``alembic upgrade head`` step in the Compose ``depends_on:`` chain. The daemon
-just verifies the DB is at the expected head and refuses to start if it is
-not. This enforces the invariant:
-
-  "Migration lag is operator error, not a daemon concern."
-
-Exported surface:
-  assert_migrations_at_head(pg)  — raises MigrationHeadMismatch if not at head.
-  MigrationHeadMismatch          — subclass of RuntimeError; carries pending/current.
-
-Note: module is named ``alembic_head_check`` (not ``migration_check``) to avoid
-the repo-wide .gitignore pattern ``**/MIGRATION_*`` which matches case-
-insensitively on macOS APFS.
-"""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import TYPE_CHECKING, AbstractSet
-
-import asyncpg
-import structlog
-
-if TYPE_CHECKING:
-    from telegram_bridge.daemon.pg import PgPool
-
-log = structlog.get_logger(__name__)
-
-# alembic.ini lives at telegram_bridge/alembic.ini — one level above this module.
-_DEFAULT_CFG_PATH: Path = Path(__file__).parent.parent / "alembic.ini"
-
-
-class MigrationHeadMismatch(RuntimeError):
-    """Raised when the DB's alembic_version does not match the script-dir heads.
-
-    Attributes:
-        installed:  revision IDs currently in alembic_version (may be empty).
-        expected:   head revision IDs from the script directory.
-    """
-
-    def __init__(self, installed: AbstractSet[str], expected: AbstractSet[str]) -> None:
-        self.installed = frozenset(installed)
-        self.expected = frozenset(expected)
-        pending = expected - installed
-        stale = installed - expected
-        parts: list[str] = []
-        if pending:
-            parts.append(f"pending={sorted(pending)}")
-        if stale:
-            parts.append(f"stale={sorted(stale)}")
-        detail = "; ".join(parts) if parts else "unknown drift"
-        super().__init__(
-            f"Alembic head mismatch — {detail}. "
-            "Run `alembic upgrade head` in the telegram_bridge package before "
-            "starting the daemon."
-        )
-
-
-def _load_expected_heads(cfg_path: Path = _DEFAULT_CFG_PATH) -> frozenset[str]:
-    """Read the alembic script directory and return the current head revision IDs.
-
-    Importing alembic here (not at module top level) keeps the import cost in
-    the normal code path zero — this function is called once at daemon boot.
-    """
-    from alembic.config import Config
-    from alembic.script import ScriptDirectory
-
-    cfg = Config(str(cfg_path))
-    # Resolve script_location relative to the cfg file's directory so it works
-    # regardless of the current working directory when the daemon boots.
-    cfg.set_main_option("script_location", str(cfg_path.parent / "alembic"))
-    script = ScriptDirectory.from_config(cfg)
-    return frozenset(script.get_heads())
-
-
-async def _query_installed_heads(pool: asyncpg.Pool) -> frozenset[str]:
-    """SELECT version_num FROM alembic_version — empty set if table missing."""
-    try:
-        async with pool.acquire() as conn:
-            rows = await conn.fetch("SELECT version_num FROM alembic_version")
-        return frozenset(row["version_num"] for row in rows)
-    except asyncpg.UndefinedTableError:
-        # Table doesn't exist yet — migrations have never been applied.
-        return frozenset()
-
-
-async def assert_migrations_at_head(pg: PgPool) -> None:
-    """Assert the DB is at the Alembic head for telegram_bridge migrations.
-
-    Reads the expected head(s) from the script directory located at
-    ``telegram_bridge/alembic/`` (hardcoded relative to this module's path —
-    the daemon module path is stable).
-
-    Raises:
-        MigrationHeadMismatch: if the installed revision set != expected heads.
-
-    Call after ``await pg.open()`` and before entering the main daemon loop.
-    On mismatch the caller should raise SystemExit so the operator runs
-    ``alembic upgrade head`` before restarting.
-    """
-    expected = _load_expected_heads()
-    installed = await _query_installed_heads(pg.pool)
-
-    log.info(
-        "migration_check",
-        expected=sorted(expected),
-        installed=sorted(installed),
-    )
-
-    if installed != expected:
-        raise MigrationHeadMismatch(installed=installed, expected=expected)
-
-    log.info("migration_check.ok", head=sorted(expected))