@foggy-projects/deepseek-harness-plugin 0.4.0-beta.8 → 0.4.0-rc.1

package/skills/foggy-deepseek-onboarding/scripts/onboarding.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 
 import argparse
 import datetime as dt
+import errno
 import hashlib
 import json
 import os
@@ -17,6 +18,7 @@ import subprocess
 import sys
 import tempfile
 import time
+import urllib.error
 import urllib.request
 import venv
 import zipfile
@@ -24,6 +26,8 @@ import zipfile
 
 STATE_SCHEMA = "foggy-deepseek-onboarding-install/v1"
 RUNTIME_STATE_SCHEMA = "foggy-deepseek-onboarding-runtime/v1"
+RUNTIME_SETTINGS_SCHEMA = "foggy-deepseek-runtime-settings/v1"
+RUNTIME_PORT_CONFLICT_SCHEMA = "foggy-deepseek-runtime-port-conflict/v1"
 ONBOARDING_STATE_SCHEMA = "foggy-deepseek-onboarding-state/v1"
 MANAGED_SKILL_SCHEMA = "foggy-managed-skill/v1"
 MANAGED_SKILL_MARKER = ".foggy-managed-skill.json"
@@ -42,25 +46,48 @@ class OnboardingError(RuntimeError):
     pass
 
 
+class RuntimeSettingsError(OnboardingError):
+    code = "RUNTIME_SETTINGS_INVALID"
+
+
+class RuntimePortUnavailableError(OnboardingError):
+    code = "RUNTIME_PORT_UNAVAILABLE"
+
+
 PROGRESS_SCHEMA = "foggy-deepseek-onboarding-progress/v1"
 ACTIVE_PROGRESS: "ProgressReporter | None" = None
 
 
 class ProgressReporter:
-    total_steps = 7
-
-    def __init__(self, path: str | None, operation_id: str | None, kind: str) -> None:
+    def __init__(
+        self,
+        path: str | None,
+        operation_id: str | None,
+        kind: str,
+        *,
+        total_steps: int = 7,
+        completion_phase: str = "complete",
+        completion_message: str = "Foggy initialization completed",
+        failure_message: str = "Foggy initialization failed",
+    ) -> None:
         self.path = Path(path).expanduser() if path else None
         self.operation_id = operation_id
         self.kind = kind
+        self.total_steps = total_steps
+        self.completion_phase = completion_phase
+        self.completion_message = completion_message
+        self.failure_message = failure_message
         self.started_at = now_utc()
         self.phase = "preflight"
         self.step_index = 0
         self.fraction = 0.0
+        self.percent_override: int | None = None
         self.message = ""
         self.current_file: str | None = None
         self.completed_files: int | None = None
         self.total_files: int | None = None
+        self.elapsed_seconds: int | None = None
+        self.timeout_seconds: int | None = None
 
     @property
     def enabled(self) -> bool:
@@ -71,7 +98,11 @@ class ProgressReporter:
             return
         bounded_step = max(0, min(self.step_index, self.total_steps))
         bounded_fraction = max(0.0, min(float(self.fraction), 1.0))
-        percent = 100 if state == "succeeded" else round(((bounded_step + bounded_fraction) / self.total_steps) * 100)
+        percent = 100 if state == "succeeded" else (
+            max(0, min(self.percent_override, 99))
+            if self.percent_override is not None
+            else round(((bounded_step + bounded_fraction) / self.total_steps) * 100)
+        )
         payload = {
             "schemaVersion": PROGRESS_SCHEMA,
             "operationId": self.operation_id,
@@ -87,6 +118,10 @@ class ProgressReporter:
         }
         if self.completed_files is not None and self.total_files is not None:
             payload["files"] = {"completed": self.completed_files, "total": self.total_files}
+        if self.elapsed_seconds is not None:
+            payload["timing"] = {"elapsedSeconds": self.elapsed_seconds}
+            if self.timeout_seconds is not None:
+                payload["timing"]["timeoutSeconds"] = self.timeout_seconds
         if finished_at:
             payload["finishedAt"] = finished_at
         if error:
@@ -104,28 +139,37 @@ class ProgressReporter:
         current_file: str | None = None,
         completed_files: int | None = None,
         total_files: int | None = None,
+        percent: int | None = None,
+        elapsed_seconds: int | None = None,
+        timeout_seconds: int | None = None,
     ) -> None:
         self.phase = phase
         self.step_index = step_index
         self.fraction = fraction
+        self.percent_override = percent
         self.message = message
         self.current_file = current_file
         self.completed_files = completed_files
         self.total_files = total_files
+        self.elapsed_seconds = elapsed_seconds
+        self.timeout_seconds = timeout_seconds
         self._write("running")
 
     def finish(self) -> None:
-        self.phase = "complete"
+        self.phase = self.completion_phase
         self.step_index = self.total_steps
         self.fraction = 0.0
-        self.message = "Foggy initialization completed"
+        self.percent_override = 100
+        self.message = self.completion_message
         self.current_file = None
         self.completed_files = None
         self.total_files = None
+        self.elapsed_seconds = None
+        self.timeout_seconds = None
         self._write("succeeded", finished_at=now_utc())
 
     def fail(self, error: object) -> None:
-        self.message = "Foggy initialization failed"
+        self.message = self.failure_message
         self._write("failed", finished_at=now_utc(), error=str(error))
 
 
@@ -173,6 +217,61 @@ def default_data_root() -> Path:
     return Path(base) / "foggy" / "deepseek-harness" if base else Path.home() / ".local" / "state" / "foggy" / "deepseek-harness"
 
 
+def validate_runtime_port(value: object) -> int:
+    if isinstance(value, bool):
+        raise RuntimeSettingsError("Runtime port must be an integer between 1024 and 65535")
+    try:
+        port = int(value)
+    except (TypeError, ValueError) as exc:
+        raise RuntimeSettingsError("Runtime port must be an integer between 1024 and 65535") from exc
+    if str(value).strip() != str(port) or port < 1024 or port > 65535:
+        raise RuntimeSettingsError("Runtime port must be an integer between 1024 and 65535")
+    return port
+
+
+def read_runtime_settings(data_root: Path, default_port: object) -> dict:
+    path = data_root / "runtime-settings.json"
+    if not path.is_file():
+        port = validate_runtime_port(default_port)
+        return {"schemaVersion": RUNTIME_SETTINGS_SCHEMA, "port": port, "source": "default", "path": str(path)}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise RuntimeSettingsError(f"Runtime settings file is invalid: {path}") from exc
+    if payload.get("schemaVersion") != RUNTIME_SETTINGS_SCHEMA:
+        raise RuntimeSettingsError(f"Unexpected Runtime settings schema in {path}")
+    port = validate_runtime_port(payload.get("runtimePort"))
+    return {"schemaVersion": RUNTIME_SETTINGS_SCHEMA, "port": port, "source": "configured", "path": str(path)}
+
+
+def assert_runtime_port_available(data_root: Path, port: int) -> None:
+    conflict_path = data_root / "last-runtime-port-conflict.json"
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        if os.name == "nt" and hasattr(socket, "SO_EXCLUSIVEADDRUSE"):
+            probe.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1)
+        try:
+            # The packaged Java server uses a wildcard listener. Checking only
+            # 127.0.0.1 misses Windows portproxy entries bound to another local address.
+            probe.bind(("0.0.0.0", port))
+        except OSError as exc:
+            message = (
+                f"Runtime port {port} is unavailable. Another application or Windows port proxy "
+                "is already listening. Choose a different port in Foggy plugin settings or release "
+                "the port before retrying."
+            )
+            conflict = {
+                "schemaVersion": RUNTIME_PORT_CONFLICT_SCHEMA,
+                "detectedAt": now_utc(),
+                "port": port,
+                "bindAddress": "0.0.0.0",
+                "runtimeUrl": f"http://127.0.0.1:{port}",
+                "message": message,
+            }
+            atomic_json(conflict_path, conflict)
+            raise RuntimePortUnavailableError(message) from exc
+    conflict_path.unlink(missing_ok=True)
+
+
 def normalized(path: str | Path) -> Path:
     return Path(path).expanduser().resolve(strict=False)
 
@@ -244,6 +343,128 @@ def command_result(command: list[str], timeout: int = 30, check: bool = False) -
     return payload
 
 
+def process_running(pid: int) -> bool:
+    """Check liveness without spawning a shell or waiting on the target process."""
+    if pid <= 0:
+        return False
+    if os.name == "nt":
+        import ctypes
+
+        synchronize = 0x00100000
+        wait_timeout = 0x00000102
+        kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+        kernel32.OpenProcess.argtypes = [ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32]
+        kernel32.OpenProcess.restype = ctypes.c_void_p
+        kernel32.WaitForSingleObject.argtypes = [ctypes.c_void_p, ctypes.c_uint32]
+        kernel32.WaitForSingleObject.restype = ctypes.c_uint32
+        kernel32.CloseHandle.argtypes = [ctypes.c_void_p]
+        handle = kernel32.OpenProcess(synchronize, False, pid)
+        if not handle:
+            return ctypes.get_last_error() == 5  # Access denied still proves the PID exists.
+        try:
+            return kernel32.WaitForSingleObject(handle, 0) == wait_timeout
+        finally:
+            kernel32.CloseHandle(handle)
+    proc_stat = Path("/proc") / str(pid) / "stat"
+    if proc_stat.is_file():
+        try:
+            fields = proc_stat.read_text(encoding="utf-8", errors="replace").rsplit(") ", 1)[1]
+            return bool(fields) and fields[0] != "Z"
+        except (OSError, IndexError):
+            pass
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError:
+        return False
+    return True
+
+
+def command_result_with_progress(
+    command: list[str],
+    timeout: int,
+    progress: ProgressReporter,
+    *,
+    phase: str,
+    step_index: int,
+    message: str,
+    readiness_timeout: int,
+    start_percent: int,
+    end_percent: int,
+    poll_interval: float = 1.0,
+    watched_pid: int | None = None,
+    watched_label: str = "Runtime",
+) -> dict:
+    started = time.monotonic()
+    try:
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+    except FileNotFoundError:
+        return {
+            "command": command[0],
+            "available": False,
+            "exitCode": None,
+            "stdout": "",
+            "stderr": "command not found",
+            "durationMs": round((time.monotonic() - started) * 1000),
+        }
+
+    stdout = ""
+    stderr = ""
+    while True:
+        try:
+            stdout, stderr = process.communicate(timeout=poll_interval)
+            break
+        except subprocess.TimeoutExpired:
+            elapsed = time.monotonic() - started
+            if watched_pid and not process_running(watched_pid):
+                process.kill()
+                process.communicate()
+                raise OnboardingError(
+                    f"{watched_label} process PID {watched_pid} exited before becoming ready "
+                    f"after {round(elapsed * 1000)} ms"
+                )
+            readiness_fraction = min(elapsed / max(readiness_timeout, 1), 1.0)
+            percent = round(start_percent + (end_percent - start_percent) * readiness_fraction)
+            progress.update(
+                phase,
+                step_index,
+                message,
+                percent=percent,
+                elapsed_seconds=max(1, round(elapsed)),
+                timeout_seconds=readiness_timeout,
+            )
+            if elapsed >= timeout:
+                process.kill()
+                stdout, stderr = process.communicate()
+                return {
+                    "command": command[0],
+                    "available": True,
+                    "exitCode": None,
+                    "stdout": stdout.strip(),
+                    "stderr": f"timed out after {timeout} seconds",
+                    "durationMs": round((time.monotonic() - started) * 1000),
+                }
+
+    return {
+        "command": command[0],
+        "available": True,
+        "exitCode": process.returncode,
+        "stdout": stdout.strip(),
+        "stderr": stderr.strip(),
+        "durationMs": round((time.monotonic() - started) * 1000),
+    }
+
+
 def sha256(path: Path) -> str:
     digest = hashlib.sha256()
     with path.open("rb") as stream:
@@ -337,11 +558,39 @@ def venv_cli(install_root: Path) -> Path:
     return install_root / "venv" / ("Scripts/foggy-runtime.exe" if os.name == "nt" else "bin/foggy-runtime")
 
 
+def replace_with_retry(source: Path, destination: Path, attempts: int = 12) -> None:
+    transient_errno = {errno.EACCES, errno.EBUSY, errno.EPERM}
+    transient_winerror = {5, 32, 33}
+    for attempt in range(attempts):
+        try:
+            os.replace(source, destination)
+            return
+        except OSError as exc:
+            transient = exc.errno in transient_errno or getattr(exc, "winerror", None) in transient_winerror
+            if not transient or attempt == attempts - 1:
+                raise
+            time.sleep(min(0.025 * (2 ** attempt), 0.4))
+
+
 def atomic_json(path: Path, payload: dict) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
-    temporary = path.with_name(path.name + ".tmp")
-    temporary.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
-    os.replace(temporary, path)
+    handle = tempfile.NamedTemporaryFile(
+        mode="w",
+        encoding="utf-8",
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+        delete=False,
+    )
+    temporary = Path(handle.name)
+    try:
+        with handle:
+            json.dump(payload, handle, indent=2, ensure_ascii=False)
+            handle.flush()
+            os.fsync(handle.fileno())
+        replace_with_retry(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
 
 
 def skill_tree_digest(root: Path) -> str:
@@ -670,7 +919,7 @@ def validate_connection(payload: dict) -> dict:
     if payload.get("schemaVersion") != CONNECTION_SCHEMA:
         raise OnboardingError(f"connection schemaVersion must be {CONNECTION_SCHEMA}")
     allowed = {
-        "schemaVersion", "name", "type", "jdbcUrl", "username", "passwordEnv",
+        "schemaVersion", "name", "type", "jdbcUrl", "username", "password", "passwordEnv",
         "opaqueProfileId", "opaqueRevision",
         "profile", "namespace", "schemas", "modelsDir", "evidenceDir", "readOnlyRecommended",
     }
@@ -682,6 +931,7 @@ def validate_connection(payload: dict) -> dict:
     for name in required:
         if not isinstance(payload.get(name), str) or not payload[name].strip():
             raise OnboardingError(f"connection.{name} must be a non-empty string")
+    password = payload.get("password")
     password_env = payload.get("passwordEnv")
     jdbc_url = None
     if opaque:
@@ -689,15 +939,19 @@ def validate_connection(payload: dict) -> dict:
             raise OnboardingError("connection.opaqueProfileId must be an opaque Foggy profile ID")
         if not isinstance(payload.get("opaqueRevision"), str) or not OPAQUE_REVISION_PATTERN.fullmatch(payload["opaqueRevision"]):
             raise OnboardingError("connection.opaqueRevision must be a sha256 revision")
-        exposed = sorted(name for name in ("jdbcUrl", "username", "passwordEnv") if name in payload)
+        exposed = sorted(name for name in ("jdbcUrl", "username", "password", "passwordEnv") if name in payload)
         if exposed:
             raise OnboardingError(f"Opaque connection plans must not contain: {', '.join(exposed)}")
     else:
+        if password is not None and not isinstance(password, str):
+            raise OnboardingError("connection.password must be a string")
         if password_env is not None and (not isinstance(password_env, str) or not ENV_NAME_PATTERN.fullmatch(password_env)):
             raise OnboardingError("connection.passwordEnv must be an environment variable name")
+        if password is not None and password_env is not None:
+            raise OnboardingError("Provide either connection.password or connection.passwordEnv, not both")
         jdbc_url = payload["jdbcUrl"].strip()
         if re.search(r"(?i)(?:password|passwd|pwd)\s*=", jdbc_url) or re.search(r"//[^/@:]+:[^/@]+@", jdbc_url):
-            raise OnboardingError("Do not embed passwords in jdbcUrl; use passwordEnv")
+            raise OnboardingError("Do not embed passwords in jdbcUrl; use password or passwordEnv")
     schemas = payload.get("schemas", [])
     if not isinstance(schemas, list) or any(not isinstance(item, str) or not item.strip() for item in schemas):
         raise OnboardingError("connection.schemas must be an array of non-empty strings")
@@ -725,6 +979,7 @@ def validate_connection(payload: dict) -> dict:
     else:
         result["jdbcUrl"] = jdbc_url
         result["username"] = payload.get("username")
+        result["password"] = password
         result["passwordEnv"] = password_env
     if payload.get("profile") is not None:
         result["profile"] = safe_profile(payload["profile"])
@@ -732,11 +987,24 @@ def validate_connection(payload: dict) -> dict:
         result["evidenceDir"] = payload["evidenceDir"].strip()
     if result["type"] not in {"sqlite", "mysql", "postgres", "postgresql"}:
         raise OnboardingError("Initial onboarding supports sqlite, mysql, postgres, and postgresql")
-    if not opaque and result["type"] != "sqlite" and not result["passwordEnv"]:
-        raise OnboardingError("Non-SQLite connections require passwordEnv")
     return result
 
 
+def persisted_connection(connection: dict) -> dict:
+    """Return the resumable connection contract without an inline development password."""
+    persisted = {key: value for key, value in connection.items() if key != "password"}
+    credential_mode = connection.get("credentialMode")
+    if credential_mode not in {"inline-development", "agent-environment", "opaque-profile", "none"}:
+        credential_mode = (
+            "inline-development" if connection.get("password") is not None
+            else "agent-environment" if connection.get("passwordEnv")
+            else "opaque-profile" if connection.get("connectionMode") == "opaque-profile"
+            else "none"
+        )
+    persisted["credentialMode"] = credential_mode
+    return persisted
+
+
 def validate_semantic_plan(payload: dict) -> dict:
     if payload.get("schemaVersion") != SEMANTIC_PLAN_SCHEMA:
         raise OnboardingError(f"semantic plan schemaVersion must be {SEMANTIC_PLAN_SCHEMA}")
@@ -1247,8 +1515,57 @@ def parse_json_output(result: dict, label: str) -> dict:
         raise OnboardingError(f"{label} did not return JSON") from exc
 
 
+def write_runtime_start_failure(
+    data_root: Path,
+    evidence_dir: Path,
+    launch: dict,
+    phase: str,
+    error: object,
+    elapsed_ms: int,
+) -> dict:
+    logs = []
+    for name in ("stdoutLog", "stderrLog"):
+        value = launch.get(name)
+        path = normalized(value) if isinstance(value, str) and value else None
+        item = {"name": name, "path": str(path) if path else None, "exists": bool(path and path.is_file())}
+        if path and path.is_file():
+            stat = path.stat()
+            item.update({"sizeBytes": stat.st_size, "modifiedAt": dt.datetime.fromtimestamp(stat.st_mtime, dt.timezone.utc).isoformat()})
+        logs.append(item)
+    pid = int(launch.get("pid", 0))
+    failure = {
+        "schemaVersion": "foggy-deepseek-runtime-start-failure/v1",
+        "failedAt": now_utc(),
+        "phase": phase,
+        "message": str(error),
+        "elapsedMs": elapsed_ms,
+        "pid": pid or None,
+        "processRunningBeforeCleanup": process_info(pid)["running"] if pid else False,
+        "runtimeUrl": launch.get("runtimeUrl"),
+        "workDir": launch.get("workDir"),
+        "evidenceDir": str(evidence_dir),
+        "logs": logs,
+    }
+    atomic_json(evidence_dir / "failure.json", failure)
+    atomic_json(data_root / "last-runtime-start-failure.json", failure)
+    return failure
+
+
 def runtime_start_command(args: argparse.Namespace) -> dict:
+    global ACTIVE_PROGRESS
     versions = load_versions()
+    readiness_timeout = args.timeout or int(versions["defaults"]["readinessTimeoutSeconds"])
+    progress = ProgressReporter(
+        getattr(args, "progress_file", None),
+        getattr(args, "operation_id", None),
+        getattr(args, "operation_kind", "runtime-start"),
+        total_steps=6,
+        completion_phase="runtime-complete",
+        completion_message="Foggy Runtime is ready",
+        failure_message="Foggy Runtime startup failed",
+    )
+    ACTIVE_PROGRESS = progress
+    progress.update("runtime-preflight", 0, "Checking Runtime state and prerequisites", percent=3)
     install_root = normalized(args.install_root or default_install_root())
     state = read_install_state(install_root)
     data_root = normalized(args.data_root or state["dataRoot"])
@@ -1267,15 +1584,25 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
             base_url = prior.get("runtimeUrl")
             if not isinstance(base_url, str) or not base_url:
                 raise OnboardingError("Recorded Runtime does not contain runtimeUrl")
-            wait_result = command_result(
+            progress.update("runtime-readiness", 3, "Revalidating the recorded Runtime", percent=30, elapsed_seconds=0, timeout_seconds=readiness_timeout)
+            wait_result = command_result_with_progress(
                 [cli, "--base-url", base_url, "--namespace", namespace, "--output", "json", "wait-ready",
-                 "--timeout-seconds", str(args.timeout or versions["defaults"]["readinessTimeoutSeconds"]),
+                 "--timeout-seconds", str(readiness_timeout),
                  "--interval-seconds", "1"],
-                timeout=(args.timeout or versions["defaults"]["readinessTimeoutSeconds"]) + 30,
+                readiness_timeout + 30,
+                progress,
+                phase="runtime-readiness",
+                step_index=3,
+                message="Revalidating the recorded Runtime",
+                readiness_timeout=readiness_timeout,
+                start_percent=30,
+                end_percent=85,
+                watched_pid=int(prior["pid"]),
             )
             wait_payload = parse_json_output(wait_result, "wait-ready")
             if wait_payload.get("success") is not True:
                 raise OnboardingError("wait-ready returned success=false for recorded Runtime")
+            progress.update("runtime-capabilities", 4, "Verifying Runtime API capabilities", percent=90)
             capabilities = parse_json_output(
                 command_result(
                     [cli, "--base-url", base_url, "--namespace", namespace, "--output", "json", "capabilities"],
@@ -1295,6 +1622,8 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
                 "schemaVersion": capabilities.get("data", {}).get("schemaVersion"),
                 "securityMode": capabilities.get("data", {}).get("securityMode"),
             }
+            progress.finish()
+            ACTIVE_PROGRESS = None
             return {
                 "success": True,
                 **prior,
@@ -1311,16 +1640,15 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
                 "productionReady": False,
             }
         existing.unlink()
-    port = args.port or int(versions["defaults"]["port"])
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
-        try:
-            probe.bind(("127.0.0.1", port))
-        except OSError as exc:
-            raise OnboardingError(f"Port {port} is not available") from exc
+    progress.update("runtime-preflight", 1, "Checking port and Runtime workspace", percent=10)
+    configured_runtime = read_runtime_settings(data_root, versions["defaults"]["port"])
+    port = validate_runtime_port(args.port) if args.port is not None else configured_runtime["port"]
+    assert_runtime_port_available(data_root, port)
     work_dir = data_root / "runtime"
     work_dir.mkdir(parents=True, exist_ok=True)
     launcher_dir = normalized(state["launcher"]["path"])
     java_exe = args.java or os.environ.get("JAVA_EXE", "java")
+    progress.update("runtime-launcher", 2, "Starting the Java Runtime through Launcher", percent=20)
     if os.name == "nt":
         shell = shutil.which("pwsh") or shutil.which("powershell")
         if not shell:
@@ -1368,12 +1696,26 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
     evidence_dir = data_root / "evidence" / f"runtime-start-{dt.datetime.now().strftime('%Y%m%d-%H%M%S')}"
     evidence_dir.mkdir(parents=True, exist_ok=True)
     atomic_json(evidence_dir / "launch.json", launch)
+    startup_started = time.monotonic()
     try:
-        wait_result = command_result([cli, "--base-url", base_url, "--namespace", namespace, "--output", "json", "wait-ready", "--timeout-seconds", str(args.timeout or versions["defaults"]["readinessTimeoutSeconds"]), "--interval-seconds", "1"], timeout=(args.timeout or versions["defaults"]["readinessTimeoutSeconds"]) + 30)
+        progress.update("runtime-readiness", 3, "Waiting for Java Runtime readiness", percent=30, elapsed_seconds=0, timeout_seconds=readiness_timeout)
+        wait_result = command_result_with_progress(
+            [cli, "--base-url", base_url, "--namespace", namespace, "--output", "json", "wait-ready", "--timeout-seconds", str(readiness_timeout), "--interval-seconds", "1"],
+            readiness_timeout + 30,
+            progress,
+            phase="runtime-readiness",
+            step_index=3,
+            message="Waiting for Java Runtime readiness",
+            readiness_timeout=readiness_timeout,
+            start_percent=30,
+            end_percent=85,
+            watched_pid=pid,
+        )
         wait_payload = parse_json_output(wait_result, "wait-ready")
         if wait_payload.get("success") is not True:
             raise OnboardingError("wait-ready returned success=false")
         atomic_json(evidence_dir / "wait-ready.json", wait_payload)
+        progress.update("runtime-capabilities", 4, "Verifying Runtime API capabilities", percent=90)
         capabilities_result = command_result([cli, "--base-url", base_url, "--namespace", namespace, "--output", "json", "capabilities"], timeout=30)
         capabilities = parse_json_output(capabilities_result, "capabilities")
         expected_contract = versions["components"]["launcher"]["runtimeApiContract"]
@@ -1382,12 +1724,21 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
         if capabilities.get("data", {}).get("securityMode") != versions["defaults"]["securityMode"]:
             raise OnboardingError("Packaged Launcher did not report the expected dev/test security mode")
         atomic_json(evidence_dir / "capabilities.json", capabilities)
-    except Exception:
+    except Exception as exc:
+        failure = write_runtime_start_failure(
+            data_root,
+            evidence_dir,
+            launch,
+            progress.phase,
+            exc,
+            round((time.monotonic() - startup_started) * 1000),
+        )
         try:
             os.kill(pid, signal.SIGTERM)
         except OSError:
             pass
-        raise
+        raise OnboardingError(f"{exc}; startup diagnostics: {failure['evidenceDir']}") from exc
+    progress.update("runtime-state", 5, "Saving verified Runtime state", percent=96, current_file="runtime-state.json")
     runtime_state = {
         "schemaVersion": RUNTIME_STATE_SCHEMA,
         "startedAt": now_utc(),
@@ -1406,6 +1757,9 @@ def runtime_start_command(args: argparse.Namespace) -> dict:
         },
     }
     atomic_json(data_root / "runtime-state.json", runtime_state)
+    (data_root / "last-runtime-start-failure.json").unlink(missing_ok=True)
+    progress.finish()
+    ACTIVE_PROGRESS = None
     return {"success": True, **runtime_state, "productionReady": False}
 
 
@@ -1446,6 +1800,108 @@ def cli_json(install_state: dict, runtime_state: dict, namespace: str, command:
     return payload
 
 
+TRANSIENT_DATASOURCE_TEST_MARKERS = (
+    "connection is not available",
+    "connection pool",
+    "request timed out",
+    "timeout waiting for connection",
+)
+
+
+def datasource_test_with_retry(
+    install_state: dict,
+    runtime_state: dict,
+    namespace: str,
+    datasource_name: str,
+    *,
+    attempts: int = 3,
+    initial_delay_seconds: float = 1.0,
+) -> dict:
+    """Retry only transient pool/readiness failures; authentication and configuration failures fail fast."""
+    for attempt in range(1, attempts + 1):
+        try:
+            result = cli_json(
+                install_state,
+                runtime_state,
+                namespace,
+                ["datasources", "test", datasource_name],
+                "datasources test",
+            )
+            if attempt > 1:
+                result = {
+                    **result,
+                    "onboardingRetry": {
+                        "attempts": attempt,
+                        "reason": "transient-datasource-readiness",
+                    },
+                }
+            return result
+        except OnboardingError as exc:
+            transient = any(marker in str(exc).lower() for marker in TRANSIENT_DATASOURCE_TEST_MARKERS)
+            if not transient or attempt == attempts:
+                raise
+            time.sleep(initial_delay_seconds * attempt)
+    raise AssertionError("unreachable")
+
+
+def runtime_api_json(
+    runtime_state: dict,
+    namespace: str,
+    method: str,
+    path: str,
+    body: dict | None,
+    label: str,
+    timeout: int = 60,
+) -> dict:
+    """Call the public Runtime API for development inputs not yet exposed by the pinned CLI."""
+    base_url = str(runtime_state["runtimeUrl"]).rstrip("/")
+    request_body = json.dumps(body).encode("utf-8") if body is not None else None
+    headers = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        "X-NS": namespace,
+    }
+    auth_code = os.environ.get("FOGGY_RUNTIME_API_AUTH_CODE")
+    if auth_code:
+        headers["X-Foggy-Runtime-Code"] = auth_code
+    request = urllib.request.Request(
+        f"{base_url}{path}",
+        data=request_body,
+        headers=headers,
+        method=method,
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            raw = response.read().decode("utf-8", errors="replace")
+    except urllib.error.HTTPError as exc:
+        raw = exc.read().decode("utf-8", errors="replace")
+        try:
+            failed = json.loads(raw)
+        except json.JSONDecodeError:
+            failed = None
+        error = failed.get("error") if isinstance(failed, dict) else None
+        detail = (
+            " | ".join(str(error.get(name)) for name in ("code", "phase", "message") if error.get(name))
+            if isinstance(error, dict)
+            else str(exc.reason)
+        )
+        raise OnboardingError(f"{label} failed with HTTP {exc.code}: {detail or exc.reason}") from exc
+    except urllib.error.URLError as exc:
+        raise OnboardingError(f"{label} could not reach Runtime: {exc.reason}") from exc
+    try:
+        payload = json.loads(raw)
+    except json.JSONDecodeError as exc:
+        raise OnboardingError(f"{label} did not return JSON") from exc
+    if payload.get("success") is not True:
+        error = payload.get("error")
+        if isinstance(error, dict):
+            detail = " | ".join(str(error.get(name)) for name in ("code", "phase", "message") if error.get(name))
+        else:
+            detail = str(error or "unknown Runtime error")
+        raise OnboardingError(f"{label} returned success=false: {detail}")
+    return payload
+
+
 CONNECTION_SECRET_KEYS = {
     "jdbcurl", "url", "username", "password", "passwordenv", "passwordref",
 }
@@ -1503,7 +1959,8 @@ def onboarding_plan_command(args: argparse.Namespace) -> dict:
     if not project_root.is_dir():
         raise OnboardingError(f"Project root not found: {project_root}")
     connection_file = normalized(args.connection_file)
-    connection = validate_connection(read_json_object(connection_file, "Connection plan"))
+    requested_connection = validate_connection(read_json_object(connection_file, "Connection plan"))
+    connection = persisted_connection(requested_connection)
     if connection.get("connectionMode") == "opaque-profile":
         require_opaque_profile_cli(install_state)
     existing = read_onboarding_state(data_root, profile, required=False)
@@ -1548,7 +2005,10 @@ def onboarding_plan_command(args: argparse.Namespace) -> dict:
         "schemaVersion": "foggy-deepseek-onboarding-plan-result/v1",
         "profile": profile,
         "statePath": str(path),
-        "connection": {**connection, "passwordEnvPresent": bool(password_env and os.environ.get(password_env))},
+        "connection": {
+            **connection,
+            "passwordEnvPresent": bool(password_env and os.environ.get(password_env)),
+        },
         "runtimeAvailable": runtime_state is not None,
         "next": "run datasource-configure --apply after reviewing the plan",
         "productionReady": False,
@@ -1579,10 +2039,11 @@ def datasource_configure_command(args: argparse.Namespace) -> dict:
     if opaque:
         plan.update({"profileId": connection["opaqueProfileId"], "revision": connection["opaqueRevision"]})
     else:
+        inline_password = getattr(args, "runtime_password", None)
         plan.update({
             "jdbcUrl": connection["jdbcUrl"],
             "username": connection.get("username"),
-            "passwordEnv": connection.get("passwordEnv"),
+            "credentialMode": connection.get("credentialMode", "none"),
         })
     if not args.apply:
         return {"success": True, "dryRun": True, "profile": state["profile"], "plan": plan, "next": "rerun with --apply after approval"}
@@ -1594,22 +2055,50 @@ def datasource_configure_command(args: argparse.Namespace) -> dict:
         ]
         label = "opaque profile configure"
     else:
+        inline_password = getattr(args, "runtime_password", None)
         password_env = connection.get("passwordEnv")
-        if password_env and not os.environ.get(password_env):
+        if connection.get("credentialMode") == "inline-development" and inline_password is None:
+            raise OnboardingError(
+                "Direct development password is not persisted; rerun onboard-datasource-run with the original connection file"
+            )
+        if inline_password is None and password_env and os.environ.get(password_env) is None:
             raise OnboardingError(f"Required password environment variable is not present: {password_env}")
-        command = ["datasources", "add", "--name", connection["name"], "--type", connection["type"], "--jdbc-url", connection["jdbcUrl"]]
-        if connection.get("username"):
-            command.extend(["--username", connection["username"]])
-        if password_env:
-            command.extend(["--password-env", password_env])
         label = "datasources add"
-    if args.replace:
-        command.append("--replace")
+        resolved_password = inline_password if inline_password is not None else (
+            os.environ.get(password_env) if password_env else None
+        )
+        command = None
     already_present = False
     try:
-        result = redact_connection_material(
-            cli_json(install_state, runtime_state, connection["namespace"], command, label)
-        )
+        if not opaque and resolved_password is not None:
+            body = {
+                "name": connection["name"],
+                "type": connection["type"],
+                "jdbcUrl": connection["jdbcUrl"],
+                "replace": bool(args.replace),
+                "enabled": True,
+                "password": resolved_password,
+            }
+            if connection.get("username"):
+                body["username"] = connection["username"]
+            result = redact_connection_material(runtime_api_json(
+                runtime_state,
+                connection["namespace"],
+                "POST",
+                "/api/v1/datasources",
+                body,
+                label,
+            ))
+        else:
+            if not opaque:
+                command = ["datasources", "add", "--name", connection["name"], "--type", connection["type"], "--jdbc-url", connection["jdbcUrl"]]
+                if connection.get("username"):
+                    command.extend(["--username", connection["username"]])
+            if args.replace:
+                command.append("--replace")
+            result = redact_connection_material(
+                cli_json(install_state, runtime_state, connection["namespace"], command, label)
+            )
     except OnboardingError as exc:
         if "DATASOURCE_ALREADY_EXISTS" not in str(exc):
             raise
@@ -1650,9 +2139,8 @@ def datasource_verify_command(args: argparse.Namespace) -> dict:
         raise OnboardingError("Datasource is not configured; run datasource-configure --apply first")
     connection = state["connection"]
     namespace = connection["namespace"]
-    tested = redact_connection_material(cli_json(
-        install_state, runtime_state, namespace,
-        ["datasources", "test", connection["name"]], "datasources test",
+    tested = redact_connection_material(datasource_test_with_retry(
+        install_state, runtime_state, namespace, connection["name"],
     ))
     if not args.bind:
         mark_step(state, "datasourceVerified", "waiting-for-binding", connectionTested=True)
@@ -2025,6 +2513,52 @@ def nested_data_objects(payload: dict) -> list[dict]:
     return objects
 
 
+def runtime_warnings(payload: dict, stage: str) -> list[dict]:
+    """Collect structured query-input and legacy warnings without inventing remediation."""
+    warnings: list[dict] = []
+    seen: set[str] = set()
+    for item in nested_data_objects(payload):
+        for field, kind in (("queryInputWarnings", "query-input"), ("warnings", "legacy")):
+            raw = item.get(field)
+            if raw is None:
+                continue
+            values = raw if isinstance(raw, list) else [raw]
+            for value in values:
+                warning = dict(value) if isinstance(value, dict) else {"message": str(value)}
+                warning["warningKind"] = kind
+                fingerprint = json.dumps(warning, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+                if fingerprint in seen:
+                    continue
+                seen.add(fingerprint)
+                warning["stages"] = [stage]
+                warnings.append(warning)
+    return warnings
+
+
+def merge_runtime_warnings(*collections: list[dict]) -> list[dict]:
+    """Deduplicate the same warning while retaining every stage that emitted it."""
+    merged: list[dict] = []
+    indexes: dict[str, int] = {}
+    for collection in collections:
+        for value in collection:
+            warning = dict(value)
+            stages = warning.pop("stages", [])
+            fingerprint = json.dumps(warning, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+            if fingerprint not in indexes:
+                warning["stages"] = []
+                indexes[fingerprint] = len(merged)
+                merged.append(warning)
+            target = merged[indexes[fingerprint]]["stages"]
+            for stage in stages:
+                if stage not in target:
+                    target.append(stage)
+    return merged
+
+
+def runtime_warning_codes(warnings: list[dict]) -> list[str]:
+    return sorted({str(item["code"]) for item in warnings if item.get("code")})
+
+
 def response_field_count(payload: dict) -> int | None:
     for item in nested_data_objects(payload):
         fields = item.get("fields")
@@ -2075,8 +2609,18 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
         ["query", "validate", query_model, "--payload", str(payload_path)], "query validate", timeout=90,
     )
     atomic_json(evidence_dir / "query-validate.json", validated)
+    validation_warnings = runtime_warnings(validated, "query-validate")
+    validation_warning_codes = runtime_warning_codes(validation_warnings)
     if not args.execute:
-        mark_step(state, "semanticVerified", "waiting-for-query-execution", queryModel=query_model, queryValidated=True)
+        mark_step(
+            state,
+            "semanticVerified",
+            "waiting-for-query-execution",
+            queryModel=query_model,
+            queryValidated=True,
+            warningCount=len(validation_warnings),
+            warningCodes=validation_warning_codes,
+        )
         state.setdefault("artifacts", {})["semanticVerifyEvidence"] = str(evidence_dir)
         path = write_onboarding_state(data_root, state)
         return {
@@ -2087,6 +2631,9 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
             "queryModel": query_model,
             "queryValidated": True,
             "queryExecuted": False,
+            "warningCount": len(validation_warnings),
+            "warningCodes": validation_warning_codes,
+            "warnings": validation_warnings,
             "evidenceDir": str(evidence_dir),
             "statePath": str(path),
             "next": "rerun semantic-verify with --execute after approving the bounded business-data query",
@@ -2097,6 +2644,9 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
         ["query", "execute", query_model, "--payload", str(payload_path)], "query execute", timeout=120,
     )
     atomic_json(evidence_dir / "query-execute.json", executed)
+    execution_warnings = runtime_warnings(executed, "query-execute")
+    warnings = merge_runtime_warnings(validation_warnings, execution_warnings)
+    warning_codes = runtime_warning_codes(warnings)
     row_count = response_row_count(executed)
     mark_step(
         state,
@@ -2108,6 +2658,9 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
         rowCount=row_count,
         queryPayloadDigest=sha256(payload_path),
         projectRoot=str(project_root),
+        warningCount=len(warnings),
+        warningCodes=warning_codes,
+        warningEvidence=str(evidence_dir),
     )
     workspace_verifications = state.setdefault("workspaceVerifications", [])
     workspace_verifications[:] = [
@@ -2119,6 +2672,9 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
         "queryModel": query_model,
         "queryPayloadDigest": sha256(payload_path),
         "rowCount": row_count,
+        "warningCount": len(warnings),
+        "warningCodes": warning_codes,
+        "warningEvidence": str(evidence_dir),
         "verifiedAt": now_utc(),
     })
     state.setdefault("artifacts", {})["semanticVerifyEvidence"] = str(evidence_dir)
@@ -2132,6 +2688,9 @@ def semantic_verify_command(args: argparse.Namespace) -> dict:
         "queryValidated": True,
         "queryExecuted": True,
         "rowCount": row_count,
+        "warningCount": len(warnings),
+        "warningCodes": warning_codes,
+        "warnings": warnings,
         "evidenceDir": str(evidence_dir),
         "statePath": str(path),
         "next": "onboarding complete; begin analysis with described field names",
@@ -2213,6 +2772,9 @@ def onboarding_list_command(args: argparse.Namespace) -> dict:
                 "completedSteps": completed,
                 "totalSteps": len(ordered_steps),
                 "steps": {name: steps.get(name, {"status": "pending"}) for name in ordered_steps},
+                "warningCount": steps.get("semanticVerified", {}).get("warningCount", 0),
+                "warningCodes": steps.get("semanticVerified", {}).get("warningCodes", []),
+                "warningEvidence": steps.get("semanticVerified", {}).get("warningEvidence"),
                 "next": next_onboarding_action(state),
             })
         except OnboardingError as exc:
@@ -2272,6 +2834,38 @@ def step_completed(state: dict, name: str) -> bool:
     return state.get("steps", {}).get(name, {}).get("status") == "completed"
 
 
+def reconcile_datasource_checkpoint(
+    state: dict,
+    install_state: dict,
+    data_root: Path,
+    runtime_state: dict,
+) -> dict | None:
+    """Reset dependent checkpoints when the live Runtime lost the datasource."""
+    if not step_completed(state, "datasourceConfigured"):
+        return None
+    connection = state["connection"]
+    listed = cli_json(
+        install_state,
+        runtime_state,
+        connection["namespace"],
+        ["datasources", "list"],
+        "datasources list for checkpoint reconciliation",
+    )
+    if matching_datasource(listed, connection) is not None:
+        return None
+    reset_steps = ("datasourceConfigured", "datasourceVerified", "schemaDiscovered")
+    for name in reset_steps:
+        mark_step(state, name, "pending", reason="runtime-datasource-missing")
+    state.get("artifacts", {}).pop("schemaDiscovery", None)
+    write_onboarding_state(data_root, state)
+    return {
+        "recovered": True,
+        "reason": "runtime-datasource-missing",
+        "resetSteps": list(reset_steps),
+        "dataSource": connection["name"],
+    }
+
+
 def bound_project_roots(state: dict) -> list[Path]:
     values = [state.get("projectRoot"), *state.get("workspaceBindings", [])]
     result: list[Path] = []
@@ -2304,9 +2898,10 @@ def bind_completed_workspace(state: dict, data_root: Path, project_root: Path) -
 
 
 def datasource_run_command(args: argparse.Namespace) -> dict:
-    _install_root, install_state, data_root, _runtime_state = onboarding_context(args, require_runtime=True)
+    _install_root, install_state, data_root, runtime_state = onboarding_context(args, require_runtime=True)
     project_root = normalized(args.project_root or Path.cwd())
     requested_connection = validate_connection(read_json_object(normalized(args.connection_file), "Connection plan"))
+    resumable_connection = persisted_connection(requested_connection)
     if not requested_connection.get("profile"):
         raise OnboardingError("Composite datasource onboarding requires connection.profile in the approved contract")
     profile = requested_connection["profile"]
@@ -2320,7 +2915,7 @@ def datasource_run_command(args: argparse.Namespace) -> dict:
     files: list[str] = []
     existing = read_onboarding_state(data_root, profile, required=False)
     if existing:
-        if existing.get("connection") != requested_connection:
+        if persisted_connection(existing.get("connection", {})) != resumable_connection:
             raise OnboardingError("Existing onboarding profile does not match the requested connection plan")
         adopted = bind_completed_workspace(existing, data_root, project_root)
         plan_result = {
@@ -2345,6 +2940,9 @@ def datasource_run_command(args: argparse.Namespace) -> dict:
         ))
     save_composite_result(evidence_dir, "01-plan.json", plan_result, files)
     state = read_onboarding_state(data_root, profile)
+    checkpoint_recovery = reconcile_datasource_checkpoint(
+        state, install_state, data_root, runtime_state,
+    )
 
     if step_completed(state, "datasourceConfigured"):
         configured = resumed_phase(profile, "datasourceConfigured")
@@ -2353,7 +2951,10 @@ def datasource_run_command(args: argparse.Namespace) -> dict:
     else:
         configure_dry = datasource_configure_command(argparse.Namespace(
             install_root=args.install_root, data_root=args.data_root, profile=profile, apply=False, replace=False,
+            runtime_password=requested_connection.get("password"),
         ))
+        if checkpoint_recovery:
+            configure_dry["checkpointRecovery"] = checkpoint_recovery
         save_composite_result(evidence_dir, "02-datasource-dry.json", configure_dry, files)
         if not args.approve_configure:
             return {
@@ -2368,7 +2969,10 @@ def datasource_run_command(args: argparse.Namespace) -> dict:
             }
         configured = datasource_configure_command(argparse.Namespace(
             install_root=args.install_root, data_root=args.data_root, profile=profile, apply=True, replace=False,
+            runtime_password=requested_connection.get("password"),
         ))
+        if checkpoint_recovery:
+            configured["checkpointRecovery"] = checkpoint_recovery
         save_composite_result(evidence_dir, "03-datasource-apply.json", configured, files)
         state = read_onboarding_state(data_root, profile)
 
@@ -2633,6 +3237,9 @@ def semantic_run_command(args: argparse.Namespace) -> dict:
             "queryValidated": True,
             "queryExecuted": True,
             "rowCount": verified_step.get("rowCount"),
+            "warningCount": verified_step.get("warningCount", 0),
+            "warningCodes": verified_step.get("warningCodes", []),
+            "warningEvidence": verified_step.get("warningEvidence"),
             "evidenceDir": str(evidence_dir),
             "evidenceFiles": files,
             "productionReady": False,
@@ -2655,6 +3262,9 @@ def semantic_run_command(args: argparse.Namespace) -> dict:
             "phaseStatus": "awaiting-query-execution-approval",
             "queryValidated": True,
             "queryExecuted": False,
+            "warningCount": query_validated.get("warningCount", 0),
+            "warningCodes": query_validated.get("warningCodes", []),
+            "warnings": query_validated.get("warnings", []),
             "evidenceDir": str(evidence_dir),
             "evidenceFiles": files,
             "next": "rerun with all three approval flags after bounded query execution is approved",
@@ -2686,6 +3296,9 @@ def semantic_run_command(args: argparse.Namespace) -> dict:
         "queryValidated": executed["queryValidated"],
         "queryExecuted": executed["queryExecuted"],
         "rowCount": executed["rowCount"],
+        "warningCount": executed.get("warningCount", 0),
+        "warningCodes": executed.get("warningCodes", []),
+        "warnings": executed.get("warnings", []),
         "evidenceDir": str(evidence_dir),
         "evidenceFiles": files,
         "productionReady": False,
@@ -2826,6 +3439,9 @@ def build_parser() -> argparse.ArgumentParser:
     start.add_argument("--namespace")
     start.add_argument("--timeout", type=int)
     start.add_argument("--java")
+    start.add_argument("--progress-file")
+    start.add_argument("--operation-id")
+    start.add_argument("--operation-kind", default="runtime-start")
     start.set_defaults(handler=runtime_start_command)
 
     stop = sub.add_parser("runtime-stop")
@@ -2977,6 +3593,10 @@ def build_parser() -> argparse.ArgumentParser:
 
 
 def main() -> None:
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if callable(reconfigure):
+            reconfigure(encoding="utf-8", errors="replace")
     args = build_parser().parse_args()
     try:
         result = args.handler(args)
@@ -2987,7 +3607,7 @@ def main() -> None:
     except OnboardingError as exc:
         if ACTIVE_PROGRESS is not None:
             ACTIVE_PROGRESS.fail(exc)
-        emit({"success": False, "error": {"code": "ONBOARDING_ERROR", "message": str(exc)}, "productionReady": False}, 1)
+        emit({"success": False, "error": {"code": getattr(exc, "code", "ONBOARDING_ERROR"), "message": str(exc)}, "productionReady": False}, 1)
     except Exception as exc:
         if ACTIVE_PROGRESS is not None:
             ACTIVE_PROGRESS.fail(exc)