@lobu/gateway 2.8.0

package/dist/orchestration/impl/k8s/deployment.js

@@ -0,0 +1,620 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.K8sDeploymentManager = exports.IMAGE_PULL_FAILURE_REASONS = exports.WORKER_SECURITY = exports.LOBU_FINALIZER = void 0;
+const k8s = __importStar(require("@kubernetes/client-node"));
+const core_1 = require("@lobu/core");
+const base_deployment_manager_1 = require("../../base-deployment-manager");
+const deployment_utils_1 = require("../../deployment-utils");
+const helpers_1 = require("./helpers");
+exports.LOBU_FINALIZER = "lobu.io/cleanup";
+exports.WORKER_SECURITY = {
+    USER_ID: 1001,
+    GROUP_ID: 1001,
+    TMP_SIZE_LIMIT: "100Mi",
+};
+const WORKER_SELECTOR_LABELS = {
+    "app.kubernetes.io/name": deployment_utils_1.BASE_WORKER_LABELS["app.kubernetes.io/name"],
+    "app.kubernetes.io/component": deployment_utils_1.BASE_WORKER_LABELS["app.kubernetes.io/component"],
+};
+exports.IMAGE_PULL_FAILURE_REASONS = new Set([
+    "ImagePullBackOff",
+    "ErrImagePull",
+    "InvalidImageName",
+    "RegistryUnavailable",
+]);
+const logger = (0, core_1.createLogger)("k8s-deployment");
+class K8sDeploymentManager extends base_deployment_manager_1.BaseDeploymentManager {
+    constructor(config, moduleEnvVarsBuilder, providerModules = []) {
+        var _a, _b, _c;
+        super(config, moduleEnvVarsBuilder, providerModules);
+        this.informer = null;
+        this.informerInitializing = false;
+        const kc = new k8s.KubeConfig();
+        try {
+            // Try in-cluster config first, then fall back to default
+            if (process.env.KUBERNETES_SERVICE_HOST) {
+                try {
+                    kc.loadFromCluster();
+                }
+                catch (_clusterError) {
+                    kc.loadFromDefault();
+                }
+            }
+            else {
+                kc.loadFromDefault();
+            }
+            // For development environments, disable TLS verification to avoid certificate issues
+            if (process.env.NODE_ENV === "development" ||
+                ((_a = process.env.KUBERNETES_SERVICE_HOST) === null || _a === void 0 ? void 0 : _a.includes("127.0.0.1")) ||
+                ((_b = process.env.KUBERNETES_SERVICE_HOST) === null || _b === void 0 ? void 0 : _b.includes("192.168")) ||
+                ((_c = process.env.KUBERNETES_SERVICE_HOST) === null || _c === void 0 ? void 0 : _c.includes("localhost"))) {
+                const cluster = kc.getCurrentCluster();
+                if (cluster &&
+                    typeof cluster === "object" &&
+                    cluster.skipTLSVerify !== true) {
+                    // Safely set skipTLSVerify property with type checking
+                    Object.defineProperty(cluster, "skipTLSVerify", {
+                        value: true,
+                        writable: true,
+                        enumerable: true,
+                        configurable: true,
+                    });
+                }
+            }
+        }
+        catch (error) {
+            logger.error("❌ Failed to load Kubernetes config:", error);
+            throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Failed to initialize Kubernetes client: ${error instanceof Error ? error.message : String(error)}`, { error }, true);
+        }
+        // Store KubeConfig for informer creation
+        this.kc = kc;
+        // Configure K8s API clients
+        this.appsV1Api = kc.makeApiClient(k8s.AppsV1Api);
+        this.coreV1Api = kc.makeApiClient(k8s.CoreV1Api);
+        this.nodeV1Api = kc.makeApiClient(k8s.NodeV1Api);
+        // API clients are already configured with authentication through makeApiClient
+        logger.info(`🔧 K8s client initialized for namespace: ${this.config.kubernetes.namespace}`);
+        // Validate namespace exists and we have access
+        this.validateNamespace();
+        // Check runtime class availability on initialization (like Docker's gVisor check)
+        this.checkRuntimeClassAvailability();
+    }
+    /**
+     * Validate that the target namespace exists and we have access to it
+     */
+    async validateNamespace() {
+        const namespace = this.config.kubernetes.namespace;
+        try {
+            await this.coreV1Api.readNamespace(namespace);
+            logger.info(`✅ Namespace '${namespace}' validated`);
+        }
+        catch (error) {
+            const k8sError = error;
+            if (k8sError.statusCode === 404) {
+                logger.error(`❌ Namespace '${namespace}' does not exist. ` +
+                    `Create it with: kubectl create namespace ${namespace}`);
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Namespace '${namespace}' does not exist`, { namespace }, true);
+            }
+            else if (k8sError.statusCode === 403) {
+                // 403 Forbidden for namespace read is expected with namespace-scoped Roles
+                // The gateway can still create resources in the namespace without cluster-level namespace read permission
+                logger.info(`ℹ️  Namespace '${namespace}' access check skipped (namespace-scoped RBAC). ` +
+                    `Will validate via resource operations.`);
+                // Don't throw - we're running in this namespace so it exists
+            }
+            else {
+                logger.warn(`⚠️  Could not validate namespace '${namespace}': ${error instanceof Error ? error.message : String(error)}`);
+                // Don't throw - let operations fail with more specific errors
+            }
+        }
+    }
+    /**
+     * Check if the configured RuntimeClass exists in the cluster
+     * Similar to Docker's checkGvisorAvailability()
+     */
+    async checkRuntimeClassAvailability() {
+        const runtimeClassName = this.config.worker.runtimeClassName || "kata";
+        try {
+            await this.nodeV1Api.readRuntimeClass(runtimeClassName);
+            logger.info(`✅ RuntimeClass '${runtimeClassName}' verified and will be used for worker isolation`);
+        }
+        catch (error) {
+            const k8sError = error;
+            if (k8sError.statusCode === 404) {
+                logger.warn(`⚠️  RuntimeClass '${runtimeClassName}' not found in cluster. ` +
+                    `Workers will use default runtime. Consider installing ${runtimeClassName} for enhanced isolation.`);
+            }
+            else {
+                logger.warn(`⚠️  Failed to verify RuntimeClass '${runtimeClassName}': ${error instanceof Error ? error.message : String(error)}`);
+            }
+            // Clear runtime class if not available or verification failed (workers will use default)
+            this.config.worker.runtimeClassName = undefined;
+        }
+    }
+    getWorkerServiceAccountName() {
+        return this.config.worker.serviceAccountName || "lobu-worker";
+    }
+    getWorkerImagePullSecrets() {
+        const configured = this.config.worker.imagePullSecrets || [];
+        const names = configured.map((name) => name.trim()).filter(Boolean);
+        if (names.length === 0)
+            return undefined;
+        return names.map((name) => ({ name }));
+    }
+    getWorkerStartupTimeoutMs() {
+        var _a;
+        const timeoutSeconds = (_a = this.config.worker.startupTimeoutSeconds) !== null && _a !== void 0 ? _a : 90;
+        return Math.max(timeoutSeconds, 5) * 1000;
+    }
+    async listRawWorkerDeployments() {
+        var _a;
+        const k8sDeployments = await this.appsV1Api.listNamespacedDeployment(this.config.kubernetes.namespace, undefined, // pretty
+        undefined, // allowWatchBookmarks
+        undefined, // _continue
+        undefined, // fieldSelector
+        "app.kubernetes.io/component=worker" // labelSelector - only worker deployments
+        );
+        const response = k8sDeployments;
+        return ((_a = response.body) === null || _a === void 0 ? void 0 : _a.items) || [];
+    }
+    /**
+     * Validate that the worker image exists and is pullable
+     * Called on gateway startup to ensure workers can be created
+     */
+    async validateWorkerImage() {
+        const imageName = this.getWorkerImageReference();
+        logger.info(`ℹ️  Worker image configured: ${imageName} (pullPolicy: ${this.config.worker.image.pullPolicy || "Always"})`);
+        if (this.config.worker.image.pullPolicy === "Never") {
+            logger.warn(`⚠️  Worker image pullPolicy is 'Never'. Ensure image ${imageName} is pre-loaded on all nodes.`);
+            return;
+        }
+        await (0, helpers_1.runImagePullPreflight)(this.coreV1Api, this.config.kubernetes.namespace, imageName, this.config.worker.image.pullPolicy || "Always", this.getWorkerServiceAccountName(), this.getWorkerImagePullSecrets());
+    }
+    async reconcileWorkerDeploymentImages() {
+        await (0, helpers_1.reconcileWorkerDeploymentImages)(this.appsV1Api, this.config.kubernetes.namespace, this.getWorkerImageReference(), this.config.worker.image.pullPolicy || "Always", this.getWorkerServiceAccountName(), this.getWorkerImagePullSecrets(), () => this.listRawWorkerDeployments());
+    }
+    async listDeployments() {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k;
+        try {
+            const now = Date.now();
+            const idleThresholdMinutes = this.config.worker.idleCleanupMinutes;
+            const veryOldDays = (0, deployment_utils_1.getVeryOldThresholdDays)(this.config);
+            const results = [];
+            for (const deployment of await this.listRawWorkerDeployments()) {
+                const deploymentName = ((_a = deployment.metadata) === null || _a === void 0 ? void 0 : _a.name) || "";
+                // Clean up orphaned finalizers on Terminating deployments (avoids extra API call)
+                if (((_b = deployment.metadata) === null || _b === void 0 ? void 0 : _b.deletionTimestamp) &&
+                    ((_d = (_c = deployment.metadata) === null || _c === void 0 ? void 0 : _c.finalizers) === null || _d === void 0 ? void 0 : _d.includes(exports.LOBU_FINALIZER))) {
+                    logger.info(`Removing orphaned finalizer from Terminating deployment ${deploymentName}`);
+                    (0, helpers_1.removeFinalizerFromResource)(this.appsV1Api, this.coreV1Api, this.config.kubernetes.namespace, "deployment", deploymentName).catch((err) => logger.warn(`Failed to remove orphaned finalizer from ${deploymentName}:`, err instanceof Error ? err.message : String(err)));
+                    continue; // Skip Terminating deployments from the active list
+                }
+                // Get last activity from annotations or fallback to creation time
+                const lastActivityStr = ((_f = (_e = deployment.metadata) === null || _e === void 0 ? void 0 : _e.annotations) === null || _f === void 0 ? void 0 : _f["lobu.io/last-activity"]) ||
+                    ((_h = (_g = deployment.metadata) === null || _g === void 0 ? void 0 : _g.annotations) === null || _h === void 0 ? void 0 : _h["lobu.io/created"]) ||
+                    ((_j = deployment.metadata) === null || _j === void 0 ? void 0 : _j.creationTimestamp);
+                const lastActivity = lastActivityStr
+                    ? new Date(lastActivityStr)
+                    : new Date();
+                const replicas = ((_k = deployment.spec) === null || _k === void 0 ? void 0 : _k.replicas) || 0;
+                results.push((0, deployment_utils_1.buildDeploymentInfoSummary)({
+                    deploymentName,
+                    lastActivity,
+                    now,
+                    idleThresholdMinutes,
+                    veryOldDays,
+                    replicas,
+                }));
+            }
+            return results;
+        }
+        catch (error) {
+            throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Failed to list deployments: ${error instanceof Error ? error.message : String(error)}`, { error }, true);
+        }
+    }
+    async createDeployment(deploymentName, username, userId, messageData) {
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
+        // Extract traceparent for distributed tracing
+        const traceparent = (_a = messageData === null || messageData === void 0 ? void 0 : messageData.platformMetadata) === null || _a === void 0 ? void 0 : _a.traceparent;
+        logger.info({ traceparent, deploymentName, userId }, "Creating K8s deployment");
+        // Use agentId for PVC naming (shared across threads in same space)
+        const agentId = messageData === null || messageData === void 0 ? void 0 : messageData.agentId;
+        if (!agentId) {
+            throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, "Missing agentId in message payload");
+        }
+        const pvcName = `lobu-workspace-${agentId}`;
+        // Check if Nix packages are configured (need init container + subPath mounts)
+        const hasNixConfig = ((_d = (_c = (_b = messageData === null || messageData === void 0 ? void 0 : messageData.nixConfig) === null || _b === void 0 ? void 0 : _b.packages) === null || _c === void 0 ? void 0 : _c.length) !== null && _d !== void 0 ? _d : 0) > 0 ||
+            !!((_e = messageData === null || messageData === void 0 ? void 0 : messageData.nixConfig) === null || _e === void 0 ? void 0 : _e.flakeUrl);
+        // Use larger PVC when Nix packages are configured (Chromium etc. need space)
+        const pvcSize = hasNixConfig ? "5Gi" : undefined;
+        await (0, helpers_1.createPVC)(this.coreV1Api, this.config.kubernetes.namespace, pvcName, agentId, (_f = this.config.worker.persistence) === null || _f === void 0 ? void 0 : _f.storageClass, traceparent, pvcSize, (_g = this.config.worker.persistence) === null || _g === void 0 ? void 0 : _g.size);
+        // Get environment variables before creating the deployment spec
+        // Include secrets (same as Docker behavior) - secrets are passed via env vars
+        const envVars = await this.generateEnvironmentVariables(username, userId, deploymentName, messageData, true // Include secrets to match Docker behavior
+        );
+        const platform = (messageData === null || messageData === void 0 ? void 0 : messageData.platform) || "unknown";
+        const workerImage = this.getWorkerImageReference();
+        const deployment = {
+            apiVersion: "apps/v1",
+            kind: "Deployment",
+            metadata: {
+                name: deploymentName,
+                namespace: this.config.kubernetes.namespace,
+                labels: Object.assign(Object.assign({}, deployment_utils_1.BASE_WORKER_LABELS), { "lobu.io/platform": platform, "lobu.io/agent-id": agentId }),
+                annotations: {
+                    "lobu.io/status": "running",
+                    "lobu.io/created": new Date().toISOString(),
+                },
+                finalizers: [exports.LOBU_FINALIZER],
+            },
+            spec: {
+                replicas: 1,
+                selector: {
+                    matchLabels: Object.assign({}, WORKER_SELECTOR_LABELS),
+                },
+                template: {
+                    metadata: {
+                        annotations: Object.assign(Object.assign(Object.assign({}, (0, deployment_utils_1.resolvePlatformDeploymentMetadata)(messageData)), { "lobu.io/created": new Date().toISOString(), "lobu.io/agent-id": agentId }), (traceparent ? { "lobu.io/traceparent": traceparent } : {})),
+                        labels: Object.assign(Object.assign({}, deployment_utils_1.BASE_WORKER_LABELS), { "lobu.io/platform": platform }),
+                    },
+                    spec: Object.assign(Object.assign(Object.assign(Object.assign({ serviceAccountName: this.getWorkerServiceAccountName(), imagePullSecrets: this.getWorkerImagePullSecrets() }, (this.config.worker.runtimeClassName
+                        ? { runtimeClassName: this.config.worker.runtimeClassName }
+                        : {})), { securityContext: {
+                            fsGroup: exports.WORKER_SECURITY.GROUP_ID,
+                            fsGroupChangePolicy: "OnRootMismatch",
+                        } }), (hasNixConfig
+                        ? {
+                            initContainers: [
+                                {
+                                    name: "nix-bootstrap",
+                                    image: workerImage,
+                                    imagePullPolicy: this.config.worker.image.pullPolicy || "Always",
+                                    command: [
+                                        "bash",
+                                        "-c",
+                                        "if [ ! -f /workspace/.nix-bootstrapped ]; then " +
+                                            'echo "Bootstrapping Nix store to PVC..." && ' +
+                                            "cp -a /nix/store /workspace/.nix-store && " +
+                                            "cp -a /nix/var /workspace/.nix-var && " +
+                                            "mkdir -p /workspace/.nix-store/.nix-pvc-mounted && " +
+                                            "touch /workspace/.nix-bootstrapped && " +
+                                            'echo "Nix bootstrap complete"; ' +
+                                            'else echo "Nix store already bootstrapped"; fi',
+                                    ],
+                                    securityContext: {
+                                        runAsUser: exports.WORKER_SECURITY.USER_ID,
+                                        runAsGroup: exports.WORKER_SECURITY.GROUP_ID,
+                                    },
+                                    volumeMounts: [
+                                        {
+                                            name: "workspace",
+                                            mountPath: "/workspace",
+                                        },
+                                    ],
+                                },
+                            ],
+                        }
+                        : {})), { containers: [
+                            {
+                                name: "worker",
+                                image: workerImage,
+                                imagePullPolicy: this.config.worker.image.pullPolicy || "Always",
+                                securityContext: {
+                                    runAsUser: exports.WORKER_SECURITY.USER_ID,
+                                    runAsGroup: exports.WORKER_SECURITY.GROUP_ID,
+                                    runAsNonRoot: true,
+                                    // Enable read-only root filesystem for security (matches Docker behavior)
+                                    readOnlyRootFilesystem: true,
+                                    // Prevent privilege escalation
+                                    allowPrivilegeEscalation: false,
+                                    // Drop all capabilities (matches Docker CAP_DROP: ALL)
+                                    capabilities: {
+                                        drop: ["ALL"],
+                                    },
+                                },
+                                env: [
+                                    // Common environment variables from base class
+                                    // (includes HTTP_PROXY, HTTPS_PROXY, NO_PROXY, NODE_ENV, DEBUG)
+                                    ...Object.entries(envVars).map(([key, value]) => ({
+                                        name: key,
+                                        value: value,
+                                    })),
+                                    // Add traceparent for distributed tracing (passed to worker)
+                                    ...(traceparent
+                                        ? [{ name: "TRACEPARENT", value: traceparent }]
+                                        : []),
+                                ],
+                                resources: {
+                                    requests: this.config.worker.resources.requests,
+                                    limits: this.config.worker.resources.limits,
+                                },
+                                volumeMounts: [
+                                    {
+                                        name: "workspace",
+                                        mountPath: "/workspace",
+                                    },
+                                    // Tmpfs mounts for writable directories (matches Docker behavior)
+                                    {
+                                        name: "tmp",
+                                        mountPath: "/tmp",
+                                    },
+                                    // /dev/shm for shared memory (needed by Chromium and other apps)
+                                    {
+                                        name: "dshm",
+                                        mountPath: "/dev/shm",
+                                    },
+                                    // When Nix packages configured, mount PVC subpaths at /nix/store and /nix/var
+                                    ...(hasNixConfig
+                                        ? [
+                                            {
+                                                name: "workspace",
+                                                mountPath: "/nix/store",
+                                                subPath: ".nix-store",
+                                            },
+                                            {
+                                                name: "workspace",
+                                                mountPath: "/nix/var",
+                                                subPath: ".nix-var",
+                                            },
+                                        ]
+                                        : []),
+                                ],
+                            },
+                        ], volumes: [
+                            {
+                                name: "workspace",
+                                // Use per-deployment PVC for session persistence across scale-to-zero
+                                persistentVolumeClaim: {
+                                    claimName: pvcName,
+                                },
+                            },
+                            // Tmpfs volumes for temporary files (in-memory, matches Docker Tmpfs)
+                            {
+                                name: "tmp",
+                                emptyDir: {
+                                    medium: "Memory",
+                                    sizeLimit: exports.WORKER_SECURITY.TMP_SIZE_LIMIT,
+                                },
+                            },
+                            // Shared memory for Chromium and other apps requiring /dev/shm
+                            {
+                                name: "dshm",
+                                emptyDir: {
+                                    medium: "Memory",
+                                    sizeLimit: "256Mi",
+                                },
+                            },
+                        ] }),
+                },
+            },
+        };
+        // Create child span for worker creation (linked to parent via traceparent)
+        const workerSpan = (0, core_1.createChildSpan)("worker_creation", traceparent, {
+            "lobu.deployment_name": deploymentName,
+            "lobu.user_id": userId,
+            "lobu.agent_id": agentId,
+        });
+        logger.info({ traceparent, deploymentName }, "Submitting deployment to K8s API");
+        try {
+            const response = await this.appsV1Api.createNamespacedDeployment(this.config.kubernetes.namespace, deployment);
+            await (0, helpers_1.waitForWorkerReady)(this.appsV1Api, this.coreV1Api, this.config.kubernetes.namespace, deploymentName, this.getWorkerStartupTimeoutMs());
+            const statusResponse = response;
+            workerSpan === null || workerSpan === void 0 ? void 0 : workerSpan.setAttribute("http.status_code", ((_h = statusResponse.response) === null || _h === void 0 ? void 0 : _h.statusCode) || 0);
+            workerSpan === null || workerSpan === void 0 ? void 0 : workerSpan.setStatus({ code: core_1.SpanStatusCode.OK });
+            workerSpan === null || workerSpan === void 0 ? void 0 : workerSpan.end();
+            logger.info({ deploymentName, status: (_j = statusResponse.response) === null || _j === void 0 ? void 0 : _j.statusCode }, "Deployment created and worker became ready");
+        }
+        catch (error) {
+            const k8sError = error;
+            // Log detailed error information
+            logger.error(`❌ Failed to create deployment ${deploymentName}:`, {
+                statusCode: k8sError.statusCode,
+                message: k8sError.message,
+                body: k8sError.body,
+                response: (_k = k8sError.response) === null || _k === void 0 ? void 0 : _k.statusMessage,
+            });
+            // Clean up the PVC that was created before the deployment failed
+            try {
+                await this.coreV1Api.deleteNamespacedPersistentVolumeClaim(pvcName, this.config.kubernetes.namespace);
+                logger.info(`Cleaned up orphaned PVC ${pvcName} after deployment creation failure`);
+            }
+            catch (pvcCleanupError) {
+                const pvcError = pvcCleanupError;
+                if (pvcError.statusCode === 404) {
+                    logger.debug(`PVC ${pvcName} already deleted, skipping cleanup`);
+                }
+                else {
+                    logger.error(`Failed to clean up orphaned PVC ${pvcName}:`, pvcCleanupError instanceof Error
+                        ? pvcCleanupError.message
+                        : String(pvcCleanupError));
+                }
+            }
+            // End span with error
+            workerSpan === null || workerSpan === void 0 ? void 0 : workerSpan.setStatus({
+                code: core_1.SpanStatusCode.ERROR,
+                message: k8sError.message || "Deployment failed",
+            });
+            workerSpan === null || workerSpan === void 0 ? void 0 : workerSpan.end();
+            // Check for specific error conditions and throw OrchestratorError
+            if (k8sError.statusCode === 409) {
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Deployment ${deploymentName} already exists`, { deploymentName, statusCode: 409 }, false);
+            }
+            else if (k8sError.statusCode === 403) {
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Insufficient permissions to create deployment ${deploymentName}`, { deploymentName, statusCode: 403 }, true);
+            }
+            else if (k8sError.statusCode === 422) {
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Invalid deployment specification for ${deploymentName}: ${JSON.stringify(k8sError.body)}`, { deploymentName, statusCode: 422, body: k8sError.body }, true);
+            }
+            else if (((_l = k8sError.message) === null || _l === void 0 ? void 0 : _l.includes("timeout")) ||
+                k8sError.code === "ETIMEDOUT") {
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `Timeout creating deployment ${deploymentName} - K8s API may be overloaded`, { deploymentName, code: k8sError.code }, true);
+            }
+            else {
+                throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_CREATE_FAILED, `HTTP request failed: ${k8sError.message || ((_m = k8sError.response) === null || _m === void 0 ? void 0 : _m.statusMessage) || "Unknown error"}`, { deploymentName, error }, true);
+            }
+        }
+    }
+    async scaleDeployment(deploymentName, replicas) {
+        var _a, _b;
+        try {
+            const deployment = await this.appsV1Api.readNamespacedDeployment(deploymentName, this.config.kubernetes.namespace);
+            if (((_b = (_a = deployment.body) === null || _a === void 0 ? void 0 : _a.spec) === null || _b === void 0 ? void 0 : _b.replicas) !== replicas) {
+                const patch = {
+                    metadata: {
+                        annotations: {
+                            "lobu.io/status": replicas > 0 ? "running" : "scaled-down",
+                        },
+                    },
+                    spec: {
+                        replicas: replicas,
+                    },
+                };
+                await this.appsV1Api.patchNamespacedDeployment(deploymentName, this.config.kubernetes.namespace, patch, undefined, undefined, undefined, undefined, undefined, {
+                    headers: {
+                        "Content-Type": "application/strategic-merge-patch+json",
+                    },
+                });
+            }
+            if (replicas > 0) {
+                await (0, helpers_1.waitForWorkerReady)(this.appsV1Api, this.coreV1Api, this.config.kubernetes.namespace, deploymentName, this.getWorkerStartupTimeoutMs());
+            }
+        }
+        catch (error) {
+            throw new core_1.OrchestratorError(core_1.ErrorCode.DEPLOYMENT_SCALE_FAILED, `Failed to scale deployment ${deploymentName}: ${error instanceof Error ? error.message : String(error)}`, { deploymentName, replicas, error }, true);
+        }
+    }
+    async deleteDeployment(deploymentName) {
+        // Remove our finalizer before deleting so the resource can be garbage-collected
+        await (0, helpers_1.removeFinalizerFromResource)(this.appsV1Api, this.coreV1Api, this.config.kubernetes.namespace, "deployment", deploymentName);
+        // Delete the deployment with propagation policy
+        try {
+            await this.appsV1Api.deleteNamespacedDeployment(deploymentName, this.config.kubernetes.namespace, undefined, undefined, undefined, undefined, "Foreground" // Wait for pods to terminate before returning
+            );
+            logger.info(`✅ Deleted deployment: ${deploymentName}`);
+        }
+        catch (error) {
+            const k8sError = error;
+            if (k8sError.statusCode === 404) {
+                logger.info(`⚠️  Deployment ${deploymentName} not found (already deleted)`);
+            }
+            else {
+                throw error;
+            }
+        }
+        // NOTE: Space PVCs are NOT deleted on deployment deletion
+        // They are shared across threads in the same space and persist
+        // for future conversations. Cleanup is done manually or via separate process.
+    }
+    /**
+     * Override reconcileDeployments to also clean up orphaned PVC finalizers.
+     * Deployment orphan cleanup is handled inside listDeployments() to avoid
+     * duplicate API calls (listDeployments already iterates raw K8s objects).
+     */
+    async reconcileDeployments() {
+        await this.reconcileWorkerDeploymentImages();
+        await (0, helpers_1.cleanupOrphanedPvcFinalizers)(this.appsV1Api, this.coreV1Api, this.config.kubernetes.namespace);
+        await super.reconcileDeployments();
+    }
+    async updateDeploymentActivity(deploymentName) {
+        try {
+            const timestamp = new Date().toISOString();
+            const patch = {
+                metadata: {
+                    annotations: {
+                        "lobu.io/last-activity": timestamp,
+                    },
+                },
+            };
+            await this.appsV1Api.patchNamespacedDeployment(deploymentName, this.config.kubernetes.namespace, patch, undefined, undefined, undefined, undefined, undefined, {
+                headers: { "Content-Type": "application/strategic-merge-patch+json" },
+            });
+        }
+        catch (error) {
+            logger.error(`❌ Failed to update activity for deployment ${deploymentName}:`, error instanceof Error ? error.message : String(error));
+            // Don't throw - activity tracking should not block message processing
+        }
+    }
+    getDispatcherHost() {
+        const dispatcherService = process.env.DISPATCHER_SERVICE_NAME || "lobu-dispatcher";
+        return `${dispatcherService}.${this.config.kubernetes.namespace}.svc.cluster.local`;
+    }
+    /**
+     * Start a watch-based informer for worker deployments.
+     * The informer maintains a local cache that is updated via K8s watch events,
+     * reducing the need for frequent list API calls.
+     */
+    async startInformer() {
+        if (this.informer || this.informerInitializing)
+            return;
+        this.informerInitializing = true;
+        const namespace = this.config.kubernetes.namespace;
+        const listFn = () => this.appsV1Api.listNamespacedDeployment(namespace, undefined, undefined, undefined, undefined, "app.kubernetes.io/component=worker");
+        try {
+            this.informer = k8s.makeInformer(this.kc, `/apis/apps/v1/namespaces/${namespace}/deployments`, listFn, "app.kubernetes.io/component=worker");
+            this.informer.on("error", (err) => {
+                logger.warn("Informer error, will auto-restart:", err instanceof Error ? err.message : String(err));
+            });
+            await this.informer.start();
+            logger.info("K8s deployment informer started");
+        }
+        catch (error) {
+            logger.warn("Failed to start informer, falling back to polling:", error instanceof Error ? error.message : String(error));
+            this.informer = null;
+        }
+        finally {
+            this.informerInitializing = false;
+        }
+    }
+    /**
+     * Stop the informer and clear the cache.
+     */
+    async stopInformer() {
+        if (this.informer) {
+            this.informer.stop();
+            this.informer = null;
+            logger.info("K8s deployment informer stopped");
+        }
+    }
+    /**
+     * Whether the informer is active and has a populated cache.
+     */
+    isInformerActive() {
+        return this.informer !== null;
+    }
+}
+exports.K8sDeploymentManager = K8sDeploymentManager;
+//# sourceMappingURL=deployment.js.map