miravo 0.4.0 → 0.6.0

package/src/lib/admin-platform-stream.ts

@@ -0,0 +1,788 @@
+/**
+ * SSE streaming handlers for platform apply/cleanup operations.
+ *
+ * Emits server-sent events as resources are created/updated/deleted,
+ * giving the console real-time progress feedback.
+ */
+
+import type { EventLog } from "@miravo/core";
+import { colors } from "consola/utils";
+import { platformLog } from "../logger.ts";
+import { formatDuration } from "../output.ts";
+import {
+  DEFAULT_APPLY_SCOPE,
+  DEFAULT_CLEANUP_SCOPE,
+  PlatformCleanupBodySchema,
+  resolveApplyContext,
+} from "./admin-platform-routes.ts";
+import type { ResourceProgress } from "./le/le-apply.ts";
+import { buildApplyPlan, executePlan } from "./le/le-apply.ts";
+import { buildCleanupPlan, executeCleanup } from "./le/le-cleanup.ts";
+import { extractTemplateId } from "./le/le-cli-helpers.ts";
+import { LEClient } from "./le/le-client.ts";
+import { formatThrownError } from "./le/le-concurrency.ts";
+import { LEDeviceHub } from "./le/le-devicehub.ts";
+import { LEDigitalTwins } from "./le/le-digital-twins.ts";
+import { DEFAULT_PLATFORM_ID, getEnvironment, touchEnvironment } from "./le/le-environments.ts";
+import type { ApplyPlan } from "./le/le-plan.ts";
+import {
+  appendPlatformOperation,
+  createPlatformOperationTracker,
+} from "./le/le-platform-operations.ts";
+import { createCatalog, resolveTemplateIdentifier } from "./template-locator.ts";
+
+// --- Active operation registry ---
+
+const activeOperations = new Map<string, AbortController>();
+
+/** Cancel an in-progress streaming operation. Returns true if found and aborted. */
+export function cancelOperation(operationId: string): boolean {
+  const ac = activeOperations.get(operationId);
+  if (!ac) return false;
+  ac.abort();
+  return true;
+}
+
+// --- SSE helpers ---
+
+type SSESend = (event: string, data: unknown) => void;
+
+function createSSEResponse(handler: (send: SSESend) => Promise<void>): Response {
+  return new Response(
+    new ReadableStream({
+      async start(controller) {
+        const encoder = new TextEncoder();
+        const send: SSESend = (event, data) => {
+          try {
+            controller.enqueue(
+              encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`),
+            );
+          } catch {
+            // Ignore errors if the stream controller is already closed (e.g., client disconnected)
+          }
+        };
+        try {
+          await handler(send);
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          send("error", { message: msg });
+        } finally {
+          try {
+            controller.close();
+          } catch {
+            // Ignore if already closed
+          }
+        }
+      },
+    }),
+    {
+      headers: {
+        "Content-Type": "text/event-stream",
+        "Cache-Control": "no-cache",
+        Connection: "keep-alive",
+      },
+    },
+  );
+}
+
+// --- Apply stream ---
+
+export function handleApplyStream(body: unknown, eventLog?: EventLog | undefined): Response {
+  return createSSEResponse(async (send) => {
+    const ctx = await resolveApplyContext(body);
+    if (ctx instanceof Response) {
+      const errorBody = (await ctx.json()) as { message?: string; error?: string };
+      send("error", { message: errorBody.message ?? "Request validation failed." });
+      return;
+    }
+
+    touchEnvironment(ctx.req.environmentId).catch(() => {});
+
+    const operationId = crypto.randomUUID();
+    const startTime = performance.now();
+    const startedAt = Date.now();
+
+    const scope = ctx.req.scope ?? DEFAULT_APPLY_SCOPE;
+    const ac = new AbortController();
+    activeOperations.set(operationId, ac);
+
+    let totalAppliedCount = 0;
+    let totalFailedCount = 0;
+    let currentPlan: ApplyPlan | undefined;
+
+    const maxAttempts = 3;
+    let attempt = 0;
+    const finalActivityLogs: Parameters<NonNullable<EventLog>["pushExternal"]>[0][] = [];
+    const tracker = createPlatformOperationTracker({
+      operationId,
+      operationType: "apply",
+      templateId: ctx.templateId,
+      environmentId: ctx.req.environmentId,
+      environmentLabel: ctx.env.label,
+      hostname: ctx.env.hostname,
+      scope,
+      filters: { area: ctx.req.area, line: ctx.req.line, instance: ctx.req.instance },
+      startedAt,
+    });
+
+    try {
+      while (attempt < maxAttempts) {
+        attempt++;
+
+        try {
+          const planOpts = {
+            templateId: ctx.templateId,
+            targetHost: ctx.env.hostname,
+            sourceHostname: ctx.env.sourceHostname,
+            scope,
+            filters: { area: ctx.req.area, line: ctx.req.line, instance: ctx.req.instance },
+            ifUnmanaged: ctx.req.ifUnmanaged ?? "fail",
+            dryRun: false,
+          };
+
+          const { plan, snapshot } = await buildApplyPlan(ctx.hub, ctx.dt, ctx.desired, planOpts);
+          currentPlan = plan;
+
+          const totalPlanned = plan.entries.length;
+          const phaseTotals = computePhaseTotals(plan.entries);
+
+          send("started", {
+            operationId,
+            type: "apply",
+            template: ctx.req.template,
+            model: ctx.req.model,
+            environment: ctx.env.label,
+            totalPlanned,
+            attempt,
+            maxAttempts,
+          });
+
+          platformLog.start(
+            `apply ${colors.bold(ctx.templateId)} ${colors.dim("\u2192")} ${colors.cyan(ctx.env.label)} · ${colors.bold(String(totalPlanned))} planned`,
+          );
+
+          let currentPhase = "";
+          const applyOpts = {
+            ...planOpts,
+            onResourceComplete: (progress: ResourceProgress) => {
+              if (progress.status === "success") {
+                totalAppliedCount++;
+              } else {
+                totalFailedCount++;
+              }
+
+              tracker.onResourceComplete(progress);
+              if (progress.phase !== currentPhase) {
+                currentPhase = progress.phase;
+                send("phase", {
+                  operationId,
+                  phase: currentPhase,
+                  total: phaseTotals.get(currentPhase) ?? 0,
+                });
+              }
+              send("resource", {
+                operationId,
+                phase: progress.phase,
+                name: progress.name,
+                action: progress.action,
+                status: progress.status,
+                ...(progress.error ? { error: progress.error } : {}),
+                index: progress.index,
+                of: progress.of,
+              });
+
+              finalActivityLogs.push({
+                type: "platform:resource",
+                operationId,
+                operationType: "apply",
+                templateId: ctx.templateId,
+                environmentLabel: ctx.env.label,
+                hostname: ctx.env.hostname,
+                phase: progress.phase,
+                resourceType: progress.resourceType,
+                resourceName: progress.name,
+                action: progress.action,
+                status: progress.status,
+                ...(progress.error ? { error: progress.error } : {}),
+              });
+            },
+          };
+
+          const blockers = plan.entries.filter((e) => e.action === "blocked");
+          if (blockers.length > 0) {
+            const duration = Math.round(performance.now() - startTime);
+            const completedAt = Date.now();
+            platformLog.warn(
+              `apply blocked · ${colors.yellow(String(blockers.length))} blocked · ${formatDuration(duration)}`,
+            );
+
+            const blockerIndices = new Map<string, number>();
+            for (const b of blockers) {
+              const idx = (blockerIndices.get(b.resourceType) ?? 0) + 1;
+              blockerIndices.set(b.resourceType, idx);
+
+              applyOpts.onResourceComplete({
+                phase: b.resourceType,
+                resourceType: b.resourceType,
+                name: b.name,
+                action: b.action,
+                status: "failed",
+                error:
+                  "blockReason" in b && typeof b.blockReason === "string"
+                    ? b.blockReason
+                    : "Resource is blocked",
+                index: idx,
+                of: phaseTotals.get(b.resourceType) ?? blockers.length,
+              });
+            }
+
+            send("complete", {
+              operationId,
+              applied: totalAppliedCount,
+              failed: totalFailedCount,
+              duration,
+              plan,
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            eventLog?.pushExternal({
+              type: "platform:operation",
+              operationId,
+              operationType: "apply",
+              templateId: ctx.templateId,
+              environmentLabel: ctx.env.label,
+              hostname: ctx.env.hostname,
+              applied: totalAppliedCount,
+              failed: totalFailedCount,
+              duration,
+            });
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalAppliedCount,
+                failedCount: totalFailedCount,
+                blockedCount: blockers.length,
+                completedAt,
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          const result = await executePlan(
+            ctx.hub,
+            ctx.dt,
+            plan,
+            ctx.desired,
+            { ...applyOpts, signal: ac.signal },
+            snapshot,
+          );
+          const duration = Math.round(performance.now() - startTime);
+          const completedAt = Date.now();
+
+          if (ac.signal.aborted) {
+            platformLog.info(
+              `apply cancelled · ${colors.green(String(totalAppliedCount))} applied · ${formatDuration(duration)}`,
+            );
+            send("cancelled", {
+              operationId,
+              applied: totalAppliedCount,
+              failed: totalFailedCount,
+              duration,
+              plan: result.plan,
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalAppliedCount,
+                failedCount: totalFailedCount,
+                completedAt,
+                error: "Cancelled",
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          if (result.failed > 0 && attempt < maxAttempts) {
+            platformLog.warn(
+              `apply attempt ${attempt} finished with ${colors.yellow(String(result.failed))} errors, retrying in 3s...`,
+            );
+            send("retrying", {
+              operationId,
+              attempt,
+              maxAttempts,
+              delayMs: 3000,
+              error: `Attempt ${attempt} finished with ${result.failed} errors`,
+            });
+            await Bun.sleep(3000);
+            continue;
+          }
+
+          if (result.failed > 0) {
+            platformLog.warn(
+              `apply done · ${colors.green(String(totalAppliedCount))} created · ${colors.yellow(String(totalFailedCount))} failed · ${formatDuration(duration)}`,
+            );
+          } else {
+            platformLog.success(
+              `apply done · ${colors.bold(String(totalAppliedCount))} created · ${formatDuration(duration)}`,
+            );
+          }
+
+          send("complete", {
+            operationId,
+            applied: totalAppliedCount,
+            failed: totalFailedCount,
+            duration,
+            plan: result.plan,
+          });
+
+          for (const log of finalActivityLogs) {
+            eventLog?.pushExternal(log);
+          }
+          eventLog?.pushExternal({
+            type: "platform:operation",
+            operationId,
+            operationType: "apply",
+            templateId: ctx.templateId,
+            environmentLabel: ctx.env.label,
+            hostname: ctx.env.hostname,
+            applied: totalAppliedCount,
+            failed: totalFailedCount,
+            duration,
+          });
+          await appendPlatformOperation(
+            tracker.finalize({
+              successCount: totalAppliedCount,
+              failedCount: totalFailedCount,
+              completedAt,
+              ...(result.error ? { error: result.error } : {}),
+            }),
+          ).catch(() => {});
+          return;
+        } catch (err) {
+          const duration = Math.round(performance.now() - startTime);
+          const completedAt = Date.now();
+
+          if (ac.signal.aborted || (err instanceof Error && err.name === "AbortError")) {
+            platformLog.info(
+              `apply cancelled · ${colors.green(String(totalAppliedCount))} applied · ${formatDuration(duration)}`,
+            );
+            send("cancelled", {
+              operationId,
+              applied: totalAppliedCount,
+              failed: totalFailedCount,
+              duration,
+              plan: currentPlan ?? {
+                sourceMode: "static",
+                targetHost: ctx.env.hostname,
+                scope: [],
+                filters: {},
+                entries: [],
+              },
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalAppliedCount,
+                failedCount: totalFailedCount,
+                completedAt,
+                error: "Cancelled",
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          const message = formatThrownError(err);
+
+          if (attempt < maxAttempts) {
+            platformLog.warn(`apply attempt ${attempt} failed: ${message}, retrying in 3s...`);
+            send("retrying", {
+              operationId,
+              attempt,
+              maxAttempts,
+              delayMs: 3000,
+              error: message,
+            });
+            await Bun.sleep(3000);
+            continue;
+          }
+
+          platformLog.error(`apply failed · ${message}`);
+          for (const log of finalActivityLogs) {
+            eventLog?.pushExternal(log);
+          }
+          await appendPlatformOperation(
+            tracker.finalize({
+              successCount: totalAppliedCount,
+              failedCount: totalFailedCount || 1,
+              completedAt,
+              error: message,
+            }),
+          ).catch(() => {});
+          throw err;
+        }
+      }
+    } finally {
+      activeOperations.delete(operationId);
+    }
+  });
+}
+
+// --- Cleanup stream ---
+
+export function handleCleanupStream(body: unknown, eventLog?: EventLog | undefined): Response {
+  return createSSEResponse(async (send) => {
+    const parsed = PlatformCleanupBodySchema.safeParse(body);
+    if (!parsed.success) {
+      send("error", { message: parsed.error.issues.map((i) => i.message).join("; ") });
+      return;
+    }
+    const req = parsed.data;
+
+    const env = await getEnvironment(req.environmentId);
+    if (!env) {
+      send("error", { message: "Environment not found." });
+      return;
+    }
+    if (env.platformId !== DEFAULT_PLATFORM_ID) {
+      send("error", {
+        message: `Environment "${env.label}" uses unsupported platform "${env.platformId}".`,
+      });
+      return;
+    }
+
+    touchEnvironment(req.environmentId).catch(() => {});
+
+    const allManaged = req.allManaged ?? false;
+    if (!allManaged && !req.template) {
+      send("error", { message: "Provide 'template' or set 'allManaged: true'." });
+      return;
+    }
+
+    const operationId = crypto.randomUUID();
+    const startTime = performance.now();
+    const startedAt = Date.now();
+
+    const client = new LEClient({ hostname: env.hostname, apiToken: env.apiToken });
+    const hub = new LEDeviceHub(client);
+    const dt = new LEDigitalTwins(client);
+    await client.checkAuth();
+
+    let templateId = "all-managed";
+    if (!allManaged && req.template) {
+      const catalog = createCatalog();
+      const templatePath = await resolveTemplateIdentifier(req.template, catalog);
+      templateId = extractTemplateId(templatePath);
+    }
+
+    const scope = req.scope ?? DEFAULT_CLEANUP_SCOPE;
+    const ac = new AbortController();
+    activeOperations.set(operationId, ac);
+
+    let totalDeletedCount = 0;
+    let totalFailedCount = 0;
+    let currentPlan: ApplyPlan | undefined;
+
+    const maxAttempts = 3;
+    let attempt = 0;
+    const finalActivityLogs: Parameters<NonNullable<EventLog>["pushExternal"]>[0][] = [];
+    const tracker = createPlatformOperationTracker({
+      operationId,
+      operationType: "cleanup",
+      templateId,
+      environmentId: req.environmentId,
+      environmentLabel: env.label,
+      hostname: env.hostname,
+      scope,
+      filters: { area: req.area, line: req.line, instance: req.instance },
+      startedAt,
+    });
+
+    try {
+      while (attempt < maxAttempts) {
+        attempt++;
+
+        try {
+          const basePlanOpts = {
+            templateId,
+            targetHost: env.hostname,
+            scope,
+            filters: { area: req.area, line: req.line, instance: req.instance },
+            allManaged,
+            dryRun: false,
+          };
+
+          const planResult = await buildCleanupPlan(hub, dt, basePlanOpts);
+          currentPlan = planResult.plan;
+          const totalPlanned = planResult.plan.entries.length;
+          const phaseTotals = computePhaseTotals(planResult.plan.entries);
+
+          send("started", {
+            operationId,
+            type: "cleanup",
+            template: req.template,
+            environment: env.label,
+            totalPlanned,
+            attempt,
+            maxAttempts,
+          });
+
+          platformLog.start(
+            `cleanup ${colors.bold(templateId)} ${colors.dim("\u2192")} ${colors.cyan(env.label)} · ${colors.bold(String(totalPlanned))} planned`,
+          );
+
+          let currentPhase = "";
+          const cleanupOpts = {
+            ...basePlanOpts,
+            onResourceComplete: (progress: ResourceProgress) => {
+              if (progress.status === "success") {
+                totalDeletedCount++;
+              } else {
+                totalFailedCount++;
+              }
+
+              tracker.onResourceComplete(progress);
+              if (progress.phase !== currentPhase) {
+                currentPhase = progress.phase;
+                send("phase", {
+                  operationId,
+                  phase: currentPhase,
+                  total: phaseTotals.get(currentPhase) ?? 0,
+                });
+              }
+              send("resource", {
+                operationId,
+                phase: progress.phase,
+                name: progress.name,
+                action: progress.action,
+                status: progress.status,
+                ...(progress.error ? { error: progress.error } : {}),
+                index: progress.index,
+                of: progress.of,
+              });
+
+              finalActivityLogs.push({
+                type: "platform:resource",
+                operationId,
+                operationType: "cleanup",
+                templateId,
+                environmentLabel: env.label,
+                hostname: env.hostname,
+                phase: progress.phase,
+                resourceType: progress.resourceType,
+                resourceName: progress.name,
+                action: progress.action,
+                status: progress.status,
+                ...(progress.error ? { error: progress.error } : {}),
+              });
+            },
+          };
+
+          if (planResult.plan.entries.length === 0) {
+            const duration = Math.round(performance.now() - startTime);
+            const completedAt = Date.now();
+            platformLog.info(`cleanup done · nothing to remove · ${formatDuration(duration)}`);
+            send("complete", {
+              operationId,
+              deleted: totalDeletedCount,
+              failed: totalFailedCount,
+              duration,
+              plan: planResult.plan,
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            eventLog?.pushExternal({
+              type: "platform:operation",
+              operationId,
+              operationType: "cleanup",
+              templateId,
+              environmentLabel: env.label,
+              hostname: env.hostname,
+              applied: totalDeletedCount,
+              failed: totalFailedCount,
+              duration,
+            });
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalDeletedCount,
+                failedCount: totalFailedCount,
+                completedAt,
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          const result = await executeCleanup(hub, dt, planResult, {
+            ...cleanupOpts,
+            signal: ac.signal,
+          });
+          const duration = Math.round(performance.now() - startTime);
+          const completedAt = Date.now();
+
+          if (ac.signal.aborted) {
+            platformLog.info(
+              `cleanup cancelled · ${colors.green(String(totalDeletedCount))} deleted · ${formatDuration(duration)}`,
+            );
+            send("cancelled", {
+              operationId,
+              deleted: totalDeletedCount,
+              failed: totalFailedCount,
+              duration,
+              plan: result.plan,
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalDeletedCount,
+                failedCount: totalFailedCount,
+                completedAt,
+                error: "Cancelled",
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          if (result.failed > 0 && attempt < maxAttempts) {
+            platformLog.warn(
+              `cleanup attempt ${attempt} finished with ${colors.yellow(String(result.failed))} errors, retrying in 3s...`,
+            );
+            send("retrying", {
+              operationId,
+              attempt,
+              maxAttempts,
+              delayMs: 3000,
+              error: `Attempt ${attempt} finished with ${result.failed} errors`,
+            });
+            await Bun.sleep(3000);
+            continue;
+          }
+
+          if (result.failed > 0) {
+            platformLog.warn(
+              `cleanup done · ${colors.green(String(totalDeletedCount))} deleted · ${colors.yellow(String(totalFailedCount))} failed · ${formatDuration(duration)}`,
+            );
+          } else {
+            platformLog.success(
+              `cleanup done · ${colors.bold(String(totalDeletedCount))} deleted · ${formatDuration(duration)}`,
+            );
+          }
+
+          send("complete", {
+            operationId,
+            deleted: totalDeletedCount,
+            failed: totalFailedCount,
+            duration,
+            plan: result.plan,
+          });
+
+          for (const log of finalActivityLogs) {
+            eventLog?.pushExternal(log);
+          }
+          eventLog?.pushExternal({
+            type: "platform:operation",
+            operationId,
+            operationType: "cleanup",
+            templateId,
+            environmentLabel: env.label,
+            hostname: env.hostname,
+            applied: totalDeletedCount,
+            failed: totalFailedCount,
+            duration,
+          });
+          await appendPlatformOperation(
+            tracker.finalize({
+              successCount: totalDeletedCount,
+              failedCount: totalFailedCount,
+              completedAt,
+              ...(result.error ? { error: result.error } : {}),
+            }),
+          ).catch(() => {});
+          return;
+        } catch (err) {
+          const duration = Math.round(performance.now() - startTime);
+          const completedAt = Date.now();
+
+          if (ac.signal.aborted || (err instanceof Error && err.name === "AbortError")) {
+            platformLog.info(
+              `cleanup cancelled · ${colors.green(String(totalDeletedCount))} deleted · ${formatDuration(duration)}`,
+            );
+            send("cancelled", {
+              operationId,
+              deleted: totalDeletedCount,
+              failed: totalFailedCount,
+              duration,
+              plan: currentPlan ?? {
+                sourceMode: "static",
+                targetHost: env.hostname,
+                scope: [],
+                filters: {},
+                entries: [],
+              },
+            });
+            for (const log of finalActivityLogs) {
+              eventLog?.pushExternal(log);
+            }
+            await appendPlatformOperation(
+              tracker.finalize({
+                successCount: totalDeletedCount,
+                failedCount: totalFailedCount,
+                completedAt,
+                error: "Cancelled",
+              }),
+            ).catch(() => {});
+            return;
+          }
+
+          const message = formatThrownError(err);
+
+          if (attempt < maxAttempts) {
+            platformLog.warn(`cleanup attempt ${attempt} failed: ${message}, retrying in 3s...`);
+            send("retrying", {
+              operationId,
+              attempt,
+              maxAttempts,
+              delayMs: 3000,
+              error: message,
+            });
+            await Bun.sleep(3000);
+            continue;
+          }
+
+          platformLog.error(`cleanup failed · ${message}`);
+          for (const log of finalActivityLogs) {
+            eventLog?.pushExternal(log);
+          }
+          await appendPlatformOperation(
+            tracker.finalize({
+              successCount: totalDeletedCount,
+              failedCount: totalFailedCount || 1,
+              completedAt,
+              error: message,
+            }),
+          ).catch(() => {});
+          throw err;
+        }
+      }
+    } finally {
+      activeOperations.delete(operationId);
+    }
+  });
+}
+
+// --- Shared helpers ---
+
+function computePhaseTotals(
+  entries: readonly { action: string; resourceType: string }[],
+): Map<string, number> {
+  const totals = new Map<string, number>();
+  for (const entry of entries) {
+    totals.set(entry.resourceType, (totals.get(entry.resourceType) ?? 0) + 1);
+  }
+  return totals;
+}