@arcote.tech/arc-cli 0.7.19 → 0.7.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcote.tech/arc-cli",
-  "version": "0.7.19",
+  "version": "0.7.20",
   "description": "CLI tool for Arc framework",
   "module": "index.ts",
   "main": "dist/index.js",
@@ -12,13 +12,13 @@
     "build": "bun build --target=bun ./src/index.ts --outdir=dist --external @arcote.tech/arc --external @arcote.tech/arc-ds --external @arcote.tech/arc-react --external @arcote.tech/platform --external '@opentelemetry/*' && chmod +x dist/index.js"
   },
   "dependencies": {
-    "@arcote.tech/arc": "^0.7.19",
-    "@arcote.tech/arc-ds": "^0.7.19",
-    "@arcote.tech/arc-react": "^0.7.19",
-    "@arcote.tech/arc-host": "^0.7.19",
-    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.19",
-    "@arcote.tech/arc-adapter-db-postgres": "^0.7.19",
-    "@arcote.tech/arc-otel": "^0.7.19",
+    "@arcote.tech/arc": "^0.7.20",
+    "@arcote.tech/arc-ds": "^0.7.20",
+    "@arcote.tech/arc-react": "^0.7.20",
+    "@arcote.tech/arc-host": "^0.7.20",
+    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.20",
+    "@arcote.tech/arc-adapter-db-postgres": "^0.7.20",
+    "@arcote.tech/arc-otel": "^0.7.20",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/api-logs": "^0.57.0",
     "@opentelemetry/core": "^1.30.0",
@@ -31,7 +31,7 @@
     "@opentelemetry/sdk-trace-base": "^1.30.0",
     "@opentelemetry/sdk-trace-node": "^1.30.0",
     "@opentelemetry/semantic-conventions": "^1.27.0",
-    "@arcote.tech/platform": "^0.7.19",
+    "@arcote.tech/platform": "^0.7.20",
     "@clack/prompts": "^0.9.0",
     "commander": "^11.1.0",
     "chokidar": "^3.5.3",

package/src/deploy/bootstrap.ts

@@ -133,6 +133,9 @@ export async function bootstrap(inputs: BootstrapInputs): Promise<void> {
   //   - stack isn't fully ready, OR
   //   - marker is missing (legacy v0.5 deploy with no .arc-state.json), OR
   //   - configHash differs from last bootstrap (deploy.arc.json changed), OR
+  //   - the CLI version changed (generators evolve — compose/Caddyfile/
+  //     observability configs must be re-rendered + re-uploaded even when
+  //     deploy.arc.json itself is unchanged), OR
   //   - registry container isn't running (e.g. legacy stack predates v0.7)
   // Without this, an old v0.5 stack (no registry container) is classified as
   // "ready" and bootstrap is skipped — then `docker login` on the next step
@@ -141,6 +144,7 @@ export async function bootstrap(inputs: BootstrapInputs): Promise<void> {
     state.kind !== "ready" ||
     state.marker === null ||
     state.marker.configHash !== inputs.configHash ||
+    state.marker.cliVersion !== inputs.cliVersion ||
     !(await isRegistryRunning(cfg));
 
   if (needUpStack) {
@@ -156,7 +160,7 @@ export async function bootstrap(inputs: BootstrapInputs): Promise<void> {
   // without forcing a full bootstrap.
   if (cfg.observability?.enabled) {
     log("Ensuring observability sidecars are running...");
-    const obsServices = ["otel-collector", "tempo", "loki", "prometheus", "grafana"];
+    const obsServices = ["otel-collector", "tempo", "loki", "prometheus", "alloy", "grafana"];
     await assertExec(
       cfg.target,
       `cd ${cfg.target.remoteDir} && docker compose pull --ignore-pull-failures ${obsServices.join(" ")} && docker compose up -d ${obsServices.join(" ")}`,
@@ -307,10 +311,11 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
   // `observability/` directory referenced by compose bind-mounts.
   if (observabilityFiles && observabilityHtpasswd) {
     // Make sure both the top-level `observability/` dir and the nested
-    // `grafana-dashboards/` exist before scp tries to land files there.
+    // `grafana-dashboards/` + `grafana-alerting/` exist before scp tries
+    // to land files there.
     await assertExec(
       cfg.target,
-      `mkdir -p ${cfg.target.remoteDir}/observability/grafana-dashboards`,
+      `mkdir -p ${cfg.target.remoteDir}/observability/grafana-dashboards ${cfg.target.remoteDir}/observability/grafana-alerting`,
     );
     // Make sure local nested dir exists too — `generateObservabilityConfigs`
     // returns relative paths like `observability/grafana-dashboards/x.json`

package/src/deploy/caddyfile.ts

@@ -20,23 +20,48 @@ export function generateCaddyfile(cfg: DeployConfig): string {
     cfg.caddy.email === "internal" ? "" : `\n    email ${cfg.caddy.email}`;
   const tlsDirective =
     cfg.caddy.email === "internal" ? "\n    tls internal" : "";
+  const observability = cfg.observability?.enabled === true;
+
+  // Access logs land on stdout as JSON → Alloy tails the container → Loki
+  // (`{compose_service="caddy"}`). Off by default in Caddy, hence per-vhost.
+  const logDirective = observability
+    ? ["    log {", "        output stdout", "        format json", "    }"]
+    : [];
 
   const lines: string[] = [];
   lines.push("# Generated by `arc platform deploy` — do not edit by hand.");
   lines.push("");
   lines.push("{");
   lines.push("    admin off");
+  if (observability) {
+    // Per-request HTTP metrics (caddy_http_*). Top-level global option —
+    // requires Caddy >= 2.9 (the `servers > metrics` form is deprecated).
+    lines.push("    metrics {");
+    lines.push("        per_host");
+    lines.push("    }");
+  }
   if (email) lines.push(`   ${email.trim()}`);
   lines.push("}");
   lines.push("");
 
+  // Exposition endpoint for Prometheus (scrapes caddy:2020 on arc-net).
+  // Plain HTTP, never exposed publicly; works with `admin off` (only the
+  // admin-API /metrics endpoint dies with the admin interface).
+  if (observability) {
+    lines.push(":2020 {");
+    lines.push("    metrics");
+    lines.push("}");
+    lines.push("");
+  }
+
   // Public blocks — one per env. When observability is on, add a `/otel/*`
   // path that forwards browser-side OTLP/HTTP to the collector. Strips the
   // /otel prefix so the collector sees the same /v1/{traces,logs,metrics}
   // paths it would receive from same-network senders.
   for (const [name, env] of Object.entries(cfg.envs)) {
     lines.push(`${env.domain} {${tlsDirective}`);
-    if (cfg.observability?.enabled) {
+    lines.push(...logDirective);
+    if (observability) {
       lines.push("    handle_path /otel/* {");
       lines.push("        reverse_proxy otel-collector:4318");
       lines.push("    }");
@@ -54,13 +79,11 @@ export function generateCaddyfile(cfg: DeployConfig): string {
   // the apex of the first env's domain (e.g. observability.app.example.com
   // when the primary env is app.example.com). Caddy issues a separate
   // ACME certificate for this hostname.
-  if (cfg.observability?.enabled) {
-    const firstEnv = Object.values(cfg.envs)[0];
-    if (firstEnv) {
-      const subdomain = cfg.observability.subdomain ?? "observability";
-      const apex = apexOf(firstEnv.domain);
-      const observabilityDomain = `${subdomain}.${apex}`;
-      lines.push(`${observabilityDomain} {${tlsDirective}`);
+  if (observability) {
+    const domain = observabilityDomain(cfg);
+    if (domain) {
+      lines.push(`${domain} {${tlsDirective}`);
+      lines.push(...logDirective);
       // Basic-auth credentials live in the same htpasswd file used for the
       // registry — bootstrap appends an "admin" line with bcrypted password.
       lines.push("    basic_auth {");
@@ -77,6 +100,7 @@ export function generateCaddyfile(cfg: DeployConfig): string {
   // 5 GiB request body cap fits real app images comfortably (default 100MB
   // triggers 413 on the first layer push).
   lines.push(`${cfg.registry.domain} {${tlsDirective}`);
+  lines.push(...logDirective);
   lines.push("    reverse_proxy registry:5000 {");
   lines.push("        header_up Host {host}");
   lines.push("    }");
@@ -88,6 +112,17 @@ export function generateCaddyfile(cfg: DeployConfig): string {
   return lines.join("\n") + "\n";
 }
 
+/** Public hostname of the Grafana UI (`<subdomain>.<apex-of-first-env>`), or
+ *  null when observability is off / no envs exist. Shared by the Caddyfile
+ *  vhost and Grafana's GF_SERVER_ROOT_URL (compose.ts). */
+export function observabilityDomain(cfg: DeployConfig): string | null {
+  if (!cfg.observability?.enabled) return null;
+  const firstEnv = Object.values(cfg.envs)[0];
+  if (!firstEnv) return null;
+  const subdomain = cfg.observability.subdomain ?? "observability";
+  return `${subdomain}.${apexOf(firstEnv.domain)}`;
+}
+
 /** Apex of a (possibly-subdomain) host. `app.example.com` → `example.com`,
  *  `example.com` → `example.com`, `example.co.uk` → `co.uk` (approximate —
  *  good enough for the observability subdomain). */

package/src/deploy/compose.ts

@@ -1,3 +1,4 @@
+import { observabilityDomain } from "./caddyfile";
 import type { DeployConfig } from "./config";
 
 // ---------------------------------------------------------------------------
@@ -21,6 +22,17 @@ export interface ComposeOptions {
   cfg: DeployConfig;
 }
 
+/** json-file log rotation for every service — without it container logs grow
+ *  unbounded and eventually fill the disk. Alloy tails these same files via
+ *  the Docker API, and rotation is handled transparently. */
+function pushLogging(lines: string[]): void {
+  lines.push("    logging:");
+  lines.push("      driver: json-file");
+  lines.push("      options:");
+  lines.push('        max-size: "10m"');
+  lines.push('        max-file: "3"');
+}
+
 export function generateCompose({ cfg }: ComposeOptions): string {
   const lines: string[] = [];
   lines.push("# Generated by `arc platform deploy` — do not edit by hand.");
@@ -29,6 +41,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
   lines.push("  caddy:");
   lines.push("    image: caddy:2-alpine");
   lines.push("    restart: unless-stopped");
+  pushLogging(lines);
   lines.push("    ports:");
   lines.push('      - "80:80"');
   lines.push('      - "443:443"');
@@ -46,6 +59,10 @@ export function generateCompose({ cfg }: ComposeOptions): string {
   lines.push("      - caddy_config:/config");
   lines.push("    networks:");
   lines.push("      - arc-net");
+  if (cfg.observability?.enabled) {
+    lines.push("    expose:");
+    lines.push('      - "2020"  # Prometheus metrics endpoint (Caddyfile :2020 site)');
+  }
   lines.push("");
 
   // Private Docker Registry — `arc platform deploy` pushes app images here,
@@ -54,6 +71,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
   lines.push("  registry:");
   lines.push("    image: registry:2");
   lines.push("    restart: unless-stopped");
+  pushLogging(lines);
   lines.push("    volumes:");
   lines.push("      - registry_data:/var/lib/registry");
   lines.push("      - ./registry-auth/htpasswd:/auth/htpasswd:ro");
@@ -81,6 +99,15 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     );
     lines.push(`    container_name: arc-${name}`);
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    // The host server exposes GET /health (packages/host healthHandler).
+    // Image base is oven/bun:1-alpine — busybox wget is available.
+    lines.push("    healthcheck:");
+    lines.push('      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:5005/health"]');
+    lines.push("      interval: 30s");
+    lines.push("      timeout: 5s");
+    lines.push("      retries: 3");
+    lines.push("      start_period: 20s");
     if (usePostgres) {
       lines.push("    depends_on:");
       lines.push(`      arc-db-${name}:`);
@@ -149,6 +176,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push(`    image: ${image}`);
     lines.push(`    container_name: arc-db-${name}`);
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    environment:");
     lines.push("      POSTGRES_USER: arc");
     lines.push("      POSTGRES_DB: arc");
@@ -186,11 +214,17 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("    image: otel/opentelemetry-collector-contrib:0.114.0");
     lines.push("    container_name: arc-otel-collector");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    // Root needed for the hostmetrics (/hostfs) + docker_stats (docker.sock)
+    // receivers — the image's default user 10001 can read neither.
+    lines.push("    user: \"0:0\"");
     lines.push("    command: [\"--config=/etc/otelcol-contrib/config.yaml\"]");
     lines.push("    volumes:");
     lines.push(
       "      - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro",
     );
+    lines.push("      - /:/hostfs:ro                                  # hostmetrics root_path");
+    lines.push("      - /var/run/docker.sock:/var/run/docker.sock:ro  # docker_stats");
     lines.push("    networks: [arc-net]");
     lines.push("    expose:");
     lines.push("      - \"4317\"  # OTLP gRPC");
@@ -206,6 +240,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("    image: grafana/tempo:2.6.1");
     lines.push("    container_name: arc-tempo");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    command: [\"-config.file=/etc/tempo.yaml\"]");
     lines.push("    user: \"0\"  # tempo writes to /var/tempo, owned by root in the image");
     lines.push("    volumes:");
@@ -221,6 +256,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("    image: grafana/loki:3.3.2");
     lines.push("    container_name: arc-loki");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    command: [\"-config.file=/etc/loki/local-config.yaml\"]");
     lines.push("    user: \"0\"");
     lines.push("    volumes:");
@@ -236,6 +272,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("    image: prom/prometheus:v2.55.1");
     lines.push("    container_name: arc-prometheus");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    command:");
     lines.push("      - \"--config.file=/etc/prometheus/prometheus.yml\"");
     lines.push("      - \"--storage.tsdb.path=/prometheus\"");
@@ -250,11 +287,38 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("      - \"9090\"  # HTTP API + remote_write receiver");
     lines.push("");
 
+    // Grafana Alloy — tails stdout/stderr of every container (Docker API)
+    // and pushes to Loki. Complements in-app OTLP logs with infra-container
+    // output and app crash logs.
+    lines.push("  alloy:");
+    lines.push("    image: grafana/alloy:v1.16.1");
+    lines.push("    container_name: arc-alloy");
+    lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    lines.push("    user: \"0\"  # docker.sock access");
+    lines.push("    command:");
+    lines.push("      - run");
+    lines.push("      - --server.http.listen-addr=0.0.0.0:12345");
+    lines.push("      - --storage.path=/var/lib/alloy/data");
+    lines.push("      - /etc/alloy/config.alloy");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/alloy-config.alloy:/etc/alloy/config.alloy:ro");
+    lines.push("      - /var/run/docker.sock:/var/run/docker.sock:ro");
+    lines.push("      - alloy_data:/var/lib/alloy/data");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"12345\"  # Alloy self-metrics (Prom scrape)");
+    lines.push("    depends_on:");
+    lines.push("      - loki");
+    lines.push("");
+
     const adminPasswordEnv = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const grafanaDomain = observabilityDomain(cfg);
     lines.push("  grafana:");
     lines.push("    image: grafana/grafana:11.4.0");
     lines.push("    container_name: arc-grafana");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    environment:");
     lines.push("      GF_SECURITY_ADMIN_USER: admin");
     lines.push(
@@ -262,6 +326,11 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     );
     lines.push("      GF_USERS_ALLOW_SIGN_UP: \"false\"");
     lines.push("      GF_AUTH_ANONYMOUS_ENABLED: \"false\"");
+    if (grafanaDomain) {
+      // Correct absolute links in alert notifications + UI redirects behind
+      // the Caddy reverse proxy.
+      lines.push(`      GF_SERVER_ROOT_URL: "https://${grafanaDomain}"`);
+    }
     // Subpath-less — the upstream URL is served from root via Caddy reverse
     // proxy on a dedicated subdomain.
     lines.push("    volumes:");
@@ -278,6 +347,9 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push(
       "      - ./observability/grafana-dashboards:/etc/grafana/provisioning/dashboards/arc:ro",
     );
+    lines.push(
+      "      - ./observability/grafana-alerting:/etc/grafana/provisioning/alerting:ro",
+    );
     lines.push("      - grafana_data:/var/lib/grafana");
     lines.push("    networks: [arc-net]");
     lines.push("    expose:");
@@ -308,6 +380,7 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("  loki_data:");
     lines.push("  prometheus_data:");
     lines.push("  grafana_data:");
+    lines.push("  alloy_data:");
   }
 
   return lines.join("\n") + "\n";

package/src/deploy/config.ts

@@ -109,6 +109,11 @@ export interface DeployObservability {
     logs?: string;
     metrics?: string;
   };
+  /** Webhook URL for Grafana alert notifications (Slack/Discord/anything that
+   *  accepts a POST). When absent, alert rules are still provisioned but no
+   *  contact point / notification policy is created — alerts are visible in
+   *  the Grafana UI only. */
+  alertWebhookUrl?: string;
 }
 
 export interface DeployRegistry {
@@ -344,6 +349,15 @@ export function validateDeployConfig(input: unknown): DeployConfig {
         metrics: optionalString(retentionRaw, "observability.retention.metrics"),
       };
     }
+    const alertWebhookUrl = optionalString(
+      observabilityRaw,
+      "observability.alertWebhookUrl",
+    );
+    if (alertWebhookUrl !== undefined && !/^https?:\/\/.+/.test(alertWebhookUrl)) {
+      throw new Error(
+        `deploy.arc.json: observability.alertWebhookUrl must be an http(s) URL (got "${alertWebhookUrl}")`,
+      );
+    }
     validated.observability = {
       enabled: enabledRaw,
       subdomain: optionalString(observabilityRaw, "observability.subdomain") ?? "observability",
@@ -351,6 +365,7 @@ export function validateDeployConfig(input: unknown): DeployConfig {
         optionalString(observabilityRaw, "observability.adminPasswordEnv") ??
         "ARC_OBSERVABILITY_PASSWORD",
       retention,
+      alertWebhookUrl,
     };
   }