@arcote.tech/arc-cli 0.7.5 → 0.7.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcote.tech/arc-cli",
-  "version": "0.7.5",
+  "version": "0.7.7",
   "description": "CLI tool for Arc framework",
   "module": "index.ts",
   "main": "dist/index.js",
@@ -9,16 +9,29 @@
     "arc": "./dist/index.js"
   },
   "scripts": {
-    "build": "bun build --target=bun ./src/index.ts --outdir=dist --external @arcote.tech/arc --external @arcote.tech/arc-ds --external @arcote.tech/arc-react --external @arcote.tech/platform && chmod +x dist/index.js"
+    "build": "bun build --target=bun ./src/index.ts --outdir=dist --external @arcote.tech/arc --external @arcote.tech/arc-ds --external @arcote.tech/arc-react --external @arcote.tech/platform --external '@opentelemetry/*' && chmod +x dist/index.js"
   },
   "dependencies": {
-    "@arcote.tech/arc": "^0.7.5",
-    "@arcote.tech/arc-ds": "^0.7.5",
-    "@arcote.tech/arc-react": "^0.7.5",
-    "@arcote.tech/arc-host": "^0.7.5",
-    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.5",
-    "@arcote.tech/arc-adapter-db-postgres": "^0.7.5",
-    "@arcote.tech/platform": "^0.7.5",
+    "@arcote.tech/arc": "^0.7.7",
+    "@arcote.tech/arc-ds": "^0.7.7",
+    "@arcote.tech/arc-react": "^0.7.7",
+    "@arcote.tech/arc-host": "^0.7.7",
+    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.7",
+    "@arcote.tech/arc-adapter-db-postgres": "^0.7.7",
+    "@arcote.tech/arc-otel": "^0.7.7",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "^0.57.0",
+    "@opentelemetry/core": "^1.30.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.57.0",
+    "@opentelemetry/exporter-metrics-otlp-http": "^0.57.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.57.0",
+    "@opentelemetry/resources": "^1.30.0",
+    "@opentelemetry/sdk-logs": "^0.57.0",
+    "@opentelemetry/sdk-metrics": "^1.30.0",
+    "@opentelemetry/sdk-trace-base": "^1.30.0",
+    "@opentelemetry/sdk-trace-node": "^1.30.0",
+    "@opentelemetry/semantic-conventions": "^1.27.0",
+    "@arcote.tech/platform": "^0.7.7",
     "@clack/prompts": "^0.9.0",
     "commander": "^11.1.0",
     "chokidar": "^3.5.3",

package/src/builder/dependency-collector.ts

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
-import { basename, join } from "path";
+import { basename, dirname, join } from "path";
+import { fileURLToPath } from "node:url";
 import { FRAMEWORK_PEERS, FRAMEWORK_PEER_SET } from "./framework-peers";
 import type { WorkspacePackage } from "./module-builder";
 
@@ -46,6 +47,38 @@ export function collectFrameworkDeps(
   for (const { name, version } of sharedDeps) {
     versions[name] = version;
   }
+
+  // OpenTelemetry deps — included unconditionally so deployed images can
+  // resolve them at runtime when telemetry is enabled. Cheap install
+  // (~5MB) and avoids per-deploy branching on `observability.enabled`.
+  // CLI's bundle externalizes @opentelemetry/* to dodge a Bun bundling
+  // bug that mixes ES5/ES6 class semantics across these packages.
+  // Resolve arc-otel from the CLI bundle's own location: arc-otel is a
+  // dep of arc-cli, not of the consumer app, so it isn't reachable from
+  // `rootDir`. Bun installs it nested under `.bun/<arc-cli-id>/...`,
+  // visible to a resolve call rooted in arc-cli's own dist directory.
+  try {
+    const cliDir = dirname(fileURLToPath(import.meta.url));
+    const arcOtelPkgPath = Bun.resolveSync(
+      "@arcote.tech/arc-otel/package.json",
+      cliDir,
+    );
+    const arcOtelPkg = JSON.parse(readFileSync(arcOtelPkgPath, "utf-8"));
+    for (const [name, spec] of Object.entries(arcOtelPkg.dependencies ?? {})) {
+      if (name.startsWith("@opentelemetry/")) {
+        versions[name] = spec as string;
+      }
+    }
+    versions["@arcote.tech/arc-otel"] = `^${arcOtelPkg.version}`;
+  } catch (e) {
+    // arc-otel not installed alongside the CLI — older CLI release without
+    // observability support, or a bundle that stripped the dep. Log once
+    // so a misconfigured deploy doesn't silently disable telemetry libs
+    // when the operator was expecting them.
+    console.warn(
+      `[arc-otel] could not resolve @arcote.tech/arc-otel — image will run without telemetry deps: ${(e as Error).message}`,
+    );
+  }
   const manifest = {
     name: "arc-platform-framework",
     private: true,

package/src/commands/platform-deploy.ts

@@ -87,6 +87,16 @@ export async function platformDeploy(
     );
   }
 
+  // Observability stack — one Grafana password per deployment (stack-wide,
+  // not per-env). Persisted in `deploy.arc.env` (globals) so subsequent
+  // deploys reuse the same login.
+  if (cfg.observability?.enabled) {
+    const key = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    ensurePersistedSecret(ws.rootDir, "globals", key, () =>
+      crypto.randomUUID().replace(/-/g, ""),
+    );
+  }
+
   // Filter envs if an arg was provided
   const targetEnvs = envArg
     ? envArg in cfg.envs
@@ -225,5 +235,11 @@ async function hashDeployConfig(rootDir: string): Promise<string> {
   const content = readFileSync(p);
   const hasher = new Bun.CryptoHasher("sha256");
   hasher.update(content);
+  // Mix in the CLI version so a new CLI release that changes compose
+  // / caddyfile / observability templates forces upStack to re-run even
+  // when the user's deploy.arc.json is byte-identical. Without this,
+  // template changes ship in the npm package but never reach the host
+  // because the marker says "no config drift, skipping bootstrap".
+  hasher.update(readCliVersion());
   return hasher.digest("hex").slice(0, 16);
 }

package/src/deploy/bootstrap.ts

@@ -1,12 +1,13 @@
 import { spawn } from "bun";
 import { mkdirSync, writeFileSync } from "fs";
 import { tmpdir } from "os";
-import { join } from "path";
+import { dirname, join } from "path";
 import { runAnsible } from "./ansible";
 import { generateCaddyfile } from "./caddyfile";
 import { generateCompose } from "./compose";
 import type { DeployConfig } from "./config";
-import { generateHtpasswd } from "./htpasswd";
+import { generateCaddyBasicAuthLine, generateHtpasswd } from "./htpasswd";
+import { generateObservabilityConfigs } from "./observability-configs";
 import { runTerraform } from "./terraform";
 import { saveDeployConfig } from "./config";
 import { ok, log, err } from "../platform/shared";
@@ -147,6 +148,22 @@ export async function bootstrap(inputs: BootstrapInputs): Promise<void> {
     ok("Docker stack up");
   }
 
+  // Observability sidecars — always reconcile state, independent of
+  // upStack. `docker compose up -d` is idempotent: when the containers
+  // already match the current compose definition, nothing happens. When
+  // the user just turned observability on (config changed but upStack
+  // ran in a previous deploy), this brings the new services to life
+  // without forcing a full bootstrap.
+  if (cfg.observability?.enabled) {
+    log("Ensuring observability sidecars are running...");
+    const obsServices = ["otel-collector", "tempo", "loki", "prometheus", "grafana"];
+    await assertExec(
+      cfg.target,
+      `cd ${cfg.target.remoteDir} && docker compose pull --ignore-pull-failures ${obsServices.join(" ")} && docker compose up -d ${obsServices.join(" ")}`,
+    );
+    ok("Observability stack up");
+  }
+
   // Keep marker fresh
   await writeStateMarker(cfg.target, {
     cliVersion: inputs.cliVersion,
@@ -225,6 +242,32 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
   writeFileSync(join(workDir, "Caddyfile"), generateCaddyfile(cfg));
   writeFileSync(join(workDir, "docker-compose.yml"), generateCompose({ cfg }));
 
+  // Observability config files + Caddy basic-auth htpasswd for Grafana.
+  // Bundled together so a deploy without `observability.enabled` skips it
+  // all (no orphan files left on the remote).
+  let observabilityFiles: Record<string, string> | null = null;
+  let observabilityHtpasswd: string | null = null;
+  if (cfg.observability?.enabled) {
+    observabilityFiles = generateObservabilityConfigs(cfg);
+    const adminPasswordEnv =
+      cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const adminPassword = process.env[adminPasswordEnv];
+    if (!adminPassword) {
+      throw new Error(
+        `bootstrap: ${adminPasswordEnv} not set — observability needs a Grafana admin password.`,
+      );
+    }
+    // Caddy basic_auth import file uses `username bcrypt_hash` syntax
+    // (space-separated), NOT the registry-style htpasswd colon format.
+    observabilityHtpasswd = await generateCaddyBasicAuthLine("admin", adminPassword);
+    for (const [relPath, contents] of Object.entries(observabilityFiles)) {
+      const fullPath = join(workDir, relPath);
+      mkdirSync(dirname(fullPath), { recursive: true });
+      writeFileSync(fullPath, contents);
+    }
+    writeFileSync(join(workDir, "observability-htpasswd"), observabilityHtpasswd);
+  }
+
   // Ensure remoteDir exists
   await assertExec(
     cfg.target,
@@ -257,6 +300,39 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
     `${cfg.target.remoteDir}/registry-auth/htpasswd`,
   );
 
+  // Upload observability assets when enabled. Caddyfile imports
+  // `observability-htpasswd` from /etc/caddy (volume-mounted), so it has
+  // to land in the same dir as Caddyfile on the host. The collector /
+  // tempo / loki / prometheus / grafana configs go into a sibling
+  // `observability/` directory referenced by compose bind-mounts.
+  if (observabilityFiles && observabilityHtpasswd) {
+    // Make sure both the top-level `observability/` dir and the nested
+    // `grafana-dashboards/` exist before scp tries to land files there.
+    await assertExec(
+      cfg.target,
+      `mkdir -p ${cfg.target.remoteDir}/observability/grafana-dashboards`,
+    );
+    // Make sure local nested dir exists too — `generateObservabilityConfigs`
+    // returns relative paths like `observability/grafana-dashboards/x.json`
+    // which mkdirSync below uses for writeFileSync's directory.
+    for (const relPath of Object.keys(observabilityFiles)) {
+      const localDir = dirname(join(workDir, relPath));
+      mkdirSync(localDir, { recursive: true });
+    }
+    for (const relPath of Object.keys(observabilityFiles)) {
+      await scpUpload(
+        cfg.target,
+        join(workDir, relPath),
+        `${cfg.target.remoteDir}/${relPath}`,
+      );
+    }
+    await scpUpload(
+      cfg.target,
+      join(workDir, "observability-htpasswd"),
+      `${cfg.target.remoteDir}/observability-htpasswd`,
+    );
+  }
+
   // Ensure /opt/arc/.env exists so docker compose doesn't error on var
   // substitution. Per-env ARC_IMAGE_<ENV> lines are written by deploy.
   await assertExec(
@@ -264,6 +340,21 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
     `touch ${cfg.target.remoteDir}/.env`,
   );
 
+  // Propagate the Grafana admin password to /opt/arc/.env when observability
+  // is enabled. Compose interpolates `${ARC_OBSERVABILITY_PASSWORD:?}` into
+  // the grafana service's GF_SECURITY_ADMIN_PASSWORD env.
+  if (cfg.observability?.enabled) {
+    const key =
+      cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const value = process.env[key];
+    if (!value) {
+      throw new Error(
+        `bootstrap: ${key} not set — observability needs a Grafana admin password.`,
+      );
+    }
+    await writeEnvLine(cfg.target, `${cfg.target.remoteDir}/.env`, key, value);
+  }
+
   // Propagate postgres sidecar passwords to /opt/arc/.env so compose can
   // interpolate `${ARC_PG_PASSWORD_<UPPER>}` for both the pg container and
   // arc-<env>'s DATABASE_URL. Local source is `deploy.arc.<env>.env` /
@@ -307,6 +398,7 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
         .join(" ")}`,
     );
   }
+
 }
 
 /**

package/src/deploy/caddyfile.ts

@@ -30,14 +30,48 @@ export function generateCaddyfile(cfg: DeployConfig): string {
   lines.push("}");
   lines.push("");
 
-  // Public blocks — one per env
+  // Public blocks — one per env. When observability is on, add a `/otel/*`
+  // path that forwards browser-side OTLP/HTTP to the collector. Strips the
+  // /otel prefix so the collector sees the same /v1/{traces,logs,metrics}
+  // paths it would receive from same-network senders.
   for (const [name, env] of Object.entries(cfg.envs)) {
     lines.push(`${env.domain} {${tlsDirective}`);
-    lines.push(`    reverse_proxy arc-${name}:5005`);
+    if (cfg.observability?.enabled) {
+      lines.push("    handle_path /otel/* {");
+      lines.push("        reverse_proxy otel-collector:4318");
+      lines.push("    }");
+      lines.push("    handle {");
+      lines.push(`        reverse_proxy arc-${name}:5005`);
+      lines.push("    }");
+    } else {
+      lines.push(`    reverse_proxy arc-${name}:5005`);
+    }
     lines.push("}");
     lines.push("");
   }
 
+  // Observability subdomain — Grafana UI behind Caddy basic-auth. Reuses
+  // the apex of the first env's domain (e.g. observability.app.example.com
+  // when the primary env is app.example.com). Caddy issues a separate
+  // ACME certificate for this hostname.
+  if (cfg.observability?.enabled) {
+    const firstEnv = Object.values(cfg.envs)[0];
+    if (firstEnv) {
+      const subdomain = cfg.observability.subdomain ?? "observability";
+      const apex = apexOf(firstEnv.domain);
+      const observabilityDomain = `${subdomain}.${apex}`;
+      lines.push(`${observabilityDomain} {${tlsDirective}`);
+      // Basic-auth credentials live in the same htpasswd file used for the
+      // registry — bootstrap appends an "admin" line with bcrypted password.
+      lines.push("    basic_auth {");
+      lines.push("        import /etc/caddy/observability-htpasswd");
+      lines.push("    }");
+      lines.push("    reverse_proxy grafana:3000");
+      lines.push("}");
+      lines.push("");
+    }
+  }
+
   // Private Docker Registry — Caddy proxies HTTPS termination + Let's Encrypt;
   // the registry:2 container handles its own htpasswd basic auth upstream.
   // 5 GiB request body cap fits real app images comfortably (default 100MB
@@ -53,3 +87,12 @@ export function generateCaddyfile(cfg: DeployConfig): string {
 
   return lines.join("\n") + "\n";
 }
+
+/** Apex of a (possibly-subdomain) host. `app.example.com` → `example.com`,
+ *  `example.com` → `example.com`, `example.co.uk` → `co.uk` (approximate —
+ *  good enough for the observability subdomain). */
+function apexOf(host: string): string {
+  const parts = host.split(".");
+  if (parts.length <= 2) return host;
+  return parts.slice(-2).join(".");
+}

package/src/deploy/compose.ts

@@ -34,6 +34,14 @@ export function generateCompose({ cfg }: ComposeOptions): string {
   lines.push('      - "443:443"');
   lines.push("    volumes:");
   lines.push("      - ./Caddyfile:/etc/caddy/Caddyfile:ro");
+  if (cfg.observability?.enabled) {
+    // Bind-mounted in even when the file is absent (Caddy reads it lazily
+    // on observability vhost requests). Always-defined avoids compose
+    // bouncing the container when observability is later disabled.
+    lines.push(
+      "      - ./observability-htpasswd:/etc/caddy/observability-htpasswd:ro",
+    );
+  }
   lines.push("      - caddy_data:/data");
   lines.push("      - caddy_config:/config");
   lines.push("    networks:");
@@ -99,8 +107,26 @@ export function generateCompose({ cfg }: ComposeOptions): string {
         `      DATABASE_URL: "postgresql://arc:\${ARC_PG_PASSWORD_${upperName}:?missing ARC_PG_PASSWORD_${upperName}}@arc-db-${name}:5432/arc"`,
       );
     }
-    // PORT + DATABASE_URL are reserved — user envVars can't override them.
-    const reserved = new Set(["PORT", "DATABASE_URL"]);
+    if (cfg.observability?.enabled) {
+      // OTel SDK reads OTEL_EXPORTER_OTLP_ENDPOINT and stripts /v1/<signal>
+      // onto it. service.name + deployment.environment land as resource
+      // attributes, which Grafana uses for service-level filtering.
+      lines.push("      ARC_OTEL_ENABLED: \"true\"");
+      lines.push("      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318");
+      lines.push(`      OTEL_SERVICE_NAME: arc-${name}`);
+      lines.push(
+        `      OTEL_RESOURCE_ATTRIBUTES: "service.namespace=arc,deployment.environment=${name}"`,
+      );
+    }
+    // PORT + DATABASE_URL + OTEL_* are reserved — user envVars can't override them.
+    const reserved = new Set([
+      "PORT",
+      "DATABASE_URL",
+      "ARC_OTEL_ENABLED",
+      "OTEL_EXPORTER_OTLP_ENDPOINT",
+      "OTEL_SERVICE_NAME",
+      "OTEL_RESOURCE_ATTRIBUTES",
+    ]);
     for (const [k, v] of Object.entries(userEnv)) {
       if (reserved.has(k)) continue;
       lines.push(`      ${k}: ${JSON.stringify(v)}`);
@@ -143,6 +169,119 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("");
   }
 
+  // Observability sidecars — only when `observability.enabled`. Five
+  // services come up together: otel-collector receives OTLP from app
+  // containers (and from the browser via the public /otel route), fans out
+  // to Tempo/Loki/Prometheus; Grafana ties them into a single UI fronted
+  // by Caddy basic-auth on a separate subdomain.
+  if (cfg.observability?.enabled) {
+    lines.push("  otel-collector:");
+    lines.push("    image: otel/opentelemetry-collector-contrib:0.114.0");
+    lines.push("    container_name: arc-otel-collector");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"--config=/etc/otelcol-contrib/config.yaml\"]");
+    lines.push("    volumes:");
+    lines.push(
+      "      - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro",
+    );
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"4317\"  # OTLP gRPC");
+    lines.push("      - \"4318\"  # OTLP HTTP");
+    lines.push("      - \"8888\"  # collector self-metrics (Prom scrape)");
+    lines.push("    depends_on:");
+    lines.push("      - tempo");
+    lines.push("      - loki");
+    lines.push("      - prometheus");
+    lines.push("");
+
+    lines.push("  tempo:");
+    lines.push("    image: grafana/tempo:2.6.1");
+    lines.push("    container_name: arc-tempo");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"-config.file=/etc/tempo.yaml\"]");
+    lines.push("    user: \"0\"  # tempo writes to /var/tempo, owned by root in the image");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/tempo.yaml:/etc/tempo.yaml:ro");
+    lines.push("      - tempo_data:/var/tempo");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3200\"  # HTTP API for Grafana");
+    lines.push("      - \"4317\"  # OTLP from collector");
+    lines.push("");
+
+    lines.push("  loki:");
+    lines.push("    image: grafana/loki:3.3.2");
+    lines.push("    container_name: arc-loki");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"-config.file=/etc/loki/local-config.yaml\"]");
+    lines.push("    user: \"0\"");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/loki-config.yaml:/etc/loki/local-config.yaml:ro");
+    lines.push("      - loki_data:/loki");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3100\"  # HTTP API");
+    lines.push("");
+
+    const metricsRetention = cfg.observability.retention?.metrics ?? "30d";
+    lines.push("  prometheus:");
+    lines.push("    image: prom/prometheus:v2.55.1");
+    lines.push("    container_name: arc-prometheus");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command:");
+    lines.push("      - \"--config.file=/etc/prometheus/prometheus.yml\"");
+    lines.push("      - \"--storage.tsdb.path=/prometheus\"");
+    lines.push(`      - "--storage.tsdb.retention.time=${metricsRetention}"`);
+    lines.push("      - \"--web.enable-remote-write-receiver\"");
+    lines.push("      - \"--enable-feature=exemplar-storage\"");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro");
+    lines.push("      - prometheus_data:/prometheus");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"9090\"  # HTTP API + remote_write receiver");
+    lines.push("");
+
+    const adminPasswordEnv = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    lines.push("  grafana:");
+    lines.push("    image: grafana/grafana:11.4.0");
+    lines.push("    container_name: arc-grafana");
+    lines.push("    restart: unless-stopped");
+    lines.push("    environment:");
+    lines.push("      GF_SECURITY_ADMIN_USER: admin");
+    lines.push(
+      `      GF_SECURITY_ADMIN_PASSWORD: \${${adminPasswordEnv}:?missing ${adminPasswordEnv}}`,
+    );
+    lines.push("      GF_USERS_ALLOW_SIGN_UP: \"false\"");
+    lines.push("      GF_AUTH_ANONYMOUS_ENABLED: \"false\"");
+    // Subpath-less — the upstream URL is served from root via Caddy reverse
+    // proxy on a dedicated subdomain.
+    lines.push("    volumes:");
+    lines.push(
+      "      - ./observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro",
+    );
+    // Dashboard auto-provisioning: a single provider config points at the
+    // bind-mounted /arc directory; each JSON in that dir becomes a live
+    // dashboard inside the "Arc" folder. Updates land within the
+    // provider's `updateIntervalSeconds` (30s).
+    lines.push(
+      "      - ./observability/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro",
+    );
+    lines.push(
+      "      - ./observability/grafana-dashboards:/etc/grafana/provisioning/dashboards/arc:ro",
+    );
+    lines.push("      - grafana_data:/var/lib/grafana");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3000\"  # Grafana UI (behind Caddy basic-auth)");
+    lines.push("    depends_on:");
+    lines.push("      - tempo");
+    lines.push("      - loki");
+    lines.push("      - prometheus");
+    lines.push("");
+  }
+
   lines.push("networks:");
   lines.push("  arc-net:");
   lines.push("");
@@ -157,6 +296,12 @@ export function generateCompose({ cfg }: ComposeOptions): string {
       lines.push(`  arc-data-${name}:`);
     }
   }
+  if (cfg.observability?.enabled) {
+    lines.push("  tempo_data:");
+    lines.push("  loki_data:");
+    lines.push("  prometheus_data:");
+    lines.push("  grafana_data:");
+  }
 
   return lines.join("\n") + "\n";
 }

package/src/deploy/config.ts

@@ -68,6 +68,35 @@ export interface DeployProvision {
   ansible?: DeployProvisionAnsible;
 }
 
+/**
+ * Optional observability stack (otel-collector + Tempo + Loki + Prometheus
+ * + Grafana). Stack-wide — applies to every env on the same VPS, accessed
+ * through a single Grafana UI at `<subdomain>.<apex-of-app-domain>`.
+ *
+ * When `enabled: true`, every arc-<env> container gets:
+ *   - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+ *   - OTEL_SERVICE_NAME=arc-<env>
+ *   - ARC_OTEL_ENABLED=true
+ *
+ * Sidecar containers come up on the same `arc-net`. Grafana password is
+ * stored in `deploy.arc.env` under `adminPasswordEnv`; first deploy auto-
+ * generates one if missing.
+ */
+export interface DeployObservability {
+  enabled: boolean;
+  /** Subdomain under the apex of the first env's domain. Default "observability". */
+  subdomain?: string;
+  /** Env var name (in `deploy.arc.env`) that holds the Grafana admin password.
+   *  Default "ARC_OBSERVABILITY_PASSWORD". CLI auto-generates one when absent. */
+  adminPasswordEnv?: string;
+  /** Optional retention overrides. Defaults: traces 7d, logs 7d, metrics 30d. */
+  retention?: {
+    traces?: string;
+    logs?: string;
+    metrics?: string;
+  };
+}
+
 export interface DeployRegistry {
   /** Full domain — Caddy reverse-proxies this to the registry:5000 container. */
   domain: string;
@@ -83,6 +112,7 @@ export interface DeployConfig {
   caddy: DeployCaddy;
   registry: DeployRegistry;
   provision?: DeployProvision;
+  observability?: DeployObservability;
 }
 
 export const DEPLOY_CONFIG_FILE = "deploy.arc.json";
@@ -281,6 +311,31 @@ export function validateDeployConfig(input: unknown): DeployConfig {
     validated.envs[name] = { domain, envVars, db };
   }
 
+  const observabilityRaw = (input as Record<string, unknown>).observability;
+  if (observabilityRaw !== undefined) {
+    if (!isObject(observabilityRaw)) throw cfgErr("observability", "object");
+    const enabledRaw = observabilityRaw.enabled;
+    if (typeof enabledRaw !== "boolean") throw cfgErr("observability.enabled", "boolean");
+    const retentionRaw = observabilityRaw.retention;
+    let retention: DeployObservability["retention"];
+    if (retentionRaw !== undefined) {
+      if (!isObject(retentionRaw)) throw cfgErr("observability.retention", "object");
+      retention = {
+        traces: optionalString(retentionRaw, "observability.retention.traces"),
+        logs: optionalString(retentionRaw, "observability.retention.logs"),
+        metrics: optionalString(retentionRaw, "observability.retention.metrics"),
+      };
+    }
+    validated.observability = {
+      enabled: enabledRaw,
+      subdomain: optionalString(observabilityRaw, "observability.subdomain") ?? "observability",
+      adminPasswordEnv:
+        optionalString(observabilityRaw, "observability.adminPasswordEnv") ??
+        "ARC_OBSERVABILITY_PASSWORD",
+      retention,
+    };
+  }
+
   const provision = (input as Record<string, unknown>).provision;
   if (provision !== undefined) {
     if (!isObject(provision)) throw cfgErr("provision", "object");

package/src/deploy/env-file.ts

@@ -58,24 +58,30 @@ export function applyDeployGlobals(globals: Record<string, string>): void {
 }
 
 /**
- * Ensure a value exists for `key` in `deploy.arc.<envName>.env`. If absent,
- * generate one via `generate()`, append it to the file (creating the file
- * if needed), and return it. Pre-existing values (from the file or
+ * Ensure a value exists for `key` in a `deploy.arc.*.env` file. If absent,
+ * generate one via `generate()`, append it to the file (creating it if
+ * needed), and return the value. Pre-existing values (from the file or
  * process.env) are returned unchanged.
  *
- * Used to bootstrap secrets the framework owns end-to-end (e.g. the
- * Postgres sidecar password) without forcing the user to hand-edit the
- * file before the first deploy.
+ * `scope`:
+ *   - `"globals"` writes to `deploy.arc.env` (cross-env secrets like the
+ *     observability Grafana password — one stack-wide value).
+ *   - An env name writes to `deploy.arc.<envName>.env` (per-env secrets
+ *     like the Postgres sidecar password — one per deployment env).
+ *
+ * Used to bootstrap secrets the framework owns end-to-end without forcing
+ * the user to hand-edit the file before the first deploy.
  */
 export function ensurePersistedSecret(
   rootDir: string,
-  envName: string,
+  scope: "globals" | string,
   key: string,
   generate: () => string,
 ): string {
   if (process.env[key]) return process.env[key]!;
 
-  const path = join(rootDir, `deploy.arc.${envName}.env`);
+  const fileName = scope === "globals" ? "deploy.arc.env" : `deploy.arc.${scope}.env`;
+  const path = join(rootDir, fileName);
   if (existsSync(path)) {
     const existing = parseEnvFile(readFileSync(path, "utf-8"), path);
     if (existing[key]) {

package/src/deploy/htpasswd.ts

@@ -26,3 +26,23 @@ export async function generateHtpasswd(
   });
   return `${user}:${hash}\n`;
 }
+
+/**
+ * Caddy's `basic_auth` directive takes a different format than the
+ * registry-style htpasswd file: `<user> <bcrypt-hash>` (space-separated,
+ * no colon). Newer Caddy syntax also accepts a quoted hash, but the
+ * unquoted bcrypt string works in both 2.7 and 2.8+.
+ */
+export async function generateCaddyBasicAuthLine(
+  user: string,
+  password: string,
+): Promise<string> {
+  if (!user || !password) {
+    throw new Error("caddy basic_auth: user and password must both be non-empty");
+  }
+  const hash = await Bun.password.hash(password, {
+    algorithm: "bcrypt",
+    cost: 10,
+  });
+  return `${user} ${hash}\n`;
+}