@arcote.tech/arc-cli 0.7.5 → 0.7.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcote.tech/arc-cli",
-  "version": "0.7.5",
+  "version": "0.7.6",
   "description": "CLI tool for Arc framework",
   "module": "index.ts",
   "main": "dist/index.js",
@@ -12,13 +12,14 @@
     "build": "bun build --target=bun ./src/index.ts --outdir=dist --external @arcote.tech/arc --external @arcote.tech/arc-ds --external @arcote.tech/arc-react --external @arcote.tech/platform && chmod +x dist/index.js"
   },
   "dependencies": {
-    "@arcote.tech/arc": "^0.7.5",
-    "@arcote.tech/arc-ds": "^0.7.5",
-    "@arcote.tech/arc-react": "^0.7.5",
-    "@arcote.tech/arc-host": "^0.7.5",
-    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.5",
-    "@arcote.tech/arc-adapter-db-postgres": "^0.7.5",
-    "@arcote.tech/platform": "^0.7.5",
+    "@arcote.tech/arc": "^0.7.6",
+    "@arcote.tech/arc-ds": "^0.7.6",
+    "@arcote.tech/arc-react": "^0.7.6",
+    "@arcote.tech/arc-host": "^0.7.6",
+    "@arcote.tech/arc-adapter-db-sqlite": "^0.7.6",
+    "@arcote.tech/arc-adapter-db-postgres": "^0.7.6",
+    "@arcote.tech/arc-otel": "^0.7.6",
+    "@arcote.tech/platform": "^0.7.6",
     "@clack/prompts": "^0.9.0",
     "commander": "^11.1.0",
     "chokidar": "^3.5.3",

package/src/commands/platform-deploy.ts

@@ -87,6 +87,16 @@ export async function platformDeploy(
     );
   }
 
+  // Observability stack — one Grafana password per deployment (stack-wide,
+  // not per-env). Persisted in `deploy.arc.env` (globals) so subsequent
+  // deploys reuse the same login.
+  if (cfg.observability?.enabled) {
+    const key = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    ensurePersistedSecret(ws.rootDir, "globals", key, () =>
+      crypto.randomUUID().replace(/-/g, ""),
+    );
+  }
+
   // Filter envs if an arg was provided
   const targetEnvs = envArg
     ? envArg in cfg.envs

package/src/deploy/bootstrap.ts

@@ -7,6 +7,7 @@ import { generateCaddyfile } from "./caddyfile";
 import { generateCompose } from "./compose";
 import type { DeployConfig } from "./config";
 import { generateHtpasswd } from "./htpasswd";
+import { generateObservabilityConfigs } from "./observability-configs";
 import { runTerraform } from "./terraform";
 import { saveDeployConfig } from "./config";
 import { ok, log, err } from "../platform/shared";
@@ -225,6 +226,30 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
   writeFileSync(join(workDir, "Caddyfile"), generateCaddyfile(cfg));
   writeFileSync(join(workDir, "docker-compose.yml"), generateCompose({ cfg }));
 
+  // Observability config files + Caddy basic-auth htpasswd for Grafana.
+  // Bundled together so a deploy without `observability.enabled` skips it
+  // all (no orphan files left on the remote).
+  let observabilityFiles: Record<string, string> | null = null;
+  let observabilityHtpasswd: string | null = null;
+  if (cfg.observability?.enabled) {
+    observabilityFiles = generateObservabilityConfigs(cfg);
+    const adminPasswordEnv =
+      cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const adminPassword = process.env[adminPasswordEnv];
+    if (!adminPassword) {
+      throw new Error(
+        `bootstrap: ${adminPasswordEnv} not set — observability needs a Grafana admin password.`,
+      );
+    }
+    // Caddy basic_auth import file uses `username bcrypt_hash` syntax.
+    observabilityHtpasswd = await generateHtpasswd("admin", adminPassword);
+    mkdirSync(join(workDir, "observability"), { recursive: true });
+    for (const [relPath, contents] of Object.entries(observabilityFiles)) {
+      writeFileSync(join(workDir, relPath), contents);
+    }
+    writeFileSync(join(workDir, "observability-htpasswd"), observabilityHtpasswd);
+  }
+
   // Ensure remoteDir exists
   await assertExec(
     cfg.target,
@@ -257,6 +282,30 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
     `${cfg.target.remoteDir}/registry-auth/htpasswd`,
   );
 
+  // Upload observability assets when enabled. Caddyfile imports
+  // `observability-htpasswd` from /etc/caddy (volume-mounted), so it has
+  // to land in the same dir as Caddyfile on the host. The collector /
+  // tempo / loki / prometheus / grafana configs go into a sibling
+  // `observability/` directory referenced by compose bind-mounts.
+  if (observabilityFiles && observabilityHtpasswd) {
+    await assertExec(
+      cfg.target,
+      `mkdir -p ${cfg.target.remoteDir}/observability`,
+    );
+    for (const relPath of Object.keys(observabilityFiles)) {
+      await scpUpload(
+        cfg.target,
+        join(workDir, relPath),
+        `${cfg.target.remoteDir}/${relPath}`,
+      );
+    }
+    await scpUpload(
+      cfg.target,
+      join(workDir, "observability-htpasswd"),
+      `${cfg.target.remoteDir}/observability-htpasswd`,
+    );
+  }
+
   // Ensure /opt/arc/.env exists so docker compose doesn't error on var
   // substitution. Per-env ARC_IMAGE_<ENV> lines are written by deploy.
   await assertExec(
@@ -264,6 +313,21 @@ async function upStack(inputs: BootstrapInputs): Promise<void> {
     `touch ${cfg.target.remoteDir}/.env`,
   );
 
+  // Propagate the Grafana admin password to /opt/arc/.env when observability
+  // is enabled. Compose interpolates `${ARC_OBSERVABILITY_PASSWORD:?}` into
+  // the grafana service's GF_SECURITY_ADMIN_PASSWORD env.
+  if (cfg.observability?.enabled) {
+    const key =
+      cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const value = process.env[key];
+    if (!value) {
+      throw new Error(
+        `bootstrap: ${key} not set — observability needs a Grafana admin password.`,
+      );
+    }
+    await writeEnvLine(cfg.target, `${cfg.target.remoteDir}/.env`, key, value);
+  }
+
   // Propagate postgres sidecar passwords to /opt/arc/.env so compose can
   // interpolate `${ARC_PG_PASSWORD_<UPPER>}` for both the pg container and
   // arc-<env>'s DATABASE_URL. Local source is `deploy.arc.<env>.env` /

package/src/deploy/caddyfile.ts

@@ -30,14 +30,48 @@ export function generateCaddyfile(cfg: DeployConfig): string {
   lines.push("}");
   lines.push("");
 
-  // Public blocks — one per env
+  // Public blocks — one per env. When observability is on, add a `/otel/*`
+  // path that forwards browser-side OTLP/HTTP to the collector. Strips the
+  // /otel prefix so the collector sees the same /v1/{traces,logs,metrics}
+  // paths it would receive from same-network senders.
   for (const [name, env] of Object.entries(cfg.envs)) {
     lines.push(`${env.domain} {${tlsDirective}`);
-    lines.push(`    reverse_proxy arc-${name}:5005`);
+    if (cfg.observability?.enabled) {
+      lines.push("    handle_path /otel/* {");
+      lines.push("        reverse_proxy otel-collector:4318");
+      lines.push("    }");
+      lines.push("    handle {");
+      lines.push(`        reverse_proxy arc-${name}:5005`);
+      lines.push("    }");
+    } else {
+      lines.push(`    reverse_proxy arc-${name}:5005`);
+    }
     lines.push("}");
     lines.push("");
   }
 
+  // Observability subdomain — Grafana UI behind Caddy basic-auth. Reuses
+  // the apex of the first env's domain (e.g. observability.app.example.com
+  // when the primary env is app.example.com). Caddy issues a separate
+  // ACME certificate for this hostname.
+  if (cfg.observability?.enabled) {
+    const firstEnv = Object.values(cfg.envs)[0];
+    if (firstEnv) {
+      const subdomain = cfg.observability.subdomain ?? "observability";
+      const apex = apexOf(firstEnv.domain);
+      const observabilityDomain = `${subdomain}.${apex}`;
+      lines.push(`${observabilityDomain} {${tlsDirective}`);
+      // Basic-auth credentials live in the same htpasswd file used for the
+      // registry — bootstrap appends an "admin" line with bcrypted password.
+      lines.push("    basic_auth {");
+      lines.push("        import /etc/caddy/observability-htpasswd");
+      lines.push("    }");
+      lines.push("    reverse_proxy grafana:3000");
+      lines.push("}");
+      lines.push("");
+    }
+  }
+
   // Private Docker Registry — Caddy proxies HTTPS termination + Let's Encrypt;
   // the registry:2 container handles its own htpasswd basic auth upstream.
   // 5 GiB request body cap fits real app images comfortably (default 100MB
@@ -53,3 +87,12 @@ export function generateCaddyfile(cfg: DeployConfig): string {
 
   return lines.join("\n") + "\n";
 }
+
+/** Apex of a (possibly-subdomain) host. `app.example.com` → `example.com`,
+ *  `example.com` → `example.com`, `example.co.uk` → `co.uk` (approximate —
+ *  good enough for the observability subdomain). */
+function apexOf(host: string): string {
+  const parts = host.split(".");
+  if (parts.length <= 2) return host;
+  return parts.slice(-2).join(".");
+}

package/src/deploy/compose.ts

@@ -34,6 +34,14 @@ export function generateCompose({ cfg }: ComposeOptions): string {
   lines.push('      - "443:443"');
   lines.push("    volumes:");
   lines.push("      - ./Caddyfile:/etc/caddy/Caddyfile:ro");
+  if (cfg.observability?.enabled) {
+    // Bind-mounted in even when the file is absent (Caddy reads it lazily
+    // on observability vhost requests). Always-defined avoids compose
+    // bouncing the container when observability is later disabled.
+    lines.push(
+      "      - ./observability-htpasswd:/etc/caddy/observability-htpasswd:ro",
+    );
+  }
   lines.push("      - caddy_data:/data");
   lines.push("      - caddy_config:/config");
   lines.push("    networks:");
@@ -99,8 +107,26 @@ export function generateCompose({ cfg }: ComposeOptions): string {
         `      DATABASE_URL: "postgresql://arc:\${ARC_PG_PASSWORD_${upperName}:?missing ARC_PG_PASSWORD_${upperName}}@arc-db-${name}:5432/arc"`,
       );
     }
-    // PORT + DATABASE_URL are reserved — user envVars can't override them.
-    const reserved = new Set(["PORT", "DATABASE_URL"]);
+    if (cfg.observability?.enabled) {
+      // OTel SDK reads OTEL_EXPORTER_OTLP_ENDPOINT and stripts /v1/<signal>
+      // onto it. service.name + deployment.environment land as resource
+      // attributes, which Grafana uses for service-level filtering.
+      lines.push("      ARC_OTEL_ENABLED: \"true\"");
+      lines.push("      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318");
+      lines.push(`      OTEL_SERVICE_NAME: arc-${name}`);
+      lines.push(
+        `      OTEL_RESOURCE_ATTRIBUTES: "service.namespace=arc,deployment.environment=${name}"`,
+      );
+    }
+    // PORT + DATABASE_URL + OTEL_* are reserved — user envVars can't override them.
+    const reserved = new Set([
+      "PORT",
+      "DATABASE_URL",
+      "ARC_OTEL_ENABLED",
+      "OTEL_EXPORTER_OTLP_ENDPOINT",
+      "OTEL_SERVICE_NAME",
+      "OTEL_RESOURCE_ATTRIBUTES",
+    ]);
     for (const [k, v] of Object.entries(userEnv)) {
       if (reserved.has(k)) continue;
       lines.push(`      ${k}: ${JSON.stringify(v)}`);
@@ -143,6 +169,107 @@ export function generateCompose({ cfg }: ComposeOptions): string {
     lines.push("");
   }
 
+  // Observability sidecars — only when `observability.enabled`. Five
+  // services come up together: otel-collector receives OTLP from app
+  // containers (and from the browser via the public /otel route), fans out
+  // to Tempo/Loki/Prometheus; Grafana ties them into a single UI fronted
+  // by Caddy basic-auth on a separate subdomain.
+  if (cfg.observability?.enabled) {
+    lines.push("  otel-collector:");
+    lines.push("    image: otel/opentelemetry-collector-contrib:0.114.0");
+    lines.push("    container_name: arc-otel-collector");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"--config=/etc/otelcol-contrib/config.yaml\"]");
+    lines.push("    volumes:");
+    lines.push(
+      "      - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro",
+    );
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"4317\"  # OTLP gRPC");
+    lines.push("      - \"4318\"  # OTLP HTTP");
+    lines.push("      - \"8888\"  # collector self-metrics (Prom scrape)");
+    lines.push("    depends_on:");
+    lines.push("      - tempo");
+    lines.push("      - loki");
+    lines.push("      - prometheus");
+    lines.push("");
+
+    lines.push("  tempo:");
+    lines.push("    image: grafana/tempo:2.6.1");
+    lines.push("    container_name: arc-tempo");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"-config.file=/etc/tempo.yaml\"]");
+    lines.push("    user: \"0\"  # tempo writes to /var/tempo, owned by root in the image");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/tempo.yaml:/etc/tempo.yaml:ro");
+    lines.push("      - tempo_data:/var/tempo");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3200\"  # HTTP API for Grafana");
+    lines.push("      - \"4317\"  # OTLP from collector");
+    lines.push("");
+
+    lines.push("  loki:");
+    lines.push("    image: grafana/loki:3.3.2");
+    lines.push("    container_name: arc-loki");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command: [\"-config.file=/etc/loki/local-config.yaml\"]");
+    lines.push("    user: \"0\"");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/loki-config.yaml:/etc/loki/local-config.yaml:ro");
+    lines.push("      - loki_data:/loki");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3100\"  # HTTP API");
+    lines.push("");
+
+    lines.push("  prometheus:");
+    lines.push("    image: prom/prometheus:v2.55.1");
+    lines.push("    container_name: arc-prometheus");
+    lines.push("    restart: unless-stopped");
+    lines.push("    command:");
+    lines.push("      - \"--config.file=/etc/prometheus/prometheus.yml\"");
+    lines.push("      - \"--storage.tsdb.path=/prometheus\"");
+    lines.push("      - \"--web.enable-remote-write-receiver\"");
+    lines.push("      - \"--enable-feature=exemplar-storage\"");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/prometheus.yml:/etc/prometheus/prometheus.yml:ro");
+    lines.push("      - prometheus_data:/prometheus");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"9090\"  # HTTP API + remote_write receiver");
+    lines.push("");
+
+    const adminPasswordEnv = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    lines.push("  grafana:");
+    lines.push("    image: grafana/grafana:11.4.0");
+    lines.push("    container_name: arc-grafana");
+    lines.push("    restart: unless-stopped");
+    lines.push("    environment:");
+    lines.push("      GF_SECURITY_ADMIN_USER: admin");
+    lines.push(
+      `      GF_SECURITY_ADMIN_PASSWORD: \${${adminPasswordEnv}:?missing ${adminPasswordEnv}}`,
+    );
+    lines.push("      GF_USERS_ALLOW_SIGN_UP: \"false\"");
+    lines.push("      GF_AUTH_ANONYMOUS_ENABLED: \"false\"");
+    // Subpath-less — the upstream URL is served from root via Caddy reverse
+    // proxy on a dedicated subdomain.
+    lines.push("    volumes:");
+    lines.push(
+      "      - ./observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro",
+    );
+    lines.push("      - grafana_data:/var/lib/grafana");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push("      - \"3000\"  # Grafana UI (behind Caddy basic-auth)");
+    lines.push("    depends_on:");
+    lines.push("      - tempo");
+    lines.push("      - loki");
+    lines.push("      - prometheus");
+    lines.push("");
+  }
+
   lines.push("networks:");
   lines.push("  arc-net:");
   lines.push("");
@@ -157,6 +284,12 @@ export function generateCompose({ cfg }: ComposeOptions): string {
       lines.push(`  arc-data-${name}:`);
     }
   }
+  if (cfg.observability?.enabled) {
+    lines.push("  tempo_data:");
+    lines.push("  loki_data:");
+    lines.push("  prometheus_data:");
+    lines.push("  grafana_data:");
+  }
 
   return lines.join("\n") + "\n";
 }

package/src/deploy/config.ts

@@ -68,6 +68,35 @@ export interface DeployProvision {
   ansible?: DeployProvisionAnsible;
 }
 
+/**
+ * Optional observability stack (otel-collector + Tempo + Loki + Prometheus
+ * + Grafana). Stack-wide — applies to every env on the same VPS, accessed
+ * through a single Grafana UI at `<subdomain>.<apex-of-app-domain>`.
+ *
+ * When `enabled: true`, every arc-<env> container gets:
+ *   - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+ *   - OTEL_SERVICE_NAME=arc-<env>
+ *   - ARC_OTEL_ENABLED=true
+ *
+ * Sidecar containers come up on the same `arc-net`. Grafana password is
+ * stored in `deploy.arc.env` under `adminPasswordEnv`; first deploy auto-
+ * generates one if missing.
+ */
+export interface DeployObservability {
+  enabled: boolean;
+  /** Subdomain under the apex of the first env's domain. Default "observability". */
+  subdomain?: string;
+  /** Env var name (in `deploy.arc.env`) that holds the Grafana admin password.
+   *  Default "ARC_OBSERVABILITY_PASSWORD". CLI auto-generates one when absent. */
+  adminPasswordEnv?: string;
+  /** Optional retention overrides. Defaults: traces 7d, logs 7d, metrics 30d. */
+  retention?: {
+    traces?: string;
+    logs?: string;
+    metrics?: string;
+  };
+}
+
 export interface DeployRegistry {
   /** Full domain — Caddy reverse-proxies this to the registry:5000 container. */
   domain: string;
@@ -83,6 +112,7 @@ export interface DeployConfig {
   caddy: DeployCaddy;
   registry: DeployRegistry;
   provision?: DeployProvision;
+  observability?: DeployObservability;
 }
 
 export const DEPLOY_CONFIG_FILE = "deploy.arc.json";
@@ -281,6 +311,31 @@ export function validateDeployConfig(input: unknown): DeployConfig {
     validated.envs[name] = { domain, envVars, db };
   }
 
+  const observabilityRaw = (input as Record<string, unknown>).observability;
+  if (observabilityRaw !== undefined) {
+    if (!isObject(observabilityRaw)) throw cfgErr("observability", "object");
+    const enabledRaw = observabilityRaw.enabled;
+    if (typeof enabledRaw !== "boolean") throw cfgErr("observability.enabled", "boolean");
+    const retentionRaw = observabilityRaw.retention;
+    let retention: DeployObservability["retention"];
+    if (retentionRaw !== undefined) {
+      if (!isObject(retentionRaw)) throw cfgErr("observability.retention", "object");
+      retention = {
+        traces: optionalString(retentionRaw, "observability.retention.traces"),
+        logs: optionalString(retentionRaw, "observability.retention.logs"),
+        metrics: optionalString(retentionRaw, "observability.retention.metrics"),
+      };
+    }
+    validated.observability = {
+      enabled: enabledRaw,
+      subdomain: optionalString(observabilityRaw, "observability.subdomain") ?? "observability",
+      adminPasswordEnv:
+        optionalString(observabilityRaw, "observability.adminPasswordEnv") ??
+        "ARC_OBSERVABILITY_PASSWORD",
+      retention,
+    };
+  }
+
   const provision = (input as Record<string, unknown>).provision;
   if (provision !== undefined) {
     if (!isObject(provision)) throw cfgErr("provision", "object");

package/src/deploy/env-file.ts

@@ -58,24 +58,30 @@ export function applyDeployGlobals(globals: Record<string, string>): void {
 }
 
 /**
- * Ensure a value exists for `key` in `deploy.arc.<envName>.env`. If absent,
- * generate one via `generate()`, append it to the file (creating the file
- * if needed), and return it. Pre-existing values (from the file or
+ * Ensure a value exists for `key` in a `deploy.arc.*.env` file. If absent,
+ * generate one via `generate()`, append it to the file (creating it if
+ * needed), and return the value. Pre-existing values (from the file or
  * process.env) are returned unchanged.
  *
- * Used to bootstrap secrets the framework owns end-to-end (e.g. the
- * Postgres sidecar password) without forcing the user to hand-edit the
- * file before the first deploy.
+ * `scope`:
+ *   - `"globals"` writes to `deploy.arc.env` (cross-env secrets like the
+ *     observability Grafana password — one stack-wide value).
+ *   - An env name writes to `deploy.arc.<envName>.env` (per-env secrets
+ *     like the Postgres sidecar password — one per deployment env).
+ *
+ * Used to bootstrap secrets the framework owns end-to-end without forcing
+ * the user to hand-edit the file before the first deploy.
  */
 export function ensurePersistedSecret(
   rootDir: string,
-  envName: string,
+  scope: "globals" | string,
   key: string,
   generate: () => string,
 ): string {
   if (process.env[key]) return process.env[key]!;
 
-  const path = join(rootDir, `deploy.arc.${envName}.env`);
+  const fileName = scope === "globals" ? "deploy.arc.env" : `deploy.arc.${scope}.env`;
+  const path = join(rootDir, fileName);
   if (existsSync(path)) {
     const existing = parseEnvFile(readFileSync(path, "utf-8"), path);
     if (existing[key]) {