@arcote.tech/arc-cli 0.7.4 → 0.7.6

package/src/deploy/config.ts

@@ -22,11 +22,24 @@ export interface DeployTarget {
   sshKey?: string;
 }
 
+/**
+ * Database choice per environment. Omit to keep the historical default
+ * (SQLite in a named volume at `/app/.arc/data`). Selecting `postgres`
+ * makes `arc platform deploy` provision a `postgres:16-alpine` sidecar
+ * service on the docker network, persist its data in
+ * `arc-pgdata-<envName>`, and inject `DATABASE_URL` into the arc container.
+ */
+export type DeployEnvDb =
+  | { type: "sqlite" }
+  | { type: "postgres"; image?: string };
+
 export interface DeployEnv {
   /** Subdomain or full domain routed by Caddy. */
   domain: string;
   /** Extra env vars passed to the arc container. */
   envVars?: Record<string, string>;
+  /** Optional storage backend selector. Default = sqlite. */
+  db?: DeployEnvDb;
 }
 
 export interface DeployCaddy {
@@ -55,6 +68,35 @@ export interface DeployProvision {
   ansible?: DeployProvisionAnsible;
 }
 
+/**
+ * Optional observability stack (otel-collector + Tempo + Loki + Prometheus
+ * + Grafana). Stack-wide — applies to every env on the same VPS, accessed
+ * through a single Grafana UI at `<subdomain>.<apex-of-app-domain>`.
+ *
+ * When `enabled: true`, every arc-<env> container gets:
+ *   - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
+ *   - OTEL_SERVICE_NAME=arc-<env>
+ *   - ARC_OTEL_ENABLED=true
+ *
+ * Sidecar containers come up on the same `arc-net`. Grafana password is
+ * stored in `deploy.arc.env` under `adminPasswordEnv`; first deploy auto-
+ * generates one if missing.
+ */
+export interface DeployObservability {
+  enabled: boolean;
+  /** Subdomain under the apex of the first env's domain. Default "observability". */
+  subdomain?: string;
+  /** Env var name (in `deploy.arc.env`) that holds the Grafana admin password.
+   *  Default "ARC_OBSERVABILITY_PASSWORD". CLI auto-generates one when absent. */
+  adminPasswordEnv?: string;
+  /** Optional retention overrides. Defaults: traces 7d, logs 7d, metrics 30d. */
+  retention?: {
+    traces?: string;
+    logs?: string;
+    metrics?: string;
+  };
+}
+
 export interface DeployRegistry {
   /** Full domain — Caddy reverse-proxies this to the registry:5000 container. */
   domain: string;
@@ -70,6 +112,7 @@ export interface DeployConfig {
   caddy: DeployCaddy;
   registry: DeployRegistry;
   provision?: DeployProvision;
+  observability?: DeployObservability;
 }
 
 export const DEPLOY_CONFIG_FILE = "deploy.arc.json";
@@ -247,7 +290,50 @@ export function validateDeployConfig(input: unknown): DeployConfig {
         envVars[k] = v;
       }
     }
-    validated.envs[name] = { domain, envVars };
+    const dbRaw = (env as Record<string, unknown>).db;
+    let db: DeployEnvDb | undefined;
+    if (dbRaw !== undefined) {
+      if (!isObject(dbRaw)) throw cfgErr(`envs.${name}.db`, "object");
+      const dbType = requireString(dbRaw, `envs.${name}.db.type`);
+      if (dbType === "sqlite") {
+        db = { type: "sqlite" };
+      } else if (dbType === "postgres") {
+        db = {
+          type: "postgres",
+          image: optionalString(dbRaw, `envs.${name}.db.image`),
+        };
+      } else {
+        throw new Error(
+          `deploy.arc.json: envs.${name}.db.type must be "sqlite" or "postgres" (got "${dbType}")`,
+        );
+      }
+    }
+    validated.envs[name] = { domain, envVars, db };
+  }
+
+  const observabilityRaw = (input as Record<string, unknown>).observability;
+  if (observabilityRaw !== undefined) {
+    if (!isObject(observabilityRaw)) throw cfgErr("observability", "object");
+    const enabledRaw = observabilityRaw.enabled;
+    if (typeof enabledRaw !== "boolean") throw cfgErr("observability.enabled", "boolean");
+    const retentionRaw = observabilityRaw.retention;
+    let retention: DeployObservability["retention"];
+    if (retentionRaw !== undefined) {
+      if (!isObject(retentionRaw)) throw cfgErr("observability.retention", "object");
+      retention = {
+        traces: optionalString(retentionRaw, "observability.retention.traces"),
+        logs: optionalString(retentionRaw, "observability.retention.logs"),
+        metrics: optionalString(retentionRaw, "observability.retention.metrics"),
+      };
+    }
+    validated.observability = {
+      enabled: enabledRaw,
+      subdomain: optionalString(observabilityRaw, "observability.subdomain") ?? "observability",
+      adminPasswordEnv:
+        optionalString(observabilityRaw, "observability.adminPasswordEnv") ??
+        "ARC_OBSERVABILITY_PASSWORD",
+      retention,
+    };
   }
 
   const provision = (input as Record<string, unknown>).provision;

package/src/deploy/env-file.ts

@@ -1,4 +1,4 @@
-import { existsSync, readFileSync } from "fs";
+import { appendFileSync, existsSync, readFileSync } from "fs";
 import { join } from "path";
 
 // ---------------------------------------------------------------------------
@@ -57,6 +57,46 @@ export function applyDeployGlobals(globals: Record<string, string>): void {
   }
 }
 
+/**
+ * Ensure a value exists for `key` in a `deploy.arc.*.env` file. If absent,
+ * generate one via `generate()`, append it to the file (creating it if
+ * needed), and return the value. Pre-existing values (from the file or
+ * process.env) are returned unchanged.
+ *
+ * `scope`:
+ *   - `"globals"` writes to `deploy.arc.env` (cross-env secrets like the
+ *     observability Grafana password — one stack-wide value).
+ *   - An env name writes to `deploy.arc.<envName>.env` (per-env secrets
+ *     like the Postgres sidecar password — one per deployment env).
+ *
+ * Used to bootstrap secrets the framework owns end-to-end without forcing
+ * the user to hand-edit the file before the first deploy.
+ */
+export function ensurePersistedSecret(
+  rootDir: string,
+  scope: "globals" | string,
+  key: string,
+  generate: () => string,
+): string {
+  if (process.env[key]) return process.env[key]!;
+
+  const fileName = scope === "globals" ? "deploy.arc.env" : `deploy.arc.${scope}.env`;
+  const path = join(rootDir, fileName);
+  if (existsSync(path)) {
+    const existing = parseEnvFile(readFileSync(path, "utf-8"), path);
+    if (existing[key]) {
+      process.env[key] = existing[key];
+      return existing[key];
+    }
+  }
+
+  const value = generate();
+  const prefix = existsSync(path) && !readFileSync(path, "utf-8").endsWith("\n") ? "\n" : "";
+  appendFileSync(path, `${prefix}${key}=${value}\n`);
+  process.env[key] = value;
+  return value;
+}
+
 // ---------------------------------------------------------------------------
 // Parser
 // ---------------------------------------------------------------------------

package/src/deploy/observability-configs.ts

@@ -0,0 +1,293 @@
+import type { DeployConfig, DeployObservability } from "./config";
+
+// ---------------------------------------------------------------------------
+// Observability stack config templates.
+//
+// All strings are deterministic for the inputs (cfg + retention) — no random
+// IDs, no timestamps — so re-running deploy with unchanged config is a no-op
+// at the file-write level. Bootstrap diffs filesystem before bouncing
+// services, so this matters.
+//
+// Defaults:
+//   - traces:  7d  retention (Tempo block storage on local disk)
+//   - logs:    7d  retention (Loki chunks on local disk)
+//   - metrics: 30d retention (Prometheus TSDB on local disk)
+//
+// Tail sampling: every error + every span >500ms + 10% random. Decided in
+// the collector so per-service SDKs can be left at always-on without
+// flooding the backend.
+// ---------------------------------------------------------------------------
+
+const DEFAULT_RETENTION = {
+  traces: "168h", // 7d
+  logs: "168h",
+  metrics: "30d",
+} as const;
+
+function pickRetention(o: DeployObservability | undefined) {
+  return {
+    traces: o?.retention?.traces ?? DEFAULT_RETENTION.traces,
+    logs: o?.retention?.logs ?? DEFAULT_RETENTION.logs,
+    metrics: o?.retention?.metrics ?? DEFAULT_RETENTION.metrics,
+  };
+}
+
+/** OpenTelemetry Collector — receives OTLP from app containers + browser,
+ *  applies tail sampling, fans out to Tempo (traces), Loki (logs),
+ *  Prometheus remote-write (metrics). */
+export function generateOtelCollectorConfig(cfg: DeployConfig): string {
+  const envNames = Object.keys(cfg.envs);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+        cors:
+          allowed_origins:
+${envNames.map((name) => `            - "https://${cfg.envs[name]!.domain}"`).join("\n")}
+          allowed_headers:
+            - traceparent
+            - tracestate
+            - content-type
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 512
+    send_batch_max_size: 1024
+
+  # Tail-based sampling — applied after a full trace has been assembled.
+  # Errors and slow traces are kept 100%, everything else at 10%.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 50000
+    expected_new_traces_per_sec: 100
+    policies:
+      - name: errors
+        type: status_code
+        status_code: { status_codes: [ERROR] }
+      - name: slow
+        type: latency
+        latency: { threshold_ms: 500 }
+      - name: random_10pct
+        type: probabilistic
+        probabilistic: { sampling_percentage: 10 }
+
+  # Drop high-cardinality / PII attributes that might slip past app-side
+  # sanitization. Belt-and-suspenders before they hit long-term storage.
+  attributes:
+    actions:
+      - key: http.request.header.authorization
+        action: delete
+      - key: http.request.header.cookie
+        action: delete
+
+exporters:
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
+    tls:
+      insecure: true
+
+  prometheusremotewrite:
+    endpoint: http://prometheus:9090/api/v1/write
+    tls:
+      insecure: true
+
+extensions:
+  health_check: {}
+  zpages: {}
+
+service:
+  extensions: [health_check, zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [tail_sampling, attributes, batch]
+      exporters: [otlp/tempo]
+    logs:
+      receivers: [otlp]
+      processors: [attributes, batch]
+      exporters: [otlphttp/loki]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheusremotewrite]
+`;
+}
+
+/** Grafana Tempo — single-binary mode with local block storage. */
+export function generateTempoConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+server:
+  http_listen_port: 3200
+  grpc_listen_port: 9095
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 1048576
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: ${retention.traces}
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics]
+`;
+}
+
+/** Loki — single-binary mode, filesystem chunks. */
+export function generateLokiConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: ${retention.logs}
+  allow_structured_metadata: true
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+`;
+}
+
+/** Prometheus — accepts remote_write from the collector, scrapes itself. */
+export function generatePrometheusConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: [localhost:9090]
+  - job_name: otel-collector
+    static_configs:
+      - targets: [otel-collector:8888]
+
+storage:
+  tsdb:
+    retention.time: ${retention.metrics}
+
+# Note: remote-write inbound is enabled via the --web.enable-remote-write-receiver
+# command-line flag (set in docker-compose), not here.
+`;
+}
+
+/** Grafana datasource provisioning — Tempo + Loki + Prometheus, all pre-wired. */
+export function generateGrafanaDatasources(): string {
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+apiVersion: 1
+datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    uid: tempo
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: loki
+        spanStartTimeShift: -5m
+        spanEndTimeShift: 5m
+      serviceMap:
+        datasourceUid: prometheus
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: "trace_id=(\\\\w+)"
+          name: TraceID
+          url: $\${__value.raw}
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    uid: prometheus
+    isDefault: true
+`;
+}
+
+/** All config files needed on the host. Returns map of relative-path → contents
+ *  so bootstrap can write+upload them in one pass. */
+export function generateObservabilityConfigs(
+  cfg: DeployConfig,
+): Record<string, string> {
+  return {
+    "observability/otel-collector-config.yaml": generateOtelCollectorConfig(cfg),
+    "observability/tempo.yaml": generateTempoConfig(cfg),
+    "observability/loki-config.yaml": generateLokiConfig(cfg),
+    "observability/prometheus.yml": generatePrometheusConfig(cfg),
+    "observability/grafana-datasources.yaml": generateGrafanaDatasources(),
+  };
+}

package/src/platform/server.ts

@@ -11,7 +11,7 @@ import {
 import { existsSync, mkdirSync } from "fs";
 import { join } from "path";
 import { readTranslationsConfig } from "../i18n";
-import type { BuildManifest, WorkspaceInfo } from "./shared";
+import { err, ok, type BuildManifest, type WorkspaceInfo } from "./shared";
 import type { BuildManifestGroup, ModuleAccess } from "@arcote.tech/platform";
 
 // ---------------------------------------------------------------------------
@@ -52,17 +52,39 @@ export async function initContextHandler(
   context: any,
   dbPath: string,
 ): Promise<ContextHandler> {
+  const factory = await resolveDbAdapterFactory(dbPath);
+  const dbAdapter = factory(context);
+  const handler = new ContextHandler(context, dbAdapter);
+  await handler.init();
+  return handler;
+}
+
+/**
+ * Pick the database adapter factory based on runtime configuration.
+ *
+ * - When `DATABASE_URL` is set (Postgres sidecar generated by deploy, or any
+ *   external Postgres the operator wired up themselves), connect over the
+ *   network using the postgres adapter.
+ * - Otherwise fall back to the SQLite file at `dbPath`. The directory is
+ *   created on demand so first-boot in a fresh data volume just works.
+ *
+ * Both adapters share the `DBAdapterFactory` contract from arc-core, so the
+ * caller never needs to know which backend is live.
+ */
+async function resolveDbAdapterFactory(dbPath: string) {
+  const databaseUrl = process.env.DATABASE_URL;
+  if (databaseUrl && databaseUrl.length > 0) {
+    const { createPostgreSQLAdapterFactoryFromUrl } = await import(
+      "@arcote.tech/arc-adapter-db-postgres"
+    );
+    return createPostgreSQLAdapterFactoryFromUrl(databaseUrl);
+  }
   const { createBunSQLiteAdapterFactory } = await import(
     "@arcote.tech/arc-adapter-db-sqlite"
   );
-
   const dbDir = dbPath.substring(0, dbPath.lastIndexOf("/"));
   if (dbDir) mkdirSync(dbDir, { recursive: true });
-
-  const dbAdapter = createBunSQLiteAdapterFactory(dbPath)(context);
-  const handler = new ContextHandler(context, dbAdapter);
-  await handler.init();
-  return handler;
+  return createBunSQLiteAdapterFactory(dbPath);
 }
 
 // ---------------------------------------------------------------------------
@@ -75,6 +97,21 @@ export function generateShellHtml(
   initial?: { file: string; hash: string },
   stylesHash?: string,
 ): string {
+  // OpenTelemetry config — injected as a global so the browser SDK chunk
+  // (lazy-loaded by start-app.ts) can pick it up without a fetch. Endpoint
+  // is same-origin so Caddy can apply CORS + auth uniformly.
+  const otelConfig = process.env.ARC_OTEL_ENABLED === "true"
+    ? {
+        enabled: true,
+        endpoint: "/otel",
+        serviceName: process.env.OTEL_SERVICE_NAME ?? `${appName}-browser`,
+        environment: process.env.NODE_ENV === "production" ? "production" : "development",
+        sampleRate: Number(process.env.ARC_OTEL_BROWSER_SAMPLE_RATE ?? "0.1"),
+      }
+    : null;
+  const otelTag = otelConfig
+    ? `\n  <script>window.__ARC_OTEL_CONFIG=${JSON.stringify(otelConfig)};</script>`
+    : "";
   // Initial bundle carries framework, public modules, and PlatformApp re-export.
   // No importmap — single Bun.build with splitting:true inlines + dedups everything
   // across initial and per-token group bundles via auto-emitted chunk-<hash>.js.
@@ -95,7 +132,7 @@ export function generateShellHtml(
   <title>${manifest?.title ?? appName}</title>${manifest?.favicon ? `\n  <link rel="icon" href="${manifest.favicon}">` : ""}${manifest ? `\n  <link rel="manifest" href="/manifest.json">` : ""}
   <link rel="stylesheet" href="/styles.css${stylesQs}" />
   <link rel="stylesheet" href="/theme.css${stylesQs}" />
-  <link rel="modulepreload" href="${initialUrl}" />
+  <link rel="modulepreload" href="${initialUrl}" />${otelTag}
 </head>
 <body>
   <div id="root"></div>
@@ -458,6 +495,30 @@ export async function startPlatformServer(
 ): Promise<PlatformServer> {
   const { ws, port, devMode, context } = opts;
   ensureModuleSigSecret(ws, !!devMode);
+
+  // OpenTelemetry — only when explicitly enabled (deploy injects the env
+  // when `observability.enabled` is set in deploy.arc.json). Dynamic import
+  // keeps the OTel SDK out of bundles that don't use it.
+  let telemetry: import("@arcote.tech/arc-otel").ArcTelemetry | undefined;
+  let telemetryShutdown: (() => Promise<void>) | undefined;
+  if (process.env.ARC_OTEL_ENABLED === "true") {
+    try {
+      const { initServerTelemetry } = await import("@arcote.tech/arc-otel/server");
+      const init = initServerTelemetry({
+        serviceName: process.env.OTEL_SERVICE_NAME ?? ws.appName,
+        environment: "server",
+        endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+        mode: devMode ? "development" : "production",
+        sampleRate: devMode ? 1.0 : 1.0, // head-based 100%, collector tail-samples
+      });
+      telemetry = init.telemetry;
+      telemetryShutdown = init.shutdown;
+      ok("Telemetry enabled — exporting to " + process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+    } catch (e) {
+      err(`Failed to init telemetry: ${(e as Error).message}`);
+    }
+  }
+
   const moduleAccessMap = opts.moduleAccess ?? new Map();
   let manifest = opts.manifest;
   const getManifest = () => manifest;
@@ -536,17 +597,14 @@ export async function startPlatformServer(
     };
   }
 
-  // Context available — use createArcServer with platform handlers on top
-  const { createBunSQLiteAdapterFactory } = await import(
-    "@arcote.tech/arc-adapter-db-sqlite"
-  );
+  // Context available — use createArcServer with platform handlers on top.
+  // `resolveDbAdapterFactory` picks SQLite or Postgres based on DATABASE_URL.
   const dbPath = opts.dbPath || join(ws.arcDir, "data", "arc.db");
-  const dbDir = dbPath.substring(0, dbPath.lastIndexOf("/"));
-  if (dbDir) mkdirSync(dbDir, { recursive: true });
+  const dbAdapterFactory = await resolveDbAdapterFactory(dbPath);
 
   const arcServer = await createArcServer({
     context,
-    dbAdapterFactory: createBunSQLiteAdapterFactory(dbPath),
+    dbAdapterFactory,
     port,
     httpHandlers: [
       // Platform-specific handlers (checked AFTER arc handlers)
@@ -556,6 +614,7 @@ export async function startPlatformServer(
       spaFallbackHandler(getShellHtml),
     ],
     onWsClose: (clientId) => cleanupClientSubs(clientId),
+    telemetry,
   });
 
   return {
@@ -564,6 +623,9 @@ export async function startPlatformServer(
     connectionManager: arcServer.connectionManager,
     setManifest,
     notifyReload,
-    stop: () => arcServer.stop(),
+    stop: () => {
+      arcServer.stop();
+      void telemetryShutdown?.();
+    },
   };
 }