@arcote.tech/arc-cli 0.7.18 → 0.7.20

package/dist/index.js

@@ -25870,6 +25870,7 @@ import {
   ATTR_SERVICE_NAME,
   ATTR_SERVICE_VERSION
 } from "@opentelemetry/semantic-conventions/incubating";
+import { format as format2 } from "util";
 import {
   context,
   propagation,
@@ -25880,6 +25881,41 @@ import {
   logs,
   SeverityNumber
 } from "@opentelemetry/api-logs";
+function patchConsole(telemetry) {
+  if (patched || !telemetry.active)
+    return () => {};
+  patched = true;
+  const originals = [];
+  for (const [method, level] of METHODS) {
+    const original = console[method].bind(console);
+    originals.push([method, original]);
+    console[method] = (...args) => {
+      original(...args);
+      if (emitting)
+        return;
+      emitting = true;
+      try {
+        const body = format2(...args);
+        if (body.startsWith("[arc-otel]"))
+          return;
+        const error = args.find((arg) => arg instanceof Error);
+        telemetry.log(level, body, error ? {
+          "exception.type": error.name,
+          "exception.message": error.message,
+          "exception.stacktrace": error.stack ?? ""
+        } : {});
+      } catch {} finally {
+        emitting = false;
+      }
+    };
+  }
+  return () => {
+    for (const [method, original] of originals) {
+      console[method] = original;
+    }
+    patched = false;
+  };
+}
 function sanitizeAttrs(input, opts = {}) {
   if (!input)
     return {};
@@ -25939,6 +25975,7 @@ class ArcTelemetry {
   meter = null;
   histograms = new Map;
   counters = new Map;
+  upDownCounters = new Map;
   constructor(config) {
     const mode = config.mode ?? "development";
     const enabled = config.enabled ?? mode !== "disabled";
@@ -26050,6 +26087,18 @@ class ArcTelemetry {
       counter.add(value, attrs);
     } catch {}
   }
+  addUpDown(name, delta, attrs = {}) {
+    if (!this.active || !this.meter)
+      return;
+    let counter = this.upDownCounters.get(name);
+    if (!counter) {
+      counter = this.meter.createUpDownCounter(name);
+      this.upDownCounters.set(name, counter);
+    }
+    try {
+      counter.add(delta, attrs);
+    } catch {}
+  }
   recordHistogram(name, value, attrs = {}) {
     if (!this.active || !this.meter)
       return;
@@ -26097,53 +26146,31 @@ function noopSpan() {
 function wrapDbAdapter(adapter, telemetry, dbSystem) {
   if (!telemetry || !telemetry.active)
     return adapter;
+  const dbAttrs = (operation, store) => ({
+    "db.system": dbSystem,
+    "db.operation.name": operation,
+    ...store ? { "db.collection.name": store } : {}
+  });
+  const measureOp = async (operation, store, fn) => {
+    const start = Date.now();
+    try {
+      return await fn();
+    } finally {
+      telemetry.measureSince("arc.db.operation.duration", start, dbAttrs(operation, store));
+    }
+  };
   const wrapRead = (tx) => ({
-    find: async (store, options) => telemetry.startSpan(`db.find ${store}`, async (span) => {
-      const start = Date.now();
-      try {
-        const rows = await tx.find(store, options);
-        span.setAttribute("db.response.row_count", rows.length);
-        return rows;
-      } finally {
-        telemetry.measureSince("arc.db.find_ms", start, {
-          "db.system": dbSystem,
-          "db.collection.name": store
-        });
-      }
-    }, {
-      kind: 3,
-      attributes: {
-        "db.system": dbSystem,
-        "db.operation.name": "find",
-        "db.collection.name": store
-      }
-    })
+    find: async (store, options) => telemetry.startSpan(`db.find ${store}`, async (span) => measureOp("find", store, async () => {
+      const rows = await tx.find(store, options);
+      span.setAttribute("db.response.row_count", rows.length);
+      return rows;
+    }), { kind: 3, attributes: dbAttrs("find", store) })
   });
   const wrapReadWrite = (tx) => ({
     ...wrapRead(tx),
-    set: async (store, data) => telemetry.startSpan(`db.set ${store}`, () => tx.set(store, data), {
-      kind: 3,
-      attributes: {
-        "db.system": dbSystem,
-        "db.operation.name": "set",
-        "db.collection.name": store
-      }
-    }),
-    remove: async (store, id3) => telemetry.startSpan(`db.remove ${store}`, () => tx.remove(store, id3), {
-      kind: 3,
-      attributes: {
-        "db.system": dbSystem,
-        "db.operation.name": "remove",
-        "db.collection.name": store
-      }
-    }),
-    commit: async () => telemetry.startSpan("db.commit", () => tx.commit(), {
-      kind: 3,
-      attributes: {
-        "db.system": dbSystem,
-        "db.operation.name": "commit"
-      }
-    })
+    set: async (store, data) => telemetry.startSpan(`db.set ${store}`, () => measureOp("set", store, () => tx.set(store, data)), { kind: 3, attributes: dbAttrs("set", store) }),
+    remove: async (store, id3) => telemetry.startSpan(`db.remove ${store}`, () => measureOp("remove", store, () => tx.remove(store, id3)), { kind: 3, attributes: dbAttrs("remove", store) }),
+    commit: async () => telemetry.startSpan("db.commit", () => measureOp("commit", undefined, () => tx.commit()), { kind: 3, attributes: dbAttrs("commit") })
   });
   return new Proxy(adapter, {
     get(target, prop) {
@@ -26206,6 +26233,7 @@ function initServerTelemetry(config) {
     logger: loggerProvider.getLogger(config.serviceName),
     meter: meterProvider.getMeter(config.serviceName)
   });
+  const restoreConsole = config.patchConsole !== false ? patchConsole(telemetry) : () => {};
   if (telemetry.config.debug) {
     console.log("[arc-otel] server init", {
       serviceName: config.serviceName,
@@ -26216,6 +26244,7 @@ function initServerTelemetry(config) {
     });
   }
   const shutdown = async () => {
+    restoreConsole();
     try {
       await Promise.all([
         tracerProvider.shutdown(),
@@ -26228,8 +26257,15 @@ function initServerTelemetry(config) {
   };
   return { telemetry, shutdown };
 }
-var DEFAULT_REDACT_KEY_PATTERN, DEFAULT_MAX_STRING_LEN = 2048, DEFAULT_MAX_JSON_LEN = 4096;
+var METHODS, patched = false, emitting = false, DEFAULT_REDACT_KEY_PATTERN, DEFAULT_MAX_STRING_LEN = 2048, DEFAULT_MAX_JSON_LEN = 4096;
 var init_init_server = __esm(() => {
+  METHODS = [
+    ["debug", "debug"],
+    ["log", "info"],
+    ["info", "info"],
+    ["warn", "warn"],
+    ["error", "error"]
+  ];
   DEFAULT_REDACT_KEY_PATTERN = /(password|passwd|token|secret|authorization|jwt|api[_-]?key|cookie|email|credit[_-]?card|ssn)/i;
 });
 
@@ -36054,18 +36090,32 @@ function generateCaddyfile(cfg) {
     email ${cfg.caddy.email}`;
   const tlsDirective = cfg.caddy.email === "internal" ? `
     tls internal` : "";
+  const observability = cfg.observability?.enabled === true;
+  const logDirective = observability ? ["    log {", "        output stdout", "        format json", "    }"] : [];
   const lines = [];
   lines.push("# Generated by `arc platform deploy` \u2014 do not edit by hand.");
   lines.push("");
   lines.push("{");
   lines.push("    admin off");
+  if (observability) {
+    lines.push("    metrics {");
+    lines.push("        per_host");
+    lines.push("    }");
+  }
   if (email)
     lines.push(`   ${email.trim()}`);
   lines.push("}");
   lines.push("");
+  if (observability) {
+    lines.push(":2020 {");
+    lines.push("    metrics");
+    lines.push("}");
+    lines.push("");
+  }
   for (const [name, env2] of Object.entries(cfg.envs)) {
     lines.push(`${env2.domain} {${tlsDirective}`);
-    if (cfg.observability?.enabled) {
+    lines.push(...logDirective);
+    if (observability) {
       lines.push("    handle_path /otel/* {");
       lines.push("        reverse_proxy otel-collector:4318");
       lines.push("    }");
@@ -36078,13 +36128,11 @@ function generateCaddyfile(cfg) {
     lines.push("}");
     lines.push("");
   }
-  if (cfg.observability?.enabled) {
-    const firstEnv = Object.values(cfg.envs)[0];
-    if (firstEnv) {
-      const subdomain = cfg.observability.subdomain ?? "observability";
-      const apex = apexOf(firstEnv.domain);
-      const observabilityDomain = `${subdomain}.${apex}`;
-      lines.push(`${observabilityDomain} {${tlsDirective}`);
+  if (observability) {
+    const domain = observabilityDomain(cfg);
+    if (domain) {
+      lines.push(`${domain} {${tlsDirective}`);
+      lines.push(...logDirective);
       lines.push("    basic_auth {");
       lines.push("        import /etc/caddy/observability-htpasswd");
       lines.push("    }");
@@ -36094,6 +36142,7 @@ function generateCaddyfile(cfg) {
     }
   }
   lines.push(`${cfg.registry.domain} {${tlsDirective}`);
+  lines.push(...logDirective);
   lines.push("    reverse_proxy registry:5000 {");
   lines.push("        header_up Host {host}");
   lines.push("    }");
@@ -36105,6 +36154,15 @@ function generateCaddyfile(cfg) {
 `) + `
 `;
 }
+function observabilityDomain(cfg) {
+  if (!cfg.observability?.enabled)
+    return null;
+  const firstEnv = Object.values(cfg.envs)[0];
+  if (!firstEnv)
+    return null;
+  const subdomain = cfg.observability.subdomain ?? "observability";
+  return `${subdomain}.${apexOf(firstEnv.domain)}`;
+}
 function apexOf(host) {
   const parts = host.split(".");
   if (parts.length <= 2)
@@ -36113,6 +36171,13 @@ function apexOf(host) {
 }
 
 // src/deploy/compose.ts
+function pushLogging(lines) {
+  lines.push("    logging:");
+  lines.push("      driver: json-file");
+  lines.push("      options:");
+  lines.push('        max-size: "10m"');
+  lines.push('        max-file: "3"');
+}
 function generateCompose({ cfg }) {
   const lines = [];
   lines.push("# Generated by `arc platform deploy` \u2014 do not edit by hand.");
@@ -36121,6 +36186,7 @@ function generateCompose({ cfg }) {
   lines.push("  caddy:");
   lines.push("    image: caddy:2-alpine");
   lines.push("    restart: unless-stopped");
+  pushLogging(lines);
   lines.push("    ports:");
   lines.push('      - "80:80"');
   lines.push('      - "443:443"');
@@ -36133,10 +36199,15 @@ function generateCompose({ cfg }) {
   lines.push("      - caddy_config:/config");
   lines.push("    networks:");
   lines.push("      - arc-net");
+  if (cfg.observability?.enabled) {
+    lines.push("    expose:");
+    lines.push('      - "2020"  # Prometheus metrics endpoint (Caddyfile :2020 site)');
+  }
   lines.push("");
   lines.push("  registry:");
   lines.push("    image: registry:2");
   lines.push("    restart: unless-stopped");
+  pushLogging(lines);
   lines.push("    volumes:");
   lines.push("      - registry_data:/var/lib/registry");
   lines.push("      - ./registry-auth/htpasswd:/auth/htpasswd:ro");
@@ -36157,6 +36228,13 @@ function generateCompose({ cfg }) {
     lines.push(`    image: \${ARC_IMAGE_${upperName}:-arc-${name}:not-deployed}`);
     lines.push(`    container_name: arc-${name}`);
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    lines.push("    healthcheck:");
+    lines.push('      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:5005/health"]');
+    lines.push("      interval: 30s");
+    lines.push("      timeout: 5s");
+    lines.push("      retries: 3");
+    lines.push("      start_period: 20s");
     if (usePostgres) {
       lines.push("    depends_on:");
       lines.push(`      arc-db-${name}:`);
@@ -36209,6 +36287,7 @@ function generateCompose({ cfg }) {
     lines.push(`    image: ${image2}`);
     lines.push(`    container_name: arc-db-${name}`);
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    environment:");
     lines.push("      POSTGRES_USER: arc");
     lines.push("      POSTGRES_DB: arc");
@@ -36233,9 +36312,13 @@ function generateCompose({ cfg }) {
     lines.push("    image: otel/opentelemetry-collector-contrib:0.114.0");
     lines.push("    container_name: arc-otel-collector");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    lines.push('    user: "0:0"');
     lines.push('    command: ["--config=/etc/otelcol-contrib/config.yaml"]');
     lines.push("    volumes:");
     lines.push("      - ./observability/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml:ro");
+    lines.push("      - /:/hostfs:ro                                  # hostmetrics root_path");
+    lines.push("      - /var/run/docker.sock:/var/run/docker.sock:ro  # docker_stats");
     lines.push("    networks: [arc-net]");
     lines.push("    expose:");
     lines.push('      - "4317"  # OTLP gRPC');
@@ -36250,6 +36333,7 @@ function generateCompose({ cfg }) {
     lines.push("    image: grafana/tempo:2.6.1");
     lines.push("    container_name: arc-tempo");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push('    command: ["-config.file=/etc/tempo.yaml"]');
     lines.push('    user: "0"  # tempo writes to /var/tempo, owned by root in the image');
     lines.push("    volumes:");
@@ -36264,6 +36348,7 @@ function generateCompose({ cfg }) {
     lines.push("    image: grafana/loki:3.3.2");
     lines.push("    container_name: arc-loki");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push('    command: ["-config.file=/etc/loki/local-config.yaml"]');
     lines.push('    user: "0"');
     lines.push("    volumes:");
@@ -36278,6 +36363,7 @@ function generateCompose({ cfg }) {
     lines.push("    image: prom/prometheus:v2.55.1");
     lines.push("    container_name: arc-prometheus");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    command:");
     lines.push('      - "--config.file=/etc/prometheus/prometheus.yml"');
     lines.push('      - "--storage.tsdb.path=/prometheus"');
@@ -36291,20 +36377,47 @@ function generateCompose({ cfg }) {
     lines.push("    expose:");
     lines.push('      - "9090"  # HTTP API + remote_write receiver');
     lines.push("");
+    lines.push("  alloy:");
+    lines.push("    image: grafana/alloy:v1.16.1");
+    lines.push("    container_name: arc-alloy");
+    lines.push("    restart: unless-stopped");
+    pushLogging(lines);
+    lines.push('    user: "0"  # docker.sock access');
+    lines.push("    command:");
+    lines.push("      - run");
+    lines.push("      - --server.http.listen-addr=0.0.0.0:12345");
+    lines.push("      - --storage.path=/var/lib/alloy/data");
+    lines.push("      - /etc/alloy/config.alloy");
+    lines.push("    volumes:");
+    lines.push("      - ./observability/alloy-config.alloy:/etc/alloy/config.alloy:ro");
+    lines.push("      - /var/run/docker.sock:/var/run/docker.sock:ro");
+    lines.push("      - alloy_data:/var/lib/alloy/data");
+    lines.push("    networks: [arc-net]");
+    lines.push("    expose:");
+    lines.push('      - "12345"  # Alloy self-metrics (Prom scrape)');
+    lines.push("    depends_on:");
+    lines.push("      - loki");
+    lines.push("");
     const adminPasswordEnv = cfg.observability.adminPasswordEnv ?? "ARC_OBSERVABILITY_PASSWORD";
+    const grafanaDomain = observabilityDomain(cfg);
     lines.push("  grafana:");
     lines.push("    image: grafana/grafana:11.4.0");
     lines.push("    container_name: arc-grafana");
     lines.push("    restart: unless-stopped");
+    pushLogging(lines);
     lines.push("    environment:");
     lines.push("      GF_SECURITY_ADMIN_USER: admin");
     lines.push(`      GF_SECURITY_ADMIN_PASSWORD: \${${adminPasswordEnv}:?missing ${adminPasswordEnv}}`);
     lines.push('      GF_USERS_ALLOW_SIGN_UP: "false"');
     lines.push('      GF_AUTH_ANONYMOUS_ENABLED: "false"');
+    if (grafanaDomain) {
+      lines.push(`      GF_SERVER_ROOT_URL: "https://${grafanaDomain}"`);
+    }
     lines.push("    volumes:");
     lines.push("      - ./observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro");
     lines.push("      - ./observability/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro");
     lines.push("      - ./observability/grafana-dashboards:/etc/grafana/provisioning/dashboards/arc:ro");
+    lines.push("      - ./observability/grafana-alerting:/etc/grafana/provisioning/alerting:ro");
     lines.push("      - grafana_data:/var/lib/grafana");
     lines.push("    networks: [arc-net]");
     lines.push("    expose:");
@@ -36334,6 +36447,7 @@ function generateCompose({ cfg }) {
     lines.push("  loki_data:");
     lines.push("  prometheus_data:");
     lines.push("  grafana_data:");
+    lines.push("  alloy_data:");
   }
   return lines.join(`
 `) + `
@@ -36408,6 +36522,64 @@ ${envNames.map((name) => `            - "https://${cfg.envs[name].domain}"`).joi
             - tracestate
             - content-type
 
+  # Host-level CPU / memory / load / disk / filesystem / network metrics.
+  # The host root is bind-mounted read-only at /hostfs (see compose).
+  hostmetrics:
+    collection_interval: 30s
+    root_path: /hostfs
+    scrapers:
+      cpu:
+        metrics:
+          system.cpu.utilization:
+            enabled: true
+      memory:
+        metrics:
+          system.memory.utilization:
+            enabled: true
+      load: {}
+      disk: {}
+      filesystem:
+        metrics:
+          system.filesystem.utilization:
+            enabled: true
+        exclude_fs_types:
+          fs_types: [autofs, binfmt_misc, bpf, cgroup2, configfs, debugfs, devpts, devtmpfs, fusectl, hugetlbfs, iso9660, mqueue, nsfs, overlay, proc, procfs, pstore, rpc_pipefs, securityfs, selinuxfs, squashfs, sysfs, tracefs, tmpfs]
+          match_type: strict
+        exclude_mount_points:
+          mount_points: ["/var/lib/docker/.*", "/run/.*", "/snap/.*", "/boot/.*"]
+          match_type: regexp
+      network: {}
+      paging: {}
+
+  # Per-container CPU / memory / network / block-IO + restarts straight from
+  # the Docker daemon (socket bind-mounted read-only, see compose).
+  docker_stats:
+    endpoint: unix:///var/run/docker.sock
+    collection_interval: 30s
+    metrics:
+      container.restarts:
+        enabled: true
+      container.uptime:
+        enabled: true
+
+connectors:
+  # Span\u2192metrics computed from 100% of spans (pipeline runs BEFORE tail
+  # sampling) \u2014 lowering the sampling policy later never skews dashboards.
+  spanmetrics:
+    histogram:
+      unit: ms
+      explicit:
+        buckets: [2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 5s, 10s]
+    metrics_flush_interval: 15s
+  # Emits traces_service_graph_* (same metric names Tempo's generator would).
+  servicegraph:
+    metrics_flush_interval: 15s
+    store:
+      ttl: 5s
+      max_items: 5000
+  # Joins the raw-trace pipeline to the sampled-storage pipeline.
+  forward: {}
+
 processors:
   batch:
     timeout: 5s
@@ -36418,7 +36590,8 @@ processors:
   # Errors + slow traces zachowywane w 100%, normalne traces r\xF3wnie\u017C 100%
   # przy obecnej skali (boostrap produkcji). Tail sampling matchuje OR po
   # policies \u2014 bez "always" policy WSZYSTKIE OK traces by\u0142yby droppowane.
-  # Obni\u017C 'random_100pct' do np. 10% gdy ruch eksploduje.
+  # Obni\u017C 'random_100pct' do np. 10% gdy ruch eksploduje \u2014 span-metrics s\u0105
+  # liczone przed samplingiem, wi\u0119c dashboardy pozostan\u0105 dok\u0142adne.
   tail_sampling:
     decision_wait: 10s
     num_traces: 50000
@@ -36443,6 +36616,34 @@ processors:
       - key: http.request.header.cookie
         action: delete
 
+  # Cardinality guard for span-metrics: the SPA fallback answers EVERY path,
+  # so raw span names (one per bot-scanned URL) would explode Prometheus
+  # series. Static assets collapse to "<METHOD> static", /route/* to
+  # "<METHOD> /route", anything else outside the known API surface to
+  # "<METHOD> other". Runs BEFORE the spanmetrics connector. Note: "$$" is a
+  # literal "$" (collector env expansion), RE2 has no lookahead \u2192 IsMatch+not.
+  transform/span_names:
+    error_mode: ignore
+    trace_statements:
+      - context: span
+        statements:
+          - set(name, Concat([attributes["http.request.method"], "static"], " ")) where IsMatch(name, "^[A-Z]+ /.*\\\\.(js|mjs|css|map|ico|png|jpe?g|svg|gif|webp|avif|woff2?|ttf|otf|txt|xml|json|webmanifest)$$")
+          - replace_pattern(name, "^([A-Z]+) /route(/.*)?$$", "$$1 /route")
+          - set(name, Concat([attributes["http.request.method"], "other"], " ")) where IsMatch(name, "^[A-Z]+ /") and not IsMatch(name, "^[A-Z]+ ((/api|/command|/query)(/.*)?|/route|/ws|/health|/otel(/.*)?|/)$$")
+
+  # Stable service.name for infra metric streams (becomes the service_name
+  # label after resource_to_telemetry_conversion).
+  resource/host:
+    attributes:
+      - key: service.name
+        value: arc-host
+        action: upsert
+  resource/docker:
+    attributes:
+      - key: service.name
+        value: arc-docker
+        action: upsert
+
 exporters:
   otlp/tempo:
     endpoint: tempo:4317
@@ -36458,6 +36659,10 @@ exporters:
     endpoint: http://prometheus:9090/api/v1/write
     tls:
       insecure: true
+    # Copy resource attributes (service.name, deployment.environment, \u2026)
+    # onto every series \u2014 dashboards filter by service_name.
+    resource_to_telemetry_conversion:
+      enabled: true
 
 extensions:
   health_check: {}
@@ -36465,19 +36670,42 @@ extensions:
 
 service:
   extensions: [health_check, zpages]
+  # Collector self-metrics. Since 0.111 the default bind is localhost only \u2014
+  # Prometheus scrapes otel-collector:8888, so listen on all interfaces.
+  telemetry:
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: "0.0.0.0"
+                port: 8888
   pipelines:
-    traces:
+    traces/in:
       receivers: [otlp]
-      processors: [tail_sampling, attributes, batch]
+      processors: [attributes, transform/span_names]
+      exporters: [spanmetrics, servicegraph, forward]
+    traces/sampled:
+      receivers: [forward]
+      processors: [tail_sampling, batch]
       exporters: [otlp/tempo]
     logs:
       receivers: [otlp]
       processors: [attributes, batch]
       exporters: [otlphttp/loki]
     metrics:
-      receivers: [otlp]
+      receivers: [otlp, spanmetrics, servicegraph]
       processors: [batch]
       exporters: [prometheusremotewrite]
+    metrics/host:
+      receivers: [hostmetrics]
+      processors: [resource/host, batch]
+      exporters: [prometheusremotewrite]
+    metrics/docker:
+      receivers: [docker_stats]
+      processors: [resource/docker, batch]
+      exporters: [prometheusremotewrite]
 `;
 }
 function generateTempoConfig(cfg) {
@@ -36513,20 +36741,9 @@ storage:
     wal:
       path: /var/tempo/wal
 
-metrics_generator:
-  registry:
-    external_labels:
-      source: tempo
-  storage:
-    path: /var/tempo/generator/wal
-    remote_write:
-      - url: http://prometheus:9090/api/v1/write
-        send_exemplars: true
-
-overrides:
-  defaults:
-    metrics_generator:
-      processors: [service-graphs, span-metrics]
+# NOTE: no metrics_generator \u2014 span-metrics + service-graph are produced by
+# the otel-collector connectors BEFORE tail sampling (accurate rates even
+# when sampling is later tightened) and remote-written to Prometheus there.
 `;
 }
 function generateLokiConfig(cfg) {
@@ -36582,11 +36799,205 @@ scrape_configs:
   - job_name: otel-collector
     static_configs:
       - targets: [otel-collector:8888]
+  - job_name: caddy
+    static_configs:
+      - targets: [caddy:2020]
+  - job_name: loki
+    static_configs:
+      - targets: [loki:3100]
+  - job_name: tempo
+    static_configs:
+      - targets: [tempo:3200]
+  - job_name: grafana
+    static_configs:
+      - targets: [grafana:3000]
+  - job_name: alloy
+    static_configs:
+      - targets: [alloy:12345]
 
 # remote-write inbound is enabled via the --web.enable-remote-write-receiver
 # command-line flag (compose.ts). Retention via --storage.tsdb.retention.time.
 `;
 }
+function generateAlloyConfig() {
+  return `// Generated by \`arc platform deploy\` \u2014 do not edit by hand.
+discovery.docker "containers" {
+  host             = "unix:///var/run/docker.sock"
+  refresh_interval = "15s"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+    target_label  = "compose_service"
+  }
+}
+
+loki.source.docker "containers" {
+  host          = "unix:///var/run/docker.sock"
+  targets       = discovery.docker.containers.targets
+  relabel_rules = discovery.relabel.containers.rules
+  labels        = { source = "docker" }
+  forward_to    = [loki.write.loki.receiver]
+}
+
+loki.write "loki" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}
+`;
+}
+function generateGrafanaAlerting(cfg) {
+  const webhookUrl = cfg.observability?.alertWebhookUrl;
+  const rules = [
+    {
+      uid: "arc-high-error-rate",
+      title: "High server error rate (>5%)",
+      expr: 'sum(rate(traces_span_metrics_calls_total{span_kind="SPAN_KIND_SERVER", status_code="STATUS_CODE_ERROR"}[5m])) / clamp_min(sum(rate(traces_span_metrics_calls_total{span_kind="SPAN_KIND_SERVER"}[5m])), 0.001)',
+      threshold: 0.05,
+      pendingFor: "5m",
+      summary: "More than 5% of server spans are errors over the last 5 minutes."
+    },
+    {
+      uid: "arc-high-latency-p95",
+      title: "High p95 latency (>1s)",
+      expr: 'histogram_quantile(0.95, sum by (le) (rate(traces_span_metrics_duration_milliseconds_bucket{span_kind="SPAN_KIND_SERVER"}[5m])))',
+      threshold: 1000,
+      pendingFor: "10m",
+      summary: "Server p95 latency above 1s for 10 minutes."
+    },
+    {
+      uid: "arc-host-disk-high",
+      title: "Host disk usage >85%",
+      expr: 'max by (mountpoint) (sum by (device, mountpoint) (system_filesystem_usage_bytes{state="used"}) / sum by (device, mountpoint) (system_filesystem_usage_bytes))',
+      threshold: 0.85,
+      pendingFor: "15m",
+      summary: "A host filesystem is more than 85% full."
+    },
+    {
+      uid: "arc-host-memory-high",
+      title: "Host memory usage >90%",
+      expr: 'sum(system_memory_usage_bytes{state="used"}) / sum(system_memory_usage_bytes)',
+      threshold: 0.9,
+      pendingFor: "10m",
+      summary: "Host memory usage above 90% for 10 minutes."
+    },
+    {
+      uid: "arc-container-restarts",
+      title: "Container restarted",
+      expr: "sum by (container_name) (increase(container_restarts_total[15m]))",
+      threshold: 0,
+      pendingFor: "0s",
+      summary: "A container restarted within the last 15 minutes."
+    },
+    {
+      uid: "arc-app-silent",
+      title: "App stopped reporting metrics",
+      expr: "absent(arc_commands_total)",
+      threshold: 0,
+      pendingFor: "10m",
+      summary: "No arc_commands_total series for 10 minutes \u2014 app down or telemetry broken."
+    },
+    {
+      uid: "arc-collector-export-failures",
+      title: "Telemetry export failures",
+      expr: 'sum(increase({__name__=~"otelcol_exporter_send_failed_(spans|metric_points|log_records)"}[15m]))',
+      threshold: 0,
+      pendingFor: "0s",
+      summary: "The otel-collector failed to export telemetry within the last 15 minutes."
+    },
+    {
+      uid: "arc-target-down",
+      title: "Scrape target down",
+      expr: "min(up)",
+      threshold: 1,
+      op: "lt",
+      pendingFor: "5m",
+      summary: "A Prometheus scrape target has been down for 5 minutes."
+    }
+  ];
+  const ruleYaml = rules.map((rule) => {
+    const op = rule.op ?? "gt";
+    return `      - uid: ${rule.uid}
+        title: ${JSON.stringify(rule.title)}
+        condition: C
+        for: ${rule.pendingFor}
+        noDataState: OK
+        execErrState: OK
+        annotations:
+          summary: ${JSON.stringify(rule.summary)}
+        labels:
+          source: arc
+        data:
+          - refId: A
+            relativeTimeRange: { from: 600, to: 0 }
+            datasourceUid: prometheus
+            model:
+              expr: ${JSON.stringify(rule.expr)}
+              instant: true
+              intervalMs: 1000
+              maxDataPoints: 43200
+              refId: A
+          - refId: B
+            relativeTimeRange: { from: 0, to: 0 }
+            datasourceUid: __expr__
+            model:
+              type: reduce
+              expression: A
+              reducer: last
+              refId: B
+          - refId: C
+            relativeTimeRange: { from: 0, to: 0 }
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: B
+              refId: C
+              conditions:
+                - evaluator:
+                    type: ${op}
+                    params: [${rule.threshold}]`;
+  }).join(`
+`);
+  const contactSection = webhookUrl ? `
+contactPoints:
+  - orgId: 1
+    name: arc-webhook
+    receivers:
+      - uid: arc-webhook
+        type: webhook
+        settings:
+          url: ${JSON.stringify(webhookUrl)}
+          httpMethod: POST
+
+policies:
+  - orgId: 1
+    receiver: arc-webhook
+    group_by: ["grafana_folder", "alertname"]
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 4h
+` : "";
+  return `# Generated by \`arc platform deploy\` \u2014 do not edit by hand.
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: arc-alerts
+    folder: Arc
+    interval: 1m
+    rules:
+${ruleYaml}
+${contactSection}`;
+}
 function generateGrafanaDatasources() {
   return `# Generated by \`arc platform deploy\` \u2014 do not edit by hand.
 apiVersion: 1
@@ -36610,10 +37021,17 @@ datasources:
     uid: loki
     jsonData:
       derivedFields:
+        # Plain-text logs that happen to contain "trace_id=<id>".
         - datasourceUid: tempo
           matcherRegex: "trace_id=(\\\\w+)"
           name: TraceID
           url: $\${__value.raw}
+        # OTLP-ingested logs \u2014 trace_id arrives as structured metadata.
+        - datasourceUid: tempo
+          matcherType: label
+          matcherRegex: trace_id
+          name: TraceID (OTLP)
+          url: $\${__value.raw}
   - name: Prometheus
     type: prometheus
     access: proxy
@@ -36656,7 +37074,7 @@ function generateArcOverviewDashboard() {
           label: "Service",
           type: "query",
           datasource: { type: "prometheus", uid: "prometheus" },
-          query: "label_values(traces_spanmetrics_calls_total, service_name)",
+          query: "label_values(traces_span_metrics_calls_total, service_name)",
           refresh: 2,
           includeAll: false,
           multi: false,
@@ -36665,29 +37083,29 @@ function generateArcOverviewDashboard() {
       ]
     },
     panels: [
-      panelStat("Request rate (req/s)", { x: 0, y: 0, w: 6, h: 4 }, 'sum(rate(traces_spanmetrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m]))', "reqps"),
-      panelStat("Error rate (%)", { x: 6, y: 0, w: 6, h: 4 }, 'sum(rate(traces_spanmetrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER", status_code="STATUS_CODE_ERROR"}[5m])) / clamp_min(sum(rate(traces_spanmetrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])), 0.001) * 100', "percent", { red: 1, orange: 0.1 }),
-      panelStat("P99 latency", { x: 12, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))', "ms", { red: 1000, orange: 300 }),
+      panelStat("Request rate (req/s)", { x: 0, y: 0, w: 6, h: 4 }, 'sum(rate(traces_span_metrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m]))', "reqps"),
+      panelStat("Error rate (%)", { x: 6, y: 0, w: 6, h: 4 }, 'sum(rate(traces_span_metrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER", status_code="STATUS_CODE_ERROR"}[5m])) / clamp_min(sum(rate(traces_span_metrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])), 0.001) * 100', "percent", { red: 1, orange: 0.1 }),
+      panelStat("P99 latency", { x: 12, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.99, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))', "ms", { red: 1000, orange: 300 }),
       panelStat("Active commands/sec", { x: 18, y: 0, w: 6, h: 4 }, 'sum(rate(arc_commands_total{service_name="$service"}[5m]))', "ops"),
-      panelTimeseries("Request rate by route", { x: 0, y: 4, w: 12, h: 8 }, 'sum by (span_name) (rate(traces_spanmetrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[1m]))', "{{span_name}}", "reqps"),
+      panelTimeseries("Request rate by route", { x: 0, y: 4, w: 12, h: 8 }, 'sum by (span_name) (rate(traces_span_metrics_calls_total{service_name="$service", span_kind="SPAN_KIND_SERVER"}[1m]))', "{{span_name}}", "reqps"),
       panelTimeseries("Latency percentiles", { x: 12, y: 4, w: 12, h: 8 }, [
         {
-          expr: 'histogram_quantile(0.5, sum(rate(traces_spanmetrics_latency_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
+          expr: 'histogram_quantile(0.5, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
           legend: "p50"
         },
         {
-          expr: 'histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
+          expr: 'histogram_quantile(0.95, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
           legend: "p95"
         },
         {
-          expr: 'histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
+          expr: 'histogram_quantile(0.99, sum(rate(traces_span_metrics_duration_milliseconds_bucket{service_name="$service", span_kind="SPAN_KIND_SERVER"}[5m])) by (le))',
           legend: "p99"
         }
-      ], "ms"),
+      ], "", "ms"),
       panelTimeseries("Commands per second", { x: 0, y: 12, w: 12, h: 8 }, 'sum by (arc_command_name) (rate(arc_commands_total{service_name="$service"}[1m]))', "{{arc_command_name}}", "ops"),
-      panelTimeseries("Command p95 latency", { x: 12, y: 12, w: 12, h: 8 }, 'histogram_quantile(0.95, sum by (arc_command_name, le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service"}[5m])))', "{{arc_command_name}}", "ms"),
-      panelTimeseries("DB find ops/sec by collection", { x: 0, y: 20, w: 12, h: 8 }, 'sum by (db_collection_name) (rate(arc_db_find_ms_milliseconds_count{service_name="$service"}[1m]))', "{{db_collection_name}}", "ops"),
-      panelTimeseries("DB find p95 latency", { x: 12, y: 20, w: 12, h: 8 }, 'histogram_quantile(0.95, sum by (db_collection_name, le) (rate(arc_db_find_ms_milliseconds_bucket{service_name="$service"}[5m])))', "{{db_collection_name}}", "ms"),
+      panelTimeseries("Command p95 latency", { x: 12, y: 12, w: 12, h: 8 }, 'histogram_quantile(0.95, sum by (arc_command_name, le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service"}[5m])))', "{{arc_command_name}}", "ms"),
+      panelTimeseries("DB ops/sec by collection", { x: 0, y: 20, w: 12, h: 8 }, 'sum by (db_collection_name, db_operation_name) (rate(arc_db_operation_duration_milliseconds_count{service_name="$service"}[1m]))', "{{db_collection_name}} {{db_operation_name}}", "ops"),
+      panelTimeseries("DB p95 latency by operation", { x: 12, y: 20, w: 12, h: 8 }, 'histogram_quantile(0.95, sum by (db_operation_name, le) (rate(arc_db_operation_duration_milliseconds_bucket{service_name="$service"}[5m])))', "{{db_operation_name}}", "ms"),
       {
         title: "Recent error logs",
         type: "logs",
@@ -36695,7 +37113,7 @@ function generateArcOverviewDashboard() {
         datasource: { type: "loki", uid: "loki" },
         targets: [
           {
-            expr: '{service_name="$service"} |= `ERROR`',
+            expr: '{service_name="$service"} | severity_text=~`ERROR|FATAL`',
             refId: "A"
           }
         ],
@@ -36729,7 +37147,7 @@ function generateArcTracesDashboard() {
           label: "Service",
           type: "query",
           datasource: { type: "prometheus", uid: "prometheus" },
-          query: "label_values(traces_spanmetrics_calls_total, service_name)",
+          query: "label_values(traces_span_metrics_calls_total, service_name)",
           refresh: 2,
           current: { text: "arc-prod", value: "arc-prod" }
         }
@@ -36828,12 +37246,23 @@ function generateArcLogsDashboard() {
           type: "textbox",
           query: "",
           current: { text: "", value: "" }
+        },
+        {
+          name: "container",
+          label: "Container",
+          type: "query",
+          datasource: { type: "loki", uid: "loki" },
+          query: "label_values(container)",
+          refresh: 2,
+          includeAll: true,
+          multi: false,
+          current: { text: "All", value: "$__all" }
         }
       ]
     },
     panels: [
-      panelStat("Logs ingested (1h)", { x: 0, y: 0, w: 6, h: 4 }, 'sum(increase({service_name="$service"}[1h]))', "short"),
-      panelStat("Errors (1h)", { x: 6, y: 0, w: 6, h: 4 }, 'sum(increase({service_name="$service", severity_text=~"ERROR|FATAL"}[1h]))', "short", { orange: 1, red: 50 }),
+      panelStat("Logs ingested (1h)", { x: 0, y: 0, w: 6, h: 4 }, 'sum(count_over_time({service_name="$service"}[1h]))', "short", undefined, LOKI_DS),
+      panelStat("Errors (1h)", { x: 6, y: 0, w: 6, h: 4 }, 'sum(count_over_time({service_name="$service"} | severity_text=~`ERROR|FATAL` [1h]))', "short", { orange: 1, red: 50 }, LOKI_DS),
       {
         title: "Log volume by severity",
         type: "timeseries",
@@ -36883,6 +37312,27 @@ function generateArcLogsDashboard() {
           dedupStrategy: "none",
           sortOrder: "Descending"
         }
+      },
+      {
+        title: "Container logs ($container)",
+        type: "logs",
+        gridPos: { x: 0, y: 26, w: 24, h: 14 },
+        datasource: { type: "loki", uid: "loki" },
+        targets: [
+          {
+            expr: '{container=~"$container"} |~ "$search"',
+            refId: "A"
+          }
+        ],
+        options: {
+          showTime: true,
+          showLabels: true,
+          showCommonLabels: false,
+          wrapLogMessage: true,
+          enableLogDetails: true,
+          dedupStrategy: "none",
+          sortOrder: "Descending"
+        }
       }
     ]
   };
@@ -36912,7 +37362,7 @@ function generateArcSamplingDashboard() {
           expr: "sum(rate(otelcol_exporter_sent_spans[1m]))",
           legend: "exported"
         }
-      ], "ops"),
+      ], "", "ops"),
       panelTimeseries("Collector queue size (BatchSpanProcessor)", { x: 0, y: 12, w: 12, h: 8 }, "otelcol_processor_batch_batch_send_size_sum / clamp_min(otelcol_processor_batch_batch_send_size_count, 1)", "avg batch size", "short"),
       panelTimeseries("Collector process memory", { x: 12, y: 12, w: 12, h: 8 }, 'process_resident_memory_bytes{job="otel-collector"}', "RSS", "bytes")
     ]
@@ -36953,24 +37403,24 @@ function generateArcCommandDashboard() {
     },
     panels: [
       panelStat("Call rate", { x: 0, y: 0, w: 6, h: 4 }, 'sum(rate(arc_commands_total{service_name="$service", arc_command_name="$command"}[5m]))', "ops"),
-      panelStat("P50 latency", { x: 6, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.5, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms"),
-      panelStat("P95 latency", { x: 12, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.95, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms", { orange: 200, red: 1000 }),
-      panelStat("P99 latency", { x: 18, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.99, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms", { orange: 500, red: 2000 }),
+      panelStat("P50 latency", { x: 6, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.5, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms"),
+      panelStat("P95 latency", { x: 12, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.95, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms", { orange: 200, red: 1000 }),
+      panelStat("P99 latency", { x: 18, y: 0, w: 6, h: 4 }, 'histogram_quantile(0.99, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))', "ms", { orange: 500, red: 2000 }),
       panelTimeseries("Call rate over time", { x: 0, y: 4, w: 12, h: 8 }, 'sum(rate(arc_commands_total{service_name="$service", arc_command_name="$command"}[1m]))', "calls/s", "ops"),
       panelTimeseries("Latency percentiles", { x: 12, y: 4, w: 12, h: 8 }, [
         {
-          expr: 'histogram_quantile(0.5, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
+          expr: 'histogram_quantile(0.5, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
           legend: "p50"
         },
         {
-          expr: 'histogram_quantile(0.95, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
+          expr: 'histogram_quantile(0.95, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
           legend: "p95"
         },
         {
-          expr: 'histogram_quantile(0.99, sum by (le) (rate(arc_command_duration_ms_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
+          expr: 'histogram_quantile(0.99, sum by (le) (rate(arc_command_duration_milliseconds_bucket{service_name="$service", arc_command_name="$command"}[5m])))',
           legend: "p99"
         }
-      ], "ms"),
+      ], "", "ms"),
       {
         title: "Recent traces (sampled)",
         type: "traces",
@@ -36989,23 +37439,108 @@ function generateArcCommandDashboard() {
   };
   return JSON.stringify(dashboard, null, 2);
 }
+function generateArcInfraDashboard() {
+  const dashboard = {
+    title: "Arc Infrastructure",
+    uid: "arc-infra",
+    schemaVersion: 39,
+    version: 1,
+    refresh: "30s",
+    time: { from: "now-3h", to: "now" },
+    tags: ["arc", "auto-provisioned"],
+    panels: [
+      panelStat("Host CPU used", { x: 0, y: 0, w: 6, h: 4 }, '100 * (1 - avg(rate(system_cpu_time_seconds_total{state="idle"}[5m])))', "percent", { orange: 70, red: 90 }),
+      panelStat("Host memory used", { x: 6, y: 0, w: 6, h: 4 }, '100 * sum(system_memory_usage_bytes{state="used"}) / sum(system_memory_usage_bytes)', "percent", { orange: 80, red: 90 }),
+      panelStat("Disk used (worst mount)", { x: 12, y: 0, w: 6, h: 4 }, '100 * max(sum by (device, mountpoint) (system_filesystem_usage_bytes{state="used"}) / sum by (device, mountpoint) (system_filesystem_usage_bytes))', "percent", { orange: 75, red: 85 }),
+      panelStat("Load (1m)", { x: 18, y: 0, w: 6, h: 4 }, "system_cpu_load_average_1m", "short"),
+      panelTimeseries("Host CPU utilization", { x: 0, y: 4, w: 12, h: 8 }, [
+        {
+          expr: '100 * (1 - avg(rate(system_cpu_time_seconds_total{state="idle"}[5m])))',
+          legend: "used %"
+        },
+        { expr: "system_cpu_load_average_1m", legend: "load 1m" },
+        { expr: "system_cpu_load_average_5m", legend: "load 5m" },
+        { expr: "system_cpu_load_average_15m", legend: "load 15m" }
+      ], "", "short"),
+      panelTimeseries("Host memory by state", { x: 12, y: 4, w: 12, h: 8 }, "sum by (state) (system_memory_usage_bytes)", "{{state}}", "bytes"),
+      panelTimeseries("Filesystem usage by mount", { x: 0, y: 12, w: 12, h: 8 }, '100 * sum by (mountpoint) (system_filesystem_usage_bytes{state="used"}) / sum by (mountpoint) (system_filesystem_usage_bytes)', "{{mountpoint}}", "percent"),
+      panelTimeseries("Disk I/O", { x: 12, y: 12, w: 12, h: 8 }, "sum by (device, direction) (rate(system_disk_io_bytes_total[5m]))", "{{device}} {{direction}}", "Bps"),
+      panelTimeseries("Network I/O", { x: 0, y: 20, w: 12, h: 8 }, 'sum by (device, direction) (rate(system_network_io_bytes_total{device!="lo"}[5m]))', "{{device}} {{direction}}", "Bps"),
+      panelTimeseries("Container restarts (24h)", { x: 12, y: 20, w: 12, h: 8 }, "sum by (container_name) (increase(container_restarts_total[24h]))", "{{container_name}}", "short"),
+      panelTimeseries("Container CPU", { x: 0, y: 28, w: 12, h: 8 }, "container_cpu_utilization_ratio", "{{container_name}}", "percent"),
+      panelTimeseries("Container memory", { x: 12, y: 28, w: 12, h: 8 }, "container_memory_usage_total_bytes", "{{container_name}}", "bytes"),
+      panelTimeseries("Container network RX", { x: 0, y: 36, w: 12, h: 8 }, "sum by (container_name) (rate(container_network_io_usage_rx_bytes_total[5m]))", "{{container_name}}", "Bps"),
+      panelTimeseries("Container network TX", { x: 12, y: 36, w: 12, h: 8 }, "sum by (container_name) (rate(container_network_io_usage_tx_bytes_total[5m]))", "{{container_name}}", "Bps")
+    ]
+  };
+  return JSON.stringify(dashboard, null, 2);
+}
+function generateArcEdgeDashboard() {
+  const dashboard = {
+    title: "Arc Caddy / Edge",
+    uid: "arc-edge",
+    schemaVersion: 39,
+    version: 1,
+    refresh: "30s",
+    time: { from: "now-1h", to: "now" },
+    tags: ["arc", "auto-provisioned"],
+    panels: [
+      panelStat("Requests/s", { x: 0, y: 0, w: 6, h: 4 }, "sum(rate(caddy_http_request_duration_seconds_count[1m]))", "reqps"),
+      panelStat("In-flight requests", { x: 6, y: 0, w: 6, h: 4 }, "sum(caddy_http_requests_in_flight)", "short"),
+      panelStat("Handler errors/s", { x: 12, y: 0, w: 6, h: 4 }, "sum(rate(caddy_http_request_errors_total[5m]))", "ops", { orange: 0.1, red: 1 }),
+      panelStat("P95 latency", { x: 18, y: 0, w: 6, h: 4 }, "histogram_quantile(0.95, sum by (le) (rate(caddy_http_request_duration_seconds_bucket[5m])))", "s", { orange: 0.3, red: 1 }),
+      panelTimeseries("Request rate by host", { x: 0, y: 4, w: 12, h: 8 }, "sum by (host) (rate(caddy_http_request_duration_seconds_count[1m]))", "{{host}}", "reqps"),
+      panelTimeseries("Responses by status code", { x: 12, y: 4, w: 12, h: 8 }, "sum by (code) (rate(caddy_http_request_duration_seconds_count[1m]))", "{{code}}", "reqps"),
+      panelTimeseries("P95 latency by host", { x: 0, y: 12, w: 12, h: 8 }, "histogram_quantile(0.95, sum by (host, le) (rate(caddy_http_request_duration_seconds_bucket[5m])))", "{{host}}", "s"),
+      panelTimeseries("4xx/5xx responses (access log)", { x: 12, y: 12, w: 12, h: 8 }, 'sum by (status) (count_over_time({compose_service="caddy"} | json | status >= 400 [$__interval]))', "{{status}}", "short", LOKI_DS),
+      {
+        title: "Access log (live)",
+        type: "logs",
+        gridPos: { x: 0, y: 20, w: 24, h: 12 },
+        datasource: { type: "loki", uid: "loki" },
+        targets: [
+          {
+            expr: '{compose_service="caddy"}',
+            refId: "A"
+          }
+        ],
+        options: {
+          showTime: true,
+          showLabels: false,
+          showCommonLabels: false,
+          wrapLogMessage: true,
+          enableLogDetails: true,
+          dedupStrategy: "none",
+          sortOrder: "Descending"
+        }
+      }
+    ]
+  };
+  return JSON.stringify(dashboard, null, 2);
+}
 function generateObservabilityConfigs(cfg) {
   return {
     "observability/otel-collector-config.yaml": generateOtelCollectorConfig(cfg),
     "observability/tempo.yaml": generateTempoConfig(cfg),
     "observability/loki-config.yaml": generateLokiConfig(cfg),
     "observability/prometheus.yml": generatePrometheusConfig(cfg),
+    "observability/alloy-config.alloy": generateAlloyConfig(),
     "observability/grafana-datasources.yaml": generateGrafanaDatasources(),
     "observability/grafana-dashboards.yaml": generateGrafanaDashboardsProvider(),
+    "observability/grafana-alerting/alerting.yaml": generateGrafanaAlerting(cfg),
     "observability/grafana-dashboards/arc-overview.json": generateArcOverviewDashboard(),
     "observability/grafana-dashboards/arc-traces.json": generateArcTracesDashboard(),
     "observability/grafana-dashboards/arc-service-map.json": generateArcServiceMapDashboard(),
     "observability/grafana-dashboards/arc-logs.json": generateArcLogsDashboard(),
     "observability/grafana-dashboards/arc-sampling.json": generateArcSamplingDashboard(),
-    "observability/grafana-dashboards/arc-command.json": generateArcCommandDashboard()
+    "observability/grafana-dashboards/arc-command.json": generateArcCommandDashboard(),
+    "observability/grafana-dashboards/arc-infra.json": generateArcInfraDashboard(),
+    "observability/grafana-dashboards/arc-edge.json": generateArcEdgeDashboard()
   };
 }
-function panelStat(title, gridPos, expr, unit, thresholds) {
+var PROMETHEUS_DS = { type: "prometheus", uid: "prometheus" };
+var LOKI_DS = { type: "loki", uid: "loki" };
+function panelStat(title, gridPos, expr, unit, thresholds, datasource = PROMETHEUS_DS) {
   const steps = [
     { color: "green", value: null }
   ];
@@ -37019,7 +37554,7 @@ function panelStat(title, gridPos, expr, unit, thresholds) {
     title,
     type: "stat",
     gridPos,
-    datasource: { type: "prometheus", uid: "prometheus" },
+    datasource,
     targets: [{ expr, refId: "A", legendFormat: title }],
     fieldConfig: {
       defaults: {
@@ -37037,7 +37572,7 @@ function panelStat(title, gridPos, expr, unit, thresholds) {
     }
   };
 }
-function panelTimeseries(title, gridPos, query, legend, unit) {
+function panelTimeseries(title, gridPos, query, legend, unit, datasource = PROMETHEUS_DS) {
   const targets = Array.isArray(query) ? query.map((q, i) => ({
     expr: q.expr,
     refId: String.fromCharCode(65 + i),
@@ -37047,7 +37582,7 @@ function panelTimeseries(title, gridPos, query, legend, unit) {
     title,
     type: "timeseries",
     gridPos,
-    datasource: { type: "prometheus", uid: "prometheus" },
+    datasource,
     targets,
     fieldConfig: {
       defaults: {
@@ -37378,11 +37913,16 @@ function validateDeployConfig(input) {
         metrics: optionalString(retentionRaw, "observability.retention.metrics")
       };
     }
+    const alertWebhookUrl = optionalString(observabilityRaw, "observability.alertWebhookUrl");
+    if (alertWebhookUrl !== undefined && !/^https?:\/\/.+/.test(alertWebhookUrl)) {
+      throw new Error(`deploy.arc.json: observability.alertWebhookUrl must be an http(s) URL (got "${alertWebhookUrl}")`);
+    }
     validated.observability = {
       enabled: enabledRaw,
       subdomain: optionalString(observabilityRaw, "observability.subdomain") ?? "observability",
       adminPasswordEnv: optionalString(observabilityRaw, "observability.adminPasswordEnv") ?? "ARC_OBSERVABILITY_PASSWORD",
-      retention
+      retention,
+      alertWebhookUrl
     };
   }
   const provision = input.provision;
@@ -37678,14 +38218,14 @@ async function bootstrap(inputs) {
     });
     ok("Host bootstrapped");
   }
-  const needUpStack = state.kind !== "ready" || state.marker === null || state.marker.configHash !== inputs.configHash || !await isRegistryRunning(cfg);
+  const needUpStack = state.kind !== "ready" || state.marker === null || state.marker.configHash !== inputs.configHash || state.marker.cliVersion !== inputs.cliVersion || !await isRegistryRunning(cfg);
   if (needUpStack) {
     await upStack(inputs);
     ok("Docker stack up");
   }
   if (cfg.observability?.enabled) {
     log2("Ensuring observability sidecars are running...");
-    const obsServices = ["otel-collector", "tempo", "loki", "prometheus", "grafana"];
+    const obsServices = ["otel-collector", "tempo", "loki", "prometheus", "alloy", "grafana"];
     await assertExec(cfg.target, `cd ${cfg.target.remoteDir} && docker compose pull --ignore-pull-failures ${obsServices.join(" ")} && docker compose up -d ${obsServices.join(" ")}`);
     ok("Observability stack up");
   }
@@ -37752,7 +38292,7 @@ async function upStack(inputs) {
   await scpUpload(cfg.target, join18(workDir, "docker-compose.yml"), `${cfg.target.remoteDir}/docker-compose.yml`);
   await scpUpload(cfg.target, join18(workDir, "htpasswd"), `${cfg.target.remoteDir}/registry-auth/htpasswd`);
   if (observabilityFiles && observabilityHtpasswd) {
-    await assertExec(cfg.target, `mkdir -p ${cfg.target.remoteDir}/observability/grafana-dashboards`);
+    await assertExec(cfg.target, `mkdir -p ${cfg.target.remoteDir}/observability/grafana-dashboards ${cfg.target.remoteDir}/observability/grafana-alerting`);
     for (const relPath of Object.keys(observabilityFiles)) {
       const localDir = dirname9(join18(workDir, relPath));
       mkdirSync12(localDir, { recursive: true });
@@ -39274,7 +39814,7 @@ class ContextHandler {
     try {
       return await this.telemetry.startSpan(`command.${commandName}`, runCommand, { attributes: baseAttrs });
     } finally {
-      this.telemetry.measureSince("arc.command.duration_ms", start, {
+      this.telemetry.measureSince("arc.command.duration", start, {
         "arc.command.name": commandName
       });
       this.telemetry.incrementCounter("arc.commands.total", 1, {
@@ -40359,6 +40899,7 @@ async function createArcServer(config) {
     websocket: {
       open(ws) {
         connectionManager.addClient(ws);
+        config.telemetry?.addUpDown("arc.ws.active_connections", 1);
       },
       async message(ws, messageStr) {
         const client = connectionManager.getClientByWs(ws);
@@ -40371,6 +40912,9 @@ async function createArcServer(config) {
           console.error("Failed to parse WS message:", error);
           return;
         }
+        config.telemetry?.incrementCounter("arc.ws.messages", 1, {
+          "messaging.message.type": String(message?.type ?? "unknown")
+        });
         const dispatch = async () => {
           try {
             for (const handler of wsHandlers) {
@@ -40408,6 +40952,7 @@ async function createArcServer(config) {
           cleanupClientSubs(client.id);
           config.onWsClose?.(client.id);
           connectionManager.removeClient(client.id);
+          config.telemetry?.addUpDown("arc.ws.active_connections", -1);
         }
       }
     }
@@ -40726,7 +41271,8 @@ async function startPlatformServer(opts) {
         endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
         mode: devMode ? "development" : "production",
         sampleRate: devMode ? 1 : 1,
-        debug: process.env.ARC_OTEL_DEBUG === "true"
+        debug: process.env.ARC_OTEL_DEBUG === "true",
+        patchConsole: process.env.ARC_OTEL_PATCH_CONSOLE !== "false"
       });
       telemetry = init2.telemetry;
       telemetryShutdown = init2.shutdown;