deepdebug-local-agent 1.0.23 → 1.1.0

package/src/observability/bundle-files.js

@@ -0,0 +1,418 @@
+/**
+ * observability/bundle-files.js
+ *
+ * The client-side observability bundle, embedded as templated strings so it
+ * ships inside the pkg-bundled agent binary (no loose asset files to lose).
+ *
+ * This is the CLIENT delivery of the same stack we run for ourselves in
+ * `vh-observability` (Grafana + Loki + Alloy + Alertmanager). The difference is
+ * only in how logs get in and where chunks live:
+ *
+ *   vh-observability (ours, Cloud Run):  logs = Cloud Logging -> Pub/Sub (gcplog)
+ *                                        storage = GCS
+ *   client bundle    (this, VPS):        logs = docker.sock (every container)
+ *                                        storage = local filesystem volume
+ *
+ * Everything downstream is identical: Loki stores + rules-evaluates, Alertmanager
+ * posts firing alerts to the gateway webhook (tenant resolved from the ?s= trigger
+ * secret, exactly like ours), and Grafana serves a logs dashboard whose "Open in
+ * Workbench" data link hits /api/v1/webhooks/from-log.
+ *
+ * Placeholders replaced by renderBundle():
+ *   __GATEWAY_URL__      e.g. https://api.visionhammer.io   (no trailing slash)
+ *   __TRIGGER_SECRET__   the tenant's vht_... alertmanager trigger secret
+ *   __ENVIRONMENT__      static env label for this client's logs (e.g. production)
+ */
+
+// ---- Loki: filesystem backend (no GCS); rules + ruler -> Alertmanager ----
+const LOKI_CONFIG = `# DeepDebug client observability - Loki (docker-compose, filesystem backend).
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+  log_level: warn
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules-store
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: 336h          # 14 days
+  reject_old_samples: true
+  # Wide enough to accept a host's existing docker log backlog on first start.
+  # loki.source.docker replays each container's full history oldest-first; a
+  # tight window (e.g. 168h) 400-rejects that backlog and queues fresh lines
+  # behind it. 720h clears the backlog cleanly; retention above still caps disk.
+  reject_old_samples_max_age: 720h
+  volume_enabled: true
+  allow_structured_metadata: true
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+
+# Ruler evaluates the LogQL error rule and pushes firing alerts to Alertmanager.
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /etc/loki/rules
+  rule_path: /loki/rules-tmp
+  alertmanager_url: http://alertmanager:9093
+  # Alertmanager v0.27 removed the legacy v1 push API; Loki's ruler defaults to
+  # POSTing /api/v1/alerts, which that Alertmanager rejects, so alerts silently
+  # never arrive. Force the v2 endpoint.
+  enable_alertmanager_v2: true
+  enable_api: true
+  ring:
+    kvstore:
+      store: inmemory
+`;
+
+// ---- Loki rule: error rate by service -> critical alert ----
+const LOKI_RULE = `# DeepDebug client error alerts. Labels service/severity/environment flow into
+# the gateway incident (AlertmanagerIngestService reads them).
+groups:
+  - name: client-errors
+    rules:
+      - alert: ClientServiceErrors
+        expr: |
+          sum by (service, environment) (rate({job="docker", service!~"deepdebug-observability-.*|vh-obs-.*|visionhammer-agent"} |~ \`(?i)\\bERROR\\b\` [5m])) > 0
+        for: 2m
+        labels:
+          severity: critical
+          origin: deepdebug-observability
+        annotations:
+          summary: "errors on {{ $labels.service }} ({{ $labels.environment }})"
+`;
+
+// ---- Alloy: tail every container via docker.sock, label by container name ----
+const ALLOY_CONFIG = `// DeepDebug client observability - Alloy.
+// Tails every Docker container on the host (docker.sock) and ships to Loki.
+// The same stream is also pushed to VisionHammer (OTLP/HTTP JSON, outbound only)
+// so Live logs and the services list see this server; auth is the tenant's
+// trigger secret, the same one the Alertmanager below uses.
+// service = container name; environment = the static label this client was
+// provisioned with; level is parsed from the line for quick filtering.
+
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/?(.*)"
+    target_label  = "service"
+  }
+  rule {
+    source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+    target_label  = "project"
+  }
+}
+
+loki.source.docker "default" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.containers.output
+  labels     = { job = "docker" }
+  forward_to = [loki.process.default.receiver]
+}
+
+loki.process "default" {
+  forward_to = [loki.write.default.receiver, otelcol.receiver.loki.visionhammer.receiver]
+
+  stage.regex {
+    expression = "(?i)(?P<level>ERROR|WARNING|WARN|INFO|DEBUG|SEVERE|FATAL|CRITICAL)"
+  }
+  stage.labels {
+    values = { level = "" }
+  }
+  stage.static_labels {
+    values = { environment = "__ENVIRONMENT__" }
+  }
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}
+
+// ---- push to VisionHammer: Loki entries -> OTLP logs (labels become attributes) ----
+otelcol.receiver.loki "visionhammer" {
+  output {
+    logs = [otelcol.processor.batch.visionhammer.input]
+  }
+}
+
+otelcol.processor.batch "visionhammer" {
+  send_batch_size     = 500
+  send_batch_max_size = 2000
+  timeout             = "5s"
+  output {
+    logs = [otelcol.exporter.otlphttp.visionhammer.input]
+  }
+}
+
+otelcol.exporter.otlphttp "visionhammer" {
+  client {
+    endpoint    = "__GATEWAY_URL__"
+    compression = "gzip"
+    headers     = { "Authorization" = "Bearer __TRIGGER_SECRET__" }
+  }
+  encoding      = "json"
+  logs_endpoint = "__GATEWAY_URL__/api/v1/webhooks/logs/otlp"
+}
+`;
+
+// ---- Alertmanager: firing alerts -> gateway webhook (tenant from ?s=) ----
+const ALERTMANAGER_CONFIG = `route:
+  receiver: deepdebug-gateway
+  group_by: ['service']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: deepdebug-gateway
+    webhook_configs:
+      - url: '__GATEWAY_URL__/api/v1/webhooks/alertmanager'
+        send_resolved: false
+        http_config:
+          authorization:
+            type: Bearer
+            credentials: '__TRIGGER_SECRET__'
+`;
+
+// ---- Grafana provisioning: Loki datasource + dashboard provider ----
+const GRAFANA_DATASOURCE = `apiVersion: 1
+datasources:
+  - name: Loki
+    uid: ddo-loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    isDefault: true
+    jsonData:
+      maxLines: 1000
+`;
+
+const GRAFANA_DASH_PROVIDER = `apiVersion: 1
+providers:
+  - name: deepdebug
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    options:
+      path: /etc/grafana/dashboards
+      foldersFromFilesStructure: false
+`;
+
+// ---- Grafana dashboard: logs + error stats, "Open in Workbench" data link ----
+const DASHBOARD = JSON.stringify({
+  uid: 'ddo-logs',
+  title: 'DeepDebug - Observability',
+  editable: true,
+  schemaVersion: 39,
+  style: 'dark',
+  tags: ['deepdebug', 'logs'],
+  time: { from: 'now-1h', to: 'now' },
+  refresh: '30s',
+  templating: {
+    list: [
+      { name: 'service', label: 'Service', type: 'query',
+        datasource: { type: 'loki', uid: 'ddo-loki' },
+        query: 'label_values(service)', refresh: 2, multi: true, includeAll: true,
+        current: { text: 'All', value: '$__all' } },
+      { name: 'search', label: 'Search', type: 'textbox', current: { text: '', value: '' } },
+    ],
+  },
+  panels: [
+    { id: 1, title: 'Errors (last 5m)', type: 'stat',
+      datasource: { type: 'loki', uid: 'ddo-loki' },
+      gridPos: { h: 5, w: 6, x: 0, y: 0 },
+      options: { colorMode: 'background', graphMode: 'area', reduceOptions: { calcs: ['lastNotNull'] } },
+      fieldConfig: { defaults: { color: { mode: 'thresholds' },
+        thresholds: { steps: [ { color: 'green', value: null }, { color: 'orange', value: 1 }, { color: 'red', value: 10 } ] } } },
+      targets: [ { refId: 'A', expr: 'sum(count_over_time({service=~"$service"} |~ `(?i)error` [5m]))' } ] },
+    { id: 2, title: 'Warnings (last 5m)', type: 'stat',
+      datasource: { type: 'loki', uid: 'ddo-loki' },
+      gridPos: { h: 5, w: 6, x: 6, y: 0 },
+      options: { colorMode: 'background', graphMode: 'area', reduceOptions: { calcs: ['lastNotNull'] } },
+      fieldConfig: { defaults: { color: { mode: 'thresholds' },
+        thresholds: { steps: [ { color: 'green', value: null }, { color: 'yellow', value: 1 } ] } } },
+      targets: [ { refId: 'A', expr: 'sum(count_over_time({service=~"$service"} |~ `(?i)warn` [5m]))' } ] },
+    { id: 3, title: 'Error rate by service', type: 'timeseries',
+      datasource: { type: 'loki', uid: 'ddo-loki' },
+      gridPos: { h: 5, w: 12, x: 12, y: 0 },
+      fieldConfig: { defaults: { custom: { drawStyle: 'line', fillOpacity: 20, showPoints: 'never' }, color: { mode: 'palette-classic' } } },
+      targets: [ { refId: 'A', legendFormat: '{{service}}', expr: 'sum by (service) (rate({service=~"$service"} |~ `(?i)error` [5m]))' } ] },
+    { id: 4, title: 'Logs', type: 'logs',
+      datasource: { type: 'loki', uid: 'ddo-loki' },
+      gridPos: { h: 12, w: 24, x: 0, y: 5 },
+      options: { showTime: true, wrapLogMessage: true, enableLogDetails: true, dedupStrategy: 'none', sortOrder: 'Descending' },
+      targets: [ { refId: 'A', expr: '{service=~"$service"} |~ `(?i)$search`' } ],
+      fieldConfig: { defaults: { links: [ {
+        title: 'Open in DeepDebug Workbench',
+        url: '__GATEWAY_URL__/api/v1/webhooks/from-log?s=__TRIGGER_SECRET__&service=${service}&environment=__ENVIRONMENT__&severity=critical&message=${__value.raw}',
+        targetBlank: true,
+      } ] } } },
+  ],
+}, null, 2);
+
+// ---- docker-compose: the four containers, Grafana bound to loopback only ----
+const DOCKER_COMPOSE = `name: deepdebug-observability
+services:
+  loki:
+    image: grafana/loki:3.1.1
+    command: -config.file=/etc/loki/loki-config.yaml
+    volumes:
+      - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro
+      - ./loki/rules:/etc/loki/rules:ro
+      - ddo-loki:/loki
+    restart: unless-stopped
+
+  alloy:
+    image: grafana/alloy:v1.3.1
+    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    volumes:
+      - ./alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ddo-alloy:/var/lib/alloy/data
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:11.1.0
+    environment:
+      - GF_SECURITY_ADMIN_USER=\${GF_ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=\${GF_ADMIN_PASSWORD:-deepdebug}
+      - GF_USERS_ALLOW_SIGN_UP=false
+      - GF_USERS_DEFAULT_THEME=dark
+      - GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/dashboards/ddo-logs.json
+    ports:
+      # Loopback only: reach it over an SSH tunnel, never expose to the internet.
+      - "127.0.0.1:3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/etc/grafana/dashboards:ro
+      - ddo-grafana:/var/lib/grafana
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  alertmanager:
+    image: prom/alertmanager:v0.27.0
+    command:
+      - --config.file=/etc/alertmanager/alertmanager.yml
+      - --storage.path=/tmp/alertmanager
+    volumes:
+      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
+    restart: unless-stopped
+
+volumes:
+  ddo-loki:
+  ddo-alloy:
+  ddo-grafana:
+`;
+
+// ---- Tunnel overlay (opt-in): expose Loki to the gateway, bearer-gated ----
+// Caddy sits in front of Loki and 403s anything without the exact bearer token,
+// so the Cloudflare quick tunnel never exposes an unauthenticated Loki. The
+// gateway's loki-logging config stores this same token.
+const CADDY_CONFIG = `# DeepDebug observability - bearer-auth gate in front of Loki.
+:8080 {
+	@unauthorized not header Authorization "Bearer __LOKI_TUNNEL_TOKEN__"
+	respond @unauthorized "forbidden" 403
+	reverse_proxy loki:3100
+}
+`;
+
+// Opt-in overlay compose. Start alongside the base file:
+//   docker compose -f docker-compose.yml -f docker-compose.tunnel.yml up -d
+// then read the public URL:
+//   docker compose logs cloudflared | grep -o 'https://[^ ]*trycloudflare.com'
+const TUNNEL_COMPOSE = `name: deepdebug-observability
+services:
+  loki-auth:
+    image: caddy:2.8
+    volumes:
+      - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro
+    depends_on:
+      - loki
+    restart: unless-stopped
+
+  cloudflared:
+    image: cloudflare/cloudflared:2024.8.3
+    command: tunnel --no-autoupdate --url http://loki-auth:8080
+    depends_on:
+      - loki-auth
+    restart: unless-stopped
+`;
+
+/** The compose project name; also used to recognise OUR own stack in detection. */
+export const COMPOSE_PROJECT = 'deepdebug-observability';
+
+/** Raw templated file map (relative path -> content, placeholders unresolved). */
+export const BUNDLE_TEMPLATES = {
+  'docker-compose.yml': DOCKER_COMPOSE,
+  'loki/loki-config.yaml': LOKI_CONFIG,
+  // Loki's local ruler reads rules from <rules_dir>/<tenant>/; with
+  // auth_enabled=false the tenant is the literal "fake". A file placed directly
+  // under rules/ (no tenant subdir) is silently ignored and no alert ever fires.
+  'loki/rules/fake/client.yaml': LOKI_RULE,
+  'alloy/config.alloy': ALLOY_CONFIG,
+  'alertmanager/alertmanager.yml': ALERTMANAGER_CONFIG,
+  'grafana/provisioning/datasources/loki.yaml': GRAFANA_DATASOURCE,
+  'grafana/provisioning/dashboards/provider.yaml': GRAFANA_DASH_PROVIDER,
+  'grafana/dashboards/ddo-logs.json': DASHBOARD,
+  // Opt-in tunnel overlay (see renderBundle lokiTunnelToken).
+  'caddy/Caddyfile': CADDY_CONFIG,
+  'docker-compose.tunnel.yml': TUNNEL_COMPOSE,
+};
+
+/**
+ * Resolve the bundle for a given client. Pure: same inputs -> same file map.
+ * @param {{gatewayUrl:string, triggerSecret:string, environment?:string,
+ *          lokiTunnelToken?:string}} opts
+ * @returns {Record<string,string>} relative path -> final file content
+ */
+export function renderBundle({ gatewayUrl, triggerSecret, environment = 'production',
+                              lokiTunnelToken = '' } = {}) {
+  if (!gatewayUrl || !triggerSecret) {
+    throw new Error('renderBundle requires gatewayUrl and triggerSecret');
+  }
+  const url = String(gatewayUrl).replace(/\/+$/, ''); // strip trailing slashes
+  const out = {};
+  for (const [path, tmpl] of Object.entries(BUNDLE_TEMPLATES)) {
+    out[path] = tmpl
+      .split('__GATEWAY_URL__').join(url)
+      .split('__TRIGGER_SECRET__').join(triggerSecret)
+      .split('__ENVIRONMENT__').join(environment)
+      .split('__LOKI_TUNNEL_TOKEN__').join(lokiTunnelToken);
+  }
+  return out;
+}

package/src/observability/detect.js

@@ -0,0 +1,151 @@
+/**
+ * observability/detect.js
+ *
+ * "Does this client already have observability?" If not, the agent offers to
+ * provision the DeepDebug bundle. The classifier is a PURE function over the
+ * host's running containers + processes so it can be unit-tested without Docker.
+ *
+ * We only ever recommend provisioning when we find NOTHING that already ships
+ * logs/metrics somewhere. Finding a real stack (Datadog, ELK, an existing
+ * Grafana, an otel-collector, ...) means the client has observability and we
+ * must NOT bulldoze it -- we report what we found and recommend 'skip'.
+ */
+
+import { exec } from 'child_process';
+import { promisify } from 'util';
+import { COMPOSE_PROJECT } from './bundle-files.js';
+
+const execAsync = promisify(exec);
+
+/**
+ * Known observability signals. Matched (case-insensitively, as substrings)
+ * against container image/name AND process command lines. Deliberately broad:
+ * a false "they have it" is safe (we skip); a false "they have nothing" is not
+ * (we would deploy on top of an existing stack).
+ */
+export const OBSERVABILITY_SIGNALS = [
+  // log shippers / agents
+  'promtail', 'fluent-bit', 'fluentbit', 'fluentd', 'vector', 'filebeat',
+  'logstash', 'otelcol', 'otel-collector', 'opentelemetry', 'telegraf',
+  // stores / backends
+  'loki', 'elasticsearch', 'opensearch', 'victoriametrics', 'victoria-metrics',
+  'prometheus', 'thanos', 'mimir', 'signoz', 'clickhouse-observability',
+  // UIs
+  'grafana', 'kibana',
+  // SaaS agents
+  'datadog', 'dd-agent', 'newrelic', 'new-relic', 'splunk', 'dynatrace',
+  'sematext', 'sumologic', 'sumo-logic', 'instana',
+];
+
+/**
+ * Classify whether the host already has observability.
+ *
+ * @param {{containers?: Array<{name?:string,image?:string,project?:string}>,
+ *          processes?: string[]}} input
+ * @returns {{hasObservability:boolean, found:string[], ours:string[],
+ *            recommendation:'provision'|'skip'}}
+ */
+export function classifyObservability({ containers = [], processes = [] } = {}) {
+  const found = new Set();
+  const ours = new Set();
+
+  const scan = (haystack, sourceLabel) => {
+    if (!haystack) return;
+    const h = String(haystack).toLowerCase();
+    for (const sig of OBSERVABILITY_SIGNALS) {
+      if (h.includes(sig)) found.add(sig);
+    }
+  };
+
+  for (const c of containers) {
+    // Our own stack must never count as "the client already has observability".
+    if (c && c.project && c.project === COMPOSE_PROJECT) {
+      if (c.name) ours.add(c.name);
+      continue;
+    }
+    scan(c && c.name);
+    scan(c && c.image);
+  }
+  for (const p of processes) scan(p);
+
+  const hits = [...found];
+  return {
+    hasObservability: hits.length > 0,
+    found: hits.sort(),
+    ours: [...ours].sort(),
+    recommendation: hits.length > 0 ? 'skip' : 'provision',
+  };
+}
+
+/** True when OUR compose stack is already running on the host. */
+export function ownStackRunning(containers = []) {
+  return containers.some((c) => c && c.project === COMPOSE_PROJECT);
+}
+
+/**
+ * Read running containers via `docker ps`. Returns [] (not throw) when Docker
+ * is absent/unreachable, so detection degrades to "process scan only".
+ */
+export async function listContainers() {
+  try {
+    // Tab-separated so names/images with spaces stay parseable.
+    const fmt = '{{.Names}}\t{{.Image}}\t{{.Label "com.docker.compose.project"}}';
+    const { stdout } = await execAsync(`docker ps --no-trunc --format '${fmt}'`, { timeout: 15000 });
+    return stdout
+      .split('\n')
+      .map((l) => l.trim())
+      .filter(Boolean)
+      .map((line) => {
+        const [name = '', image = '', project = ''] = line.split('\t');
+        return { name, image, project };
+      });
+  } catch {
+    return [];
+  }
+}
+
+/** Best-effort process list (Linux/macOS). Returns [] on failure. */
+export async function listProcesses() {
+  try {
+    const { stdout } = await execAsync('ps -eo args=', { timeout: 10000 });
+    return stdout.split('\n').map((l) => l.trim()).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/** Whether `docker` (and the compose plugin) are usable on this host. */
+export async function dockerAvailable() {
+  try {
+    await execAsync('docker version --format "{{.Server.Version}}"', { timeout: 15000 });
+  } catch {
+    return { docker: false, compose: false };
+  }
+  let compose = false;
+  try {
+    await execAsync('docker compose version', { timeout: 15000 });
+    compose = true;
+  } catch {
+    compose = false;
+  }
+  return { docker: true, compose };
+}
+
+/**
+ * Full host observability probe: containers + processes + docker availability,
+ * classified. This is what the /observability/status route returns.
+ */
+export async function detectObservability() {
+  const [containers, processes, docker] = await Promise.all([
+    listContainers(),
+    listProcesses(),
+    dockerAvailable(),
+  ]);
+  const classification = classifyObservability({ containers, processes });
+  return {
+    ...classification,
+    ownStackRunning: ownStackRunning(containers),
+    docker,
+    containersScanned: containers.length,
+  };
+}