@arcote.tech/arc-cli 0.7.5 → 0.7.6

package/src/deploy/observability-configs.ts

@@ -0,0 +1,293 @@
+import type { DeployConfig, DeployObservability } from "./config";
+
+// ---------------------------------------------------------------------------
+// Observability stack config templates.
+//
+// All strings are deterministic for the inputs (cfg + retention) — no random
+// IDs, no timestamps — so re-running deploy with unchanged config is a no-op
+// at the file-write level. Bootstrap diffs filesystem before bouncing
+// services, so this matters.
+//
+// Defaults:
+//   - traces:  7d  retention (Tempo block storage on local disk)
+//   - logs:    7d  retention (Loki chunks on local disk)
+//   - metrics: 30d retention (Prometheus TSDB on local disk)
+//
+// Tail sampling: every error + every span >500ms + 10% random. Decided in
+// the collector so per-service SDKs can be left at always-on without
+// flooding the backend.
+// ---------------------------------------------------------------------------
+
+const DEFAULT_RETENTION = {
+  traces: "168h", // 7d
+  logs: "168h",
+  metrics: "30d",
+} as const;
+
+function pickRetention(o: DeployObservability | undefined) {
+  return {
+    traces: o?.retention?.traces ?? DEFAULT_RETENTION.traces,
+    logs: o?.retention?.logs ?? DEFAULT_RETENTION.logs,
+    metrics: o?.retention?.metrics ?? DEFAULT_RETENTION.metrics,
+  };
+}
+
+/** OpenTelemetry Collector — receives OTLP from app containers + browser,
+ *  applies tail sampling, fans out to Tempo (traces), Loki (logs),
+ *  Prometheus remote-write (metrics). */
+export function generateOtelCollectorConfig(cfg: DeployConfig): string {
+  const envNames = Object.keys(cfg.envs);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+        cors:
+          allowed_origins:
+${envNames.map((name) => `            - "https://${cfg.envs[name]!.domain}"`).join("\n")}
+          allowed_headers:
+            - traceparent
+            - tracestate
+            - content-type
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 512
+    send_batch_max_size: 1024
+
+  # Tail-based sampling — applied after a full trace has been assembled.
+  # Errors and slow traces are kept 100%, everything else at 10%.
+  tail_sampling:
+    decision_wait: 10s
+    num_traces: 50000
+    expected_new_traces_per_sec: 100
+    policies:
+      - name: errors
+        type: status_code
+        status_code: { status_codes: [ERROR] }
+      - name: slow
+        type: latency
+        latency: { threshold_ms: 500 }
+      - name: random_10pct
+        type: probabilistic
+        probabilistic: { sampling_percentage: 10 }
+
+  # Drop high-cardinality / PII attributes that might slip past app-side
+  # sanitization. Belt-and-suspenders before they hit long-term storage.
+  attributes:
+    actions:
+      - key: http.request.header.authorization
+        action: delete
+      - key: http.request.header.cookie
+        action: delete
+
+exporters:
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+  otlphttp/loki:
+    endpoint: http://loki:3100/otlp
+    tls:
+      insecure: true
+
+  prometheusremotewrite:
+    endpoint: http://prometheus:9090/api/v1/write
+    tls:
+      insecure: true
+
+extensions:
+  health_check: {}
+  zpages: {}
+
+service:
+  extensions: [health_check, zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [tail_sampling, attributes, batch]
+      exporters: [otlp/tempo]
+    logs:
+      receivers: [otlp]
+      processors: [attributes, batch]
+      exporters: [otlphttp/loki]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheusremotewrite]
+`;
+}
+
+/** Grafana Tempo — single-binary mode with local block storage. */
+export function generateTempoConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+server:
+  http_listen_port: 3200
+  grpc_listen_port: 9095
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 1048576
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: ${retention.traces}
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics]
+`;
+}
+
+/** Loki — single-binary mode, filesystem chunks. */
+export function generateLokiConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  retention_period: ${retention.logs}
+  allow_structured_metadata: true
+
+compactor:
+  working_directory: /loki/compactor
+  retention_enabled: true
+  delete_request_store: filesystem
+`;
+}
+
+/** Prometheus — accepts remote_write from the collector, scrapes itself. */
+export function generatePrometheusConfig(cfg: DeployConfig): string {
+  const retention = pickRetention(cfg.observability);
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: [localhost:9090]
+  - job_name: otel-collector
+    static_configs:
+      - targets: [otel-collector:8888]
+
+storage:
+  tsdb:
+    retention.time: ${retention.metrics}
+
+# Note: remote-write inbound is enabled via the --web.enable-remote-write-receiver
+# command-line flag (set in docker-compose), not here.
+`;
+}
+
+/** Grafana datasource provisioning — Tempo + Loki + Prometheus, all pre-wired. */
+export function generateGrafanaDatasources(): string {
+  return `# Generated by \`arc platform deploy\` — do not edit by hand.
+apiVersion: 1
+datasources:
+  - name: Tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200
+    uid: tempo
+    jsonData:
+      tracesToLogsV2:
+        datasourceUid: loki
+        spanStartTimeShift: -5m
+        spanEndTimeShift: 5m
+      serviceMap:
+        datasourceUid: prometheus
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    uid: loki
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: "trace_id=(\\\\w+)"
+          name: TraceID
+          url: $\${__value.raw}
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    uid: prometheus
+    isDefault: true
+`;
+}
+
+/** All config files needed on the host. Returns map of relative-path → contents
+ *  so bootstrap can write+upload them in one pass. */
+export function generateObservabilityConfigs(
+  cfg: DeployConfig,
+): Record<string, string> {
+  return {
+    "observability/otel-collector-config.yaml": generateOtelCollectorConfig(cfg),
+    "observability/tempo.yaml": generateTempoConfig(cfg),
+    "observability/loki-config.yaml": generateLokiConfig(cfg),
+    "observability/prometheus.yml": generatePrometheusConfig(cfg),
+    "observability/grafana-datasources.yaml": generateGrafanaDatasources(),
+  };
+}

package/src/platform/server.ts

@@ -11,7 +11,7 @@ import {
 import { existsSync, mkdirSync } from "fs";
 import { join } from "path";
 import { readTranslationsConfig } from "../i18n";
-import type { BuildManifest, WorkspaceInfo } from "./shared";
+import { err, ok, type BuildManifest, type WorkspaceInfo } from "./shared";
 import type { BuildManifestGroup, ModuleAccess } from "@arcote.tech/platform";
 
 // ---------------------------------------------------------------------------
@@ -97,6 +97,21 @@ export function generateShellHtml(
   initial?: { file: string; hash: string },
   stylesHash?: string,
 ): string {
+  // OpenTelemetry config — injected as a global so the browser SDK chunk
+  // (lazy-loaded by start-app.ts) can pick it up without a fetch. Endpoint
+  // is same-origin so Caddy can apply CORS + auth uniformly.
+  const otelConfig = process.env.ARC_OTEL_ENABLED === "true"
+    ? {
+        enabled: true,
+        endpoint: "/otel",
+        serviceName: process.env.OTEL_SERVICE_NAME ?? `${appName}-browser`,
+        environment: process.env.NODE_ENV === "production" ? "production" : "development",
+        sampleRate: Number(process.env.ARC_OTEL_BROWSER_SAMPLE_RATE ?? "0.1"),
+      }
+    : null;
+  const otelTag = otelConfig
+    ? `\n  <script>window.__ARC_OTEL_CONFIG=${JSON.stringify(otelConfig)};</script>`
+    : "";
   // Initial bundle carries framework, public modules, and PlatformApp re-export.
   // No importmap — single Bun.build with splitting:true inlines + dedups everything
   // across initial and per-token group bundles via auto-emitted chunk-<hash>.js.
@@ -117,7 +132,7 @@ export function generateShellHtml(
   <title>${manifest?.title ?? appName}</title>${manifest?.favicon ? `\n  <link rel="icon" href="${manifest.favicon}">` : ""}${manifest ? `\n  <link rel="manifest" href="/manifest.json">` : ""}
   <link rel="stylesheet" href="/styles.css${stylesQs}" />
   <link rel="stylesheet" href="/theme.css${stylesQs}" />
-  <link rel="modulepreload" href="${initialUrl}" />
+  <link rel="modulepreload" href="${initialUrl}" />${otelTag}
 </head>
 <body>
   <div id="root"></div>
@@ -480,6 +495,30 @@ export async function startPlatformServer(
 ): Promise<PlatformServer> {
   const { ws, port, devMode, context } = opts;
   ensureModuleSigSecret(ws, !!devMode);
+
+  // OpenTelemetry — only when explicitly enabled (deploy injects the env
+  // when `observability.enabled` is set in deploy.arc.json). Dynamic import
+  // keeps the OTel SDK out of bundles that don't use it.
+  let telemetry: import("@arcote.tech/arc-otel").ArcTelemetry | undefined;
+  let telemetryShutdown: (() => Promise<void>) | undefined;
+  if (process.env.ARC_OTEL_ENABLED === "true") {
+    try {
+      const { initServerTelemetry } = await import("@arcote.tech/arc-otel/server");
+      const init = initServerTelemetry({
+        serviceName: process.env.OTEL_SERVICE_NAME ?? ws.appName,
+        environment: "server",
+        endpoint: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+        mode: devMode ? "development" : "production",
+        sampleRate: devMode ? 1.0 : 1.0, // head-based 100%, collector tail-samples
+      });
+      telemetry = init.telemetry;
+      telemetryShutdown = init.shutdown;
+      ok("Telemetry enabled — exporting to " + process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+    } catch (e) {
+      err(`Failed to init telemetry: ${(e as Error).message}`);
+    }
+  }
+
   const moduleAccessMap = opts.moduleAccess ?? new Map();
   let manifest = opts.manifest;
   const getManifest = () => manifest;
@@ -575,6 +614,7 @@ export async function startPlatformServer(
       spaFallbackHandler(getShellHtml),
     ],
     onWsClose: (clientId) => cleanupClientSubs(clientId),
+    telemetry,
   });
 
   return {
@@ -583,6 +623,9 @@ export async function startPlatformServer(
     connectionManager: arcServer.connectionManager,
     setManifest,
     notifyReload,
-    stop: () => arcServer.stop(),
+    stop: () => {
+      arcServer.stop();
+      void telemetryShutdown?.();
+    },
   };
 }