@envsync-cloud/deploy-cli 0.6.10 → 0.6.12

package/README.md

@@ -81,9 +81,16 @@ npx @envsync-cloud/deploy-cli deploy
 
 The staged flow is:
 - `setup` writes desired config
-- `bootstrap` resets the existing EnvSync deployment, then starts base infra, runs OpenFGA and miniKMS migrations, starts runtime infra, and persists generated runtime env state
+- `bootstrap` resets the existing EnvSync deployment, then starts base infra, runs OpenFGA and miniKMS migrations, starts runtime infra, initializes ClickStack sources and dashboards, and persists generated runtime env state
 - `deploy` starts the pending API and frontend services
 
+Self-hosted observability routing is:
+- `https://obs.<root-domain>/` for ClickStack UI
+- `https://obs.<root-domain>/api/...` for ClickStack API
+- `https://obs.<root-domain>/v1/{traces,logs,metrics}` for browser OTLP
+
+Both frontends receive `otelEndpoint = https://obs.<root-domain>` in the generated `runtime-config.js`.
+
 Check service health:
 
 ```bash

package/dist/index.js

@@ -25,6 +25,7 @@ var KEYCLOAK_REALM_FILE = "/opt/envsync/deploy/keycloak-realm.envsync.json";
 var NGINX_WEB_CONF = "/opt/envsync/deploy/nginx-web.conf";
 var NGINX_LANDING_CONF = "/opt/envsync/deploy/nginx-landing.conf";
 var OTEL_AGENT_CONF = "/opt/envsync/deploy/otel-agent.yaml";
+var CLICKSTACK_CLICKHOUSE_CONF = "/opt/envsync/deploy/clickhouse-listen.xml";
 var INTERNAL_CONFIG_JSON = "/opt/envsync/deploy/config.json";
 var STACK_VOLUMES = [
   "postgres_data",
@@ -611,6 +612,19 @@ function renderTraefikDynamicConfig(config) {
     "        stsSeconds: 31536000",
     "    gzip:",
     "      compress: {}",
+    "    otel-cors:",
+    "      headers:",
+    "        accessControlAllowOriginList:",
+    `          - https://${hosts.landing}`,
+    `          - https://${hosts.app}`,
+    "        accessControlAllowMethods:",
+    "          - POST",
+    "          - OPTIONS",
+    "        accessControlAllowHeaders:",
+    "          - Content-Type",
+    "          - Authorization",
+    "        accessControlAllowCredentials: false",
+    "        addVaryHeader: true",
     "  services:",
     "    envsync-api:",
     "      weighted:",
@@ -635,7 +649,15 @@ function renderTraefikDynamicConfig(config) {
     "      loadBalancer:",
     "        servers:",
     "          - url: http://web_nginx:8080",
-    "    browser-otlp:",
+    "    clickstack-ui:",
+    "      loadBalancer:",
+    "        servers:",
+    "          - url: http://clickstack:8080",
+    "    clickstack-api:",
+    "      loadBalancer:",
+    "        servers:",
+    "          - url: http://clickstack:8000",
+    "    clickstack-otlp:",
     "      loadBalancer:",
     "        servers:",
     `          - url: http://clickstack:${config.services.clickstack_otlp_http_port}`,
@@ -650,16 +672,23 @@ function renderTraefikDynamicConfig(config) {
     "      service: web",
     "      entryPoints: [websecure]",
     "      tls: {}",
-    "    landing-otlp-router:",
-    `      rule: Host(\`${hosts.landing}\`) && (PathPrefix(\`/v1/traces\`) || PathPrefix(\`/v1/logs\`) || PathPrefix(\`/v1/metrics\`))`,
-    "      service: browser-otlp",
+    "    obs-otlp-router:",
+    `      rule: Host(\`${hosts.obs}\`) && (PathPrefix(\`/v1/traces\`) || PathPrefix(\`/v1/logs\`) || PathPrefix(\`/v1/metrics\`))`,
+    "      service: clickstack-otlp",
+    "      middlewares: [otel-cors]",
     "      priority: 100",
     "      entryPoints: [websecure]",
     "      tls: {}",
-    "    web-otlp-router:",
-    `      rule: Host(\`${hosts.app}\`) && (PathPrefix(\`/v1/traces\`) || PathPrefix(\`/v1/logs\`) || PathPrefix(\`/v1/metrics\`))`,
-    "      service: browser-otlp",
-    "      priority: 100",
+    "    obs-api-router:",
+    `      rule: Host(\`${hosts.obs}\`) && PathPrefix(\`/api\`)`,
+    "      service: clickstack-api",
+    "      priority: 90",
+    "      entryPoints: [websecure]",
+    "      tls: {}",
+    "    obs-ui-router:",
+    `      rule: Host(\`${hosts.obs}\`)`,
+    "      service: clickstack-ui",
+    "      priority: 10",
     "      entryPoints: [websecure]",
     "      tls: {}",
     "    api-router:",
@@ -684,7 +713,7 @@ function renderNginxConf(kind) {
 }
 function renderFrontendRuntimeConfig(config, kind) {
   const hosts = domainMap(config.domain.root_domain);
-  const otelEndpoint = kind === "web" ? `https://${hosts.app}` : `https://${hosts.landing}`;
+  const otelEndpoint = `https://${hosts.obs}`;
   return `window.__ENVSYNC_RUNTIME_CONFIG__ = ${JSON.stringify({
     apiBaseUrl: `https://${hosts.api}`,
     appBaseUrl: `https://${hosts.app}`,
@@ -729,10 +758,23 @@ function renderOtelAgentConfig(config) {
     "      exporters: [otlphttp/clickstack]"
   ].join("\n") + "\n";
 }
+function renderClickstackClickHouseConfig() {
+  return [
+    "<clickhouse>",
+    "  <listen_host>0.0.0.0</listen_host>",
+    "  <listen_try>1</listen_try>",
+    "</clickhouse>"
+  ].join("\n") + "\n";
+}
 function renderStack(config, runtimeEnv, mode) {
   const hosts = domainMap(config.domain.root_domain);
   const includeRuntimeInfra = mode !== "base";
   const includeAppServices = mode === "full";
+  const stackName = config.services.stack_name;
+  const s3RouterName = `${stackName}-s3-router`;
+  const s3ServiceName = `${stackName}-s3-service`;
+  const s3ConsoleRouterName = `${stackName}-s3-console-router`;
+  const s3ConsoleServiceName = `${stackName}-s3-console-service`;
   const apiEnvironment = {
     ...runtimeEnv,
     OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-agent:4318",
@@ -809,16 +851,16 @@ ${renderEnvList({
     deploy:
       labels:
         - traefik.enable=true
-        - traefik.http.routers.s3.rule=Host(\`${hosts.s3}\`)
-        - traefik.http.routers.s3.entrypoints=websecure
-        - traefik.http.routers.s3.tls.certresolver=letsencrypt
-        - traefik.http.routers.s3.service=s3
-        - traefik.http.services.s3.loadbalancer.server.port=9000
-        - traefik.http.routers.s3-console.rule=Host(\`${hosts.s3Console}\`)
-        - traefik.http.routers.s3-console.entrypoints=websecure
-        - traefik.http.routers.s3-console.tls.certresolver=letsencrypt
-        - traefik.http.routers.s3-console.service=s3-console
-        - traefik.http.services.s3-console.loadbalancer.server.port=9001
+        - traefik.http.routers.${s3RouterName}.rule=Host(\`${hosts.s3}\`)
+        - traefik.http.routers.${s3RouterName}.entrypoints=websecure
+        - traefik.http.routers.${s3RouterName}.tls.certresolver=letsencrypt
+        - traefik.http.routers.${s3RouterName}.service=${s3ServiceName}
+        - traefik.http.services.${s3ServiceName}.loadbalancer.server.port=9000
+        - traefik.http.routers.${s3ConsoleRouterName}.rule=Host(\`${hosts.s3Console}\`)
+        - traefik.http.routers.${s3ConsoleRouterName}.entrypoints=websecure
+        - traefik.http.routers.${s3ConsoleRouterName}.tls.certresolver=letsencrypt
+        - traefik.http.routers.${s3ConsoleRouterName}.service=${s3ConsoleServiceName}
+        - traefik.http.services.${s3ConsoleServiceName}.loadbalancer.server.port=9001
 
   keycloak_db:
     image: postgres:17
@@ -911,18 +953,26 @@ ${renderEnvList({
 
   clickstack:
     image: ${config.images.clickstack}
+    environment:
+${renderEnvList({
+    HYPERDX_APP_URL: `https://${hosts.obs}`,
+    HYPERDX_APP_PORT: "443",
+    HYPERDX_API_URL: `https://${hosts.obs}`,
+    HYPERDX_API_PORT: "443",
+    FRONTEND_URL: `https://${hosts.obs}`
+  })}
     volumes:
       - clickstack_data:/data/db
       - clickstack_ch_data:/var/lib/clickhouse
       - clickstack_ch_logs:/var/log/clickhouse-server
+      - ${CLICKSTACK_CLICKHOUSE_CONF}:/etc/clickhouse-server/config.d/envsync-listen-host.xml:ro
     networks: [envsync]
-    deploy:
-      labels:
-        - traefik.enable=true
-        - traefik.http.routers.obs.rule=Host(\`${hosts.obs}\`)
-        - traefik.http.routers.obs.entrypoints=websecure
-        - traefik.http.routers.obs.tls.certresolver=letsencrypt
-        - traefik.http.services.obs.loadbalancer.server.port=8080
+    healthcheck:
+      test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080 || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 10
+      start_period: 180s
 
   otel-agent:
     image: ${config.images.otel_agent}
@@ -992,6 +1042,7 @@ function writeDeployArtifacts(config, generated) {
   writeFileMaybe(NGINX_WEB_CONF, renderNginxConf("web"));
   writeFileMaybe(NGINX_LANDING_CONF, renderNginxConf("landing"));
   writeFileMaybe(OTEL_AGENT_CONF, renderOtelAgentConfig(config));
+  writeFileMaybe(CLICKSTACK_CLICKHOUSE_CONF, renderClickstackClickHouseConfig());
   logSuccess(currentOptions.dryRun ? "Deploy artifacts previewed" : "Deploy artifacts written");
 }
 function saveDesiredConfig(config) {
@@ -1265,6 +1316,32 @@ function runBootstrapInit(config) {
     openfgaModelId: result.openfgaModelId
   };
 }
+function runClickstackBootstrap(config) {
+  logStep("Running ClickStack self-host bootstrap");
+  if (currentOptions.dryRun) {
+    logDryRun("Would bootstrap ClickStack sources and dashboards");
+    logCommand("node", [path.join(REPO_ROOT, "scripts/bootstrap-clickstack-selfhost.mjs")]);
+    return;
+  }
+  const deadline = Date.now() + 180 * 1e3;
+  let lastError = "unknown error";
+  while (Date.now() < deadline) {
+    try {
+      run("node", [path.join(REPO_ROOT, "scripts/bootstrap-clickstack-selfhost.mjs")], {
+        env: {
+          ENVSYNC_STACK_NAME: config.services.stack_name,
+          ENVSYNC_ROOT_DOMAIN: config.domain.root_domain
+        }
+      });
+      logSuccess("ClickStack self-host bootstrap completed");
+      return;
+    } catch (error) {
+      lastError = error instanceof Error ? error.message : String(error);
+      sleepSeconds(3);
+    }
+  }
+  throw new Error(`Timed out bootstrapping ClickStack: ${lastError}`);
+}
 function parseBootstrapInitJson(output) {
   const trimmed = output.trim();
   if (!trimmed) {
@@ -1591,7 +1668,20 @@ async function cmdBootstrap() {
   waitForHttpService(config, "keycloak management readiness", "http://keycloak:9000/health/ready", 180);
   waitForHttpService(config, "openfga", "http://openfga:8090/stores");
   waitForTcpService(config, "minikms", "minikms", 50051);
+  waitForHttpService(config, "clickstack ui", "http://clickstack:8080", 180);
   const initResult = runBootstrapInit(config);
+  const persistedGenerated = normalizeGeneratedState({
+    openfga: {
+      store_id: initResult.openfgaStoreId,
+      model_id: initResult.openfgaModelId
+    },
+    secrets: nextGenerated.secrets,
+    bootstrap: nextGenerated.bootstrap
+  });
+  if (!currentOptions.dryRun) {
+    writeDeployArtifacts(config, persistedGenerated);
+  }
+  runClickstackBootstrap(config);
   if (currentOptions.dryRun) {
     logDryRun("Skipping generated OpenFGA ID persistence in preview mode");
     logSuccess("Bootstrap dry-run completed");
@@ -1654,6 +1744,7 @@ async function cmdHealth(asJson) {
   const hosts = domainMap(config.domain.root_domain);
   const services = listStackServices(config);
   const stackName = config.services.stack_name;
+  const traefikDynamic = exists(TRAEFIK_DYNAMIC_FILE) ? fs.readFileSync(TRAEFIK_DYNAMIC_FILE, "utf8") : "";
   const bootstrapServices = {
     postgres: serviceHealth(services, `${stackName}_postgres`),
     redis: serviceHealth(services, `${stackName}_redis`),
@@ -1673,6 +1764,25 @@ async function cmdHealth(asJson) {
       web: serviceHealth(services, `${stackName}_web_nginx`),
       landing: serviceHealth(services, `${stackName}_landing_nginx`)
     },
+    observability: {
+      service: serviceHealth(services, `${stackName}_clickstack`),
+      obs_ui: {
+        url: `https://${hosts.obs}`,
+        configured: traefikDynamic.includes("obs-ui-router")
+      },
+      obs_api: {
+        url: `https://${hosts.obs}/api`,
+        configured: traefikDynamic.includes("obs-api-router")
+      },
+      obs_otlp: {
+        url: `https://${hosts.obs}/v1/traces`,
+        configured: traefikDynamic.includes("obs-otlp-router")
+      },
+      frontend_otel_endpoint: {
+        web: `https://${hosts.obs}`,
+        landing: `https://${hosts.obs}`
+      }
+    },
     public: {
       landing: `https://${hosts.landing}`,
       app: `https://${hosts.app}`,
@@ -1794,6 +1904,7 @@ async function cmdBackup() {
   writeFile(path.join(staged, "traefik-dynamic.yaml"), fs.readFileSync(TRAEFIK_DYNAMIC_FILE, "utf8"));
   writeFile(path.join(staged, "keycloak-realm.envsync.json"), fs.readFileSync(KEYCLOAK_REALM_FILE, "utf8"));
   writeFile(path.join(staged, "otel-agent.yaml"), fs.readFileSync(OTEL_AGENT_CONF, "utf8"));
+  writeFile(path.join(staged, "clickhouse-listen.xml"), fs.readFileSync(CLICKSTACK_CLICKHOUSE_CONF, "utf8"));
   const volumesDir = path.join(staged, "volumes");
   for (const volume of STACK_VOLUMES) {
     backupDockerVolume(stackVolumeName(config, volume), path.join(volumesDir, volume));
@@ -1834,6 +1945,7 @@ async function cmdRestore(archivePath) {
   writeFile(TRAEFIK_DYNAMIC_FILE, fs.readFileSync(path.join(restoreRoot, "traefik-dynamic.yaml"), "utf8"));
   writeFile(KEYCLOAK_REALM_FILE, fs.readFileSync(path.join(restoreRoot, "keycloak-realm.envsync.json"), "utf8"));
   writeFile(OTEL_AGENT_CONF, fs.readFileSync(path.join(restoreRoot, "otel-agent.yaml"), "utf8"));
+  writeFile(CLICKSTACK_CLICKHOUSE_CONF, fs.readFileSync(path.join(restoreRoot, "clickhouse-listen.xml"), "utf8"));
   const config = loadConfig();
   for (const volume of STACK_VOLUMES) {
     restoreDockerVolume(stackVolumeName(config, volume), path.join(restoreRoot, "volumes", volume));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@envsync-cloud/deploy-cli",
-  "version": "0.6.10",
+  "version": "0.6.12",
   "description": "CLI for self-hosted EnvSync deployment on Docker Swarm",
   "type": "module",
   "bin": {