openmates 0.24.0-alpha.0 → 0.24.0-alpha.2

package/dist/{chunk-2KYUZC7W.js → chunk-VEQ3R7MK.js}

@@ -32710,6 +32710,7 @@ var RUNTIME_CHECKS = {
     { id: "http.role_health", required: true, timeoutSeconds: 10 },
     { id: "core.database", required: true, timeoutSeconds: 10 },
     { id: "core.cache", required: true, timeoutSeconds: 10 },
+    { id: "core.cms_cache_consistency", required: true, timeoutSeconds: 10 },
     { id: "core.vault", required: true, timeoutSeconds: 10 },
     { id: "core.worker_queue", required: true, timeoutSeconds: CELERY_PROBE_CHECK_TIMEOUT_SECONDS },
     { id: "core.scheduler_freshness", required: true, timeoutSeconds: CELERY_PROBE_CHECK_TIMEOUT_SECONDS },
@@ -32981,6 +32982,19 @@ function planRestore(input) {
     steps: input.yes === true ? ["stop", "restore", "start", "health-check"] : ["confirm", "stop", "restore", "start", "health-check"]
   };
 }
+function resolveTemplateSource(input) {
+  const role = parseServerRole(input.role);
+  const definition = ROLE_DEFINITIONS[role];
+  if (input.templateUrl) return { type: "url", url: input.templateUrl };
+  if (input.packagedTemplateExists && input.packageVersion && input.imageTag === `v${input.packageVersion}`) {
+    return { type: "packaged", path: definition.templatePath };
+  }
+  return {
+    type: "github-raw",
+    ref: input.templateRef ?? "dev",
+    path: `frontend/packages/openmates-cli/${definition.templatePath}`
+  };
+}
 function parseSecretEnvKey(envKey) {
   if (!envKey.startsWith("SECRET__")) return null;
   const parts = envKey.slice("SECRET__".length).split("__", 2);
@@ -33991,6 +34005,18 @@ async function deliverRuntimeNotification(config, payload) {
   }
   return Promise.all(deliveries);
 }
+var CMS_CACHE_INSPECT_FORMAT = '{{range .Config.Env}}{{if eq . "CACHE_SKIP_ALLOWED=true"}}CACHE_SKIP_ALLOWED=true,{{end}}{{if eq . "CACHE_AUTO_PURGE=true"}}CACHE_AUTO_PURGE=true,{{end}}{{end}}';
+function evaluateCmsCacheConsistency(input) {
+  const result = { id: "core.cms_cache_consistency", required: true, duration_ms: 0 };
+  if (!input.containerFound || !input.inspectionSucceeded) {
+    return { ...result, status: "failed", failureClass: "configuration", sanitized_reason: "cms_cache_inspection_unavailable" };
+  }
+  const flags = new Set(input.filteredEnvironment.split(",").map((value) => value.trim()));
+  if (!flags.has("CACHE_SKIP_ALLOWED=true") || !flags.has("CACHE_AUTO_PURGE=true")) {
+    return { ...result, status: "failed", failureClass: "configuration", sanitized_reason: "cms_cache_flags_missing_or_disabled" };
+  }
+  return { ...result, status: "passed" };
+}
 function applyRuntimeCheckResults(state, results, timestamp) {
   const current = state ?? initialRuntimeIncidentState();
   const checks = { ...current.checks ?? {} };
@@ -34369,6 +34395,7 @@ var ROLE_TEMPLATE_FILES = {
 var CORE_NO_WEBAPP_TEMPLATE_FILE = join15("core", "docker-compose.no-webapp.yml");
 var CORE_PROMTAIL_CONFIG_FILE = join15("backend", "core", "monitoring", "promtail", "promtail-config.yaml");
 var CORE_ALERTMANAGER_CONFIG_FILE = join15("backend", "core", "monitoring", "alertmanager", "alertmanager.yml");
+var CORE_PROMETHEUS_CONFIG_DIR = join15("backend", "core", "monitoring", "prometheus");
 var COMPOSE_OVERRIDE2 = join15("backend", "core", "docker-compose.override.yml");
 var DEFAULT_INSTALL_PATH = join15(homedir6(), "openmates");
 var REPO_URL = "https://github.com/glowingkitty/OpenMates.git";
@@ -34931,6 +34958,19 @@ function packagedCaddyTemplatePath(role) {
 function packagedCoreAlertmanagerTemplatePath() {
   return join15(dirname8(new URL(import.meta.url).pathname), "..", "templates", "core", "monitoring", "alertmanager", "alertmanager.yml");
 }
+function ensureCorePrometheusRuntimeFiles(installPath) {
+  const packagedDir = join15(dirname8(new URL(import.meta.url).pathname), "..", "templates", "core", "monitoring", "prometheus");
+  const runtimeDir = join15(installPath, CORE_PROMETHEUS_CONFIG_DIR);
+  for (const fileName of ["prometheus.yml", "alert_rules.yml"]) {
+    const packagedPath = join15(packagedDir, fileName);
+    if (!existsSync13(packagedPath)) throw new Error(`Packaged Prometheus file not found: ${packagedPath}`);
+  }
+  mkdirSync13(runtimeDir, { recursive: true });
+  for (const fileName of ["prometheus.yml", "alert_rules.yml"]) {
+    const runtimePath = join15(runtimeDir, fileName);
+    if (!existsSync13(runtimePath)) copyFileSync2(join15(packagedDir, fileName), runtimePath);
+  }
+}
 function readOfficialCloudNoWebappComposeTemplate() {
   const packaged = packagedNoWebappTemplatePath();
   if (existsSync13(packaged)) return readFileSync14(packaged, "utf-8");
@@ -34957,29 +34997,35 @@ function fileHash(path2) {
   if (!existsSync13(path2)) return null;
   return createHash18("sha256").update(readFileSync14(path2)).digest("hex");
 }
-async function loadSelfHostComposeTemplate(templateRef, role) {
+async function loadSelfHostComposeTemplate(templateRef, role, imageTag, packageVersion = getPackageVersion()) {
   const templateDir = process.env.OPENMATES_SELFHOST_TEMPLATE_DIR;
   if (templateDir) {
     return readFileSync14(join15(resolve7(templateDir), ROLE_TEMPLATE_FILES[role]), "utf-8");
   }
-  const overrideUrl = process.env.OPENMATES_SELFHOST_COMPOSE_URL;
-  if (overrideUrl) {
-    return fetchText(overrideUrl);
-  }
   const packaged = packagedTemplatePath(role);
-  if (existsSync13(packaged)) return readFileSync14(packaged, "utf-8");
-  return fetchText(`https://raw.githubusercontent.com/glowingkitty/OpenMates/${templateRef}/${ROLE_IMAGE_COMPOSE_FILES[role]}`);
+  const source = resolveTemplateSource({
+    role,
+    packagedTemplateExists: existsSync13(packaged),
+    templateUrl: process.env.OPENMATES_SELFHOST_COMPOSE_URL,
+    templateRef,
+    imageTag,
+    packageVersion
+  });
+  if (source.type === "packaged") return readFileSync14(packaged, "utf-8");
+  if (source.type === "url") return fetchText(source.url);
+  return fetchText(`https://raw.githubusercontent.com/glowingkitty/OpenMates/${source.ref}/${source.path}`);
 }
 async function writeImageModeRuntimeFiles(installPath, imageTag, role) {
   const roleDir = join15(installPath, "backend", role === "core" ? "core" : role);
   const vaultConfigDir = join15(roleDir, "vault", "config");
   mkdirSync13(vaultConfigDir, { recursive: true });
   mkdirSync13(join15(installPath, "config", "providers"), { recursive: true });
-  writeFileSync8(join15(installPath, ROLE_IMAGE_COMPOSE_FILES[role]), await loadSelfHostComposeTemplate(templateRefForImageTag(imageTag, getPackageVersion()), role));
+  writeFileSync8(join15(installPath, ROLE_IMAGE_COMPOSE_FILES[role]), await loadSelfHostComposeTemplate(templateRefForImageTag(imageTag, getPackageVersion()), role, imageTag));
   if (role === "core") {
     writeFileSync8(join15(installPath, OFFICIAL_CLOUD_NO_WEBAPP_COMPOSE_FILE), readOfficialCloudNoWebappComposeTemplate());
   }
   if (role === "core") {
+    ensureCorePrometheusRuntimeFiles(installPath);
     const promtailConfigPath = join15(installPath, CORE_PROMTAIL_CONFIG_FILE);
     mkdirSync13(dirname8(promtailConfigPath), { recursive: true });
     writeFileSync8(promtailConfigPath, SELFHOST_PROMTAIL_CONFIG_TEMPLATE);
@@ -36259,6 +36305,28 @@ async function installContinuousUpdateService(flags) {
   }
   console.log(`Installed ${plan.timerName}.`);
 }
+function inspectCmsCacheConsistency(installPath, withOverrides, installMode) {
+  const composeResult = spawnSync(
+    "docker",
+    [...composeArgs(installPath, withOverrides, installMode, "core"), "ps", "-q", "cms"],
+    { cwd: installPath, encoding: "utf-8", timeout: 5e3 }
+  );
+  const containerIds = composeResult.status === 0 ? composeResult.stdout.trim().split(/\s+/).filter(Boolean) : [];
+  const containerId = containerIds.length === 1 ? containerIds[0] : void 0;
+  if (!containerId) {
+    return evaluateCmsCacheConsistency({ containerFound: false, inspectionSucceeded: false, filteredEnvironment: "" });
+  }
+  const inspectResult = spawnSync(
+    "docker",
+    ["inspect", "--format", CMS_CACHE_INSPECT_FORMAT, containerId],
+    { cwd: installPath, encoding: "utf-8", timeout: 5e3 }
+  );
+  return evaluateCmsCacheConsistency({
+    containerFound: true,
+    inspectionSucceeded: inspectResult.status === 0,
+    filteredEnvironment: inspectResult.status === 0 ? inspectResult.stdout : ""
+  });
+}
 function runRuntimeVerification(installPath, role, config) {
   const envText = existsSync13(join15(installPath, ".env")) ? readFileSync14(join15(installPath, ".env"), "utf-8") : "";
   const mode = resolveRuntimeDeploymentMode({
@@ -36285,6 +36353,11 @@ function runRuntimeVerification(installPath, role, config) {
     throw new Error(`Runtime verifier returned invalid output (${result.status ?? "unknown"}).`);
   }
   output.checks = output.checks.map((check) => ({ ...check, failureClass: check.failureClass ?? check.failure_class }));
+  if (role === "core") {
+    const cmsCacheCheck = inspectCmsCacheConsistency(installPath, withOverrides, installMode);
+    output.checks.push(cmsCacheCheck);
+    if (cmsCacheCheck.status !== "passed") output.status = "failed";
+  }
   if (mode.effectiveMode === "official_cloud") {
     const destinations = runtimeNotificationConfig(installPath);
     const configuredCount = [destinations.email, destinations.discordWebhookUrl, destinations.genericWebhook].filter(Boolean).length;

package/dist/cli.js

@@ -19,7 +19,7 @@ import {
   serializeToYaml,
   shouldRequireTrustedAccountGuard,
   waitForWorkflowRun
-} from "./chunk-2KYUZC7W.js";
+} from "./chunk-VEQ3R7MK.js";
 import "./chunk-IWKP55ZB.js";
 import "./chunk-BBKJZ23O.js";
 export {

package/dist/index.js

@@ -43,7 +43,7 @@ import {
   selectAssistantMessagesForSpeech,
   serializeToYaml,
   summarizeAssistantSpeech
-} from "./chunk-2KYUZC7W.js";
+} from "./chunk-VEQ3R7MK.js";
 import "./chunk-IWKP55ZB.js";
 import "./chunk-BBKJZ23O.js";
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmates",
-  "version": "0.24.0-alpha.0",
+  "version": "0.24.0-alpha.2",
   "description": "OpenMates CLI and SDK",
   "type": "module",
   "main": "dist/index.js",

package/templates/core/docker-compose.selfhost.yml

@@ -232,12 +232,12 @@ services:
     mem_limit: ${CORE_WORKER_MEMORY_LIMIT:-3g}
     environment:
       <<: *openmates-worker-env
-      CELERY_QUEUES: persistence,health_check,server_stats,demo,e2e_tests,push
+      CELERY_QUEUES: persistence,health_check,server_stats,demo,e2e_tests,push,leaderboard
       CELERY_AUTOSCALE_MAX: ${CORE_WORKER_CONCURRENCY:-${CELERY_AUTOSCALE_MAX:-3}}
       CELERY_AUTOSCALE_MIN: ${CELERY_AUTOSCALE_MIN:-1}
       CELERY_METRICS_PORT: "9109"
     command: >
-      sh -c "chown -R celeryuser:celeryuser /vault-data && gosu celeryuser python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=persistence,health_check,server_stats,demo,e2e_tests,push --concurrency=$${CELERY_AUTOSCALE_MAX} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1"
+      sh -c "chown -R celeryuser:celeryuser /vault-data && gosu celeryuser python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=persistence,health_check,server_stats,demo,e2e_tests,push,leaderboard --concurrency=$${CELERY_AUTOSCALE_MAX} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1"
 
   user-tasks-worker:
     <<: *openmates-worker-base

package/templates/core/monitoring/prometheus/alert_rules.yml

@@ -0,0 +1,164 @@
+# Prometheus alert rules for OpenMates.
+# These fire when monitored conditions breach thresholds, and Alertmanager
+# routes the resulting notifications (with throttling to prevent email floods).
+#
+# Metric names used here must match what the API actually exposes.
+# API metrics are defined in backend/core/api/app/services/metrics.py:
+#   - api_requests_total         (Counter, labels: method, endpoint, status_code)
+#   - api_request_duration_seconds (Histogram, labels: method, endpoint)
+#   - celery_task_failures_total (Counter, labels: task_name, queue)  — added in celery_config.py
+#
+# Architecture context: See docs/architecture/logging-and-monitoring.md
+
+groups:
+  # -------------------------------------------------------------------------
+  # API health
+  # -------------------------------------------------------------------------
+  - name: api_alerts
+    rules:
+      - alert: HighErrorRate
+        expr: |
+          (
+            sum(rate(api_requests_total{status_code=~"5.."}[5m]))
+            /
+            sum(rate(api_requests_total[5m]))
+          ) > 0.05
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "API 5xx error rate above 5%"
+          description: >-
+            Over the last 5 minutes, {{ $value | humanizePercentage }} of
+            requests returned 5xx errors.
+
+      - alert: HighRequestLatency
+        expr: |
+          histogram_quantile(0.95,
+            sum(rate(api_request_duration_seconds_bucket[5m])) by (le)
+          ) > 5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "P95 request latency above 5 s"
+          description: >-
+            P95 latency is {{ $value | humanizeDuration }}.
+
+      - alert: APIDown
+        expr: up{job="api"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "API container is down"
+          description: "Prometheus cannot scrape the API metrics endpoint."
+
+  # -------------------------------------------------------------------------
+  # Celery task health
+  # Requires celery_task_failures_total counter — added in celery_config.py
+  # worker_process_init signal starts a prometheus_client HTTP server per worker.
+  # -------------------------------------------------------------------------
+  - name: celery_alerts
+    rules:
+      - alert: TaskFailureSpike
+        expr: |
+          sum(increase(celery_task_failures_total[15m])) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "More than 10 Celery task failures in 15 min"
+          description: >-
+            {{ $value }} task failures detected in the last 15 minutes.
+
+  # -------------------------------------------------------------------------
+  # Container / resource health (cAdvisor)
+  # -------------------------------------------------------------------------
+  - name: container_alerts
+    rules:
+      - alert: ContainerRestarting
+        expr: |
+          increase(container_restart_count[30m]) > 3
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container {{ $labels.name }} restarted >3 times in 30 min"
+          description: >-
+            Container {{ $labels.name }} has restarted {{ $value }} times.
+
+      - alert: HighMemoryUsage
+        expr: |
+          (
+            container_memory_usage_bytes{name!="",container_label_com_docker_compose_project="openmates"}
+            /
+            container_spec_memory_limit_bytes{name!="",container_label_com_docker_compose_project="openmates"}
+          ) > 0.90
+          and on(instance, id, name, environment, container_label_com_docker_compose_project)
+          container_spec_memory_limit_bytes{name!="",container_label_com_docker_compose_project="openmates"} > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container {{ $labels.name }} memory above 90%"
+          description: >-
+             Memory usage is at {{ $value | humanizePercentage }}.
+
+  - name: host_resource_alerts
+    rules:
+      - alert: HostDiskSpaceWarning
+        expr: |
+          node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          /
+          node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          < 0.15
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Host disk usage above 85% on {{ $labels.mountpoint }}"
+          description: "Disk usage is above 85% for {{ $labels.mountpoint }}."
+
+      - alert: HostDiskSpaceCritical
+        expr: |
+          node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          /
+          node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          < 0.05
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Host disk usage above 95% on {{ $labels.mountpoint }}"
+          description: "Disk usage is above 95% for {{ $labels.mountpoint }}."
+
+      - alert: OperationalReportStale
+        expr: |
+          (time() - operational_report_last_success_timestamp_seconds > 93600)
+          or
+          (
+            (time() - operational_report_monitoring_started_timestamp_seconds > 93600)
+            unless on(environment) operational_report_last_success_timestamp_seconds
+          )
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.environment }} operational report is stale"
+          description: "No accepted operational report has been recorded for more than 26 hours."
+
+  # -------------------------------------------------------------------------
+  # Prometheus self-monitoring
+  # -------------------------------------------------------------------------
+  - name: prometheus_self
+    rules:
+      - alert: PrometheusTargetDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Scrape target {{ $labels.job }} is down"
+          description: >-
+            Prometheus cannot reach {{ $labels.instance }} (job={{ $labels.job }}).

package/templates/core/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,96 @@
+global:
+  scrape_interval: 30s
+  scrape_timeout: 10s
+  evaluation_interval: 30s
+  external_labels:
+    environment: "${SERVER_ENVIRONMENT}"
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+      scheme: http
+      timeout: 10s
+
+rule_files:
+  - "/etc/prometheus/alert_rules.yml"
+
+# Self-host image mode keeps metrics locally; remote write requires separately provisioned credentials.
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "api"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["api:8000"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  - job_name: "cadvisor"
+    scrape_interval: 60s
+    scrape_timeout: 10s
+    static_configs:
+      - targets: ["cadvisor:8080"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  - job_name: "node"
+    scrape_interval: 60s
+    static_configs:
+      - targets: ["node-exporter:9100"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  # Celery worker Prometheus metrics servers.
+  # Each worker starts a prometheus_client HTTP server on CELERY_METRICS_PORT
+  # via the worker_process_init signal in celery_config.py.
+  # Port assignments (must match CELERY_METRICS_PORT in docker-compose.yml):
+  #   task-worker:        9101
+  #   user-init-worker:   9110
+  #   core-worker:        9109
+  #   user-tasks-worker:  9112
+  #   reminder-worker:    9111
+  #   app-ai-worker:      9102
+  #   app-images-worker:  9103
+  #   app-pdf-worker:     9104
+  - job_name: "celery-task-worker"
+    static_configs:
+      - targets: ["task-worker:9101"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-user-init-worker"
+    static_configs:
+      - targets: ["user-init-worker:9110"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-core-worker"
+    static_configs:
+      - targets: ["core-worker:9109"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-user-tasks-worker"
+    static_configs:
+      - targets: ["user-tasks-worker:9112"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-reminder-worker"
+    static_configs:
+      - targets: ["reminder-worker:9111"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-ai-worker"
+    static_configs:
+      - targets: ["app-ai-worker:9102"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-images-worker"
+    static_configs:
+      - targets: ["app-images-worker:9103"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-pdf-worker"
+    static_configs:
+      - targets: ["app-pdf-worker:9104"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}

package/templates/upload/docker-compose.yml

@@ -21,6 +21,7 @@ services:
       VAULT_AUTO_UNSEAL: "true"
     volumes:
       - vault-setup-data:/app/data
+      - vault-app-data:/app/app-data
     networks: [uploads]
     depends_on: [vault]
     restart: on-failure
@@ -45,7 +46,7 @@ services:
       CLAMAV_HOST: clamav
       CLAMAV_PORT: 3310
     volumes:
-      - vault-setup-data:/vault-data:ro
+      - vault-app-data:/vault-data:ro
     networks: [uploads]
     depends_on: [clamav, vault-setup]
     healthcheck:
@@ -81,3 +82,5 @@ volumes:
     name: openmates-uploads-vault-data
   vault-setup-data:
     name: openmates-uploads-vault-setup-data
+  vault-app-data:
+    name: openmates-uploads-vault-app-data