openmates 0.24.0-alpha.1 → 0.24.0

package/dist/{chunk-ETF6SSYS.js → chunk-VEQ3R7MK.js}

@@ -34395,6 +34395,7 @@ var ROLE_TEMPLATE_FILES = {
 var CORE_NO_WEBAPP_TEMPLATE_FILE = join15("core", "docker-compose.no-webapp.yml");
 var CORE_PROMTAIL_CONFIG_FILE = join15("backend", "core", "monitoring", "promtail", "promtail-config.yaml");
 var CORE_ALERTMANAGER_CONFIG_FILE = join15("backend", "core", "monitoring", "alertmanager", "alertmanager.yml");
+var CORE_PROMETHEUS_CONFIG_DIR = join15("backend", "core", "monitoring", "prometheus");
 var COMPOSE_OVERRIDE2 = join15("backend", "core", "docker-compose.override.yml");
 var DEFAULT_INSTALL_PATH = join15(homedir6(), "openmates");
 var REPO_URL = "https://github.com/glowingkitty/OpenMates.git";
@@ -34957,6 +34958,19 @@ function packagedCaddyTemplatePath(role) {
 function packagedCoreAlertmanagerTemplatePath() {
   return join15(dirname8(new URL(import.meta.url).pathname), "..", "templates", "core", "monitoring", "alertmanager", "alertmanager.yml");
 }
+function ensureCorePrometheusRuntimeFiles(installPath) {
+  const packagedDir = join15(dirname8(new URL(import.meta.url).pathname), "..", "templates", "core", "monitoring", "prometheus");
+  const runtimeDir = join15(installPath, CORE_PROMETHEUS_CONFIG_DIR);
+  for (const fileName of ["prometheus.yml", "alert_rules.yml"]) {
+    const packagedPath = join15(packagedDir, fileName);
+    if (!existsSync13(packagedPath)) throw new Error(`Packaged Prometheus file not found: ${packagedPath}`);
+  }
+  mkdirSync13(runtimeDir, { recursive: true });
+  for (const fileName of ["prometheus.yml", "alert_rules.yml"]) {
+    const runtimePath = join15(runtimeDir, fileName);
+    if (!existsSync13(runtimePath)) copyFileSync2(join15(packagedDir, fileName), runtimePath);
+  }
+}
 function readOfficialCloudNoWebappComposeTemplate() {
   const packaged = packagedNoWebappTemplatePath();
   if (existsSync13(packaged)) return readFileSync14(packaged, "utf-8");
@@ -35011,6 +35025,7 @@ async function writeImageModeRuntimeFiles(installPath, imageTag, role) {
     writeFileSync8(join15(installPath, OFFICIAL_CLOUD_NO_WEBAPP_COMPOSE_FILE), readOfficialCloudNoWebappComposeTemplate());
   }
   if (role === "core") {
+    ensureCorePrometheusRuntimeFiles(installPath);
     const promtailConfigPath = join15(installPath, CORE_PROMTAIL_CONFIG_FILE);
     mkdirSync13(dirname8(promtailConfigPath), { recursive: true });
     writeFileSync8(promtailConfigPath, SELFHOST_PROMTAIL_CONFIG_TEMPLATE);

package/dist/cli.js

@@ -19,7 +19,7 @@ import {
   serializeToYaml,
   shouldRequireTrustedAccountGuard,
   waitForWorkflowRun
-} from "./chunk-ETF6SSYS.js";
+} from "./chunk-VEQ3R7MK.js";
 import "./chunk-IWKP55ZB.js";
 import "./chunk-BBKJZ23O.js";
 export {

package/dist/index.js

@@ -43,7 +43,7 @@ import {
   selectAssistantMessagesForSpeech,
   serializeToYaml,
   summarizeAssistantSpeech
-} from "./chunk-ETF6SSYS.js";
+} from "./chunk-VEQ3R7MK.js";
 import "./chunk-IWKP55ZB.js";
 import "./chunk-BBKJZ23O.js";
 export {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openmates",
-  "version": "0.24.0-alpha.1",
+  "version": "0.24.0",
   "description": "OpenMates CLI and SDK",
   "type": "module",
   "main": "dist/index.js",

package/templates/core/docker-compose.selfhost.yml

@@ -232,12 +232,12 @@ services:
     mem_limit: ${CORE_WORKER_MEMORY_LIMIT:-3g}
     environment:
       <<: *openmates-worker-env
-      CELERY_QUEUES: persistence,health_check,server_stats,demo,e2e_tests,push
+      CELERY_QUEUES: persistence,health_check,server_stats,demo,e2e_tests,push,leaderboard
       CELERY_AUTOSCALE_MAX: ${CORE_WORKER_CONCURRENCY:-${CELERY_AUTOSCALE_MAX:-3}}
       CELERY_AUTOSCALE_MIN: ${CELERY_AUTOSCALE_MIN:-1}
       CELERY_METRICS_PORT: "9109"
     command: >
-      sh -c "chown -R celeryuser:celeryuser /vault-data && gosu celeryuser python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=persistence,health_check,server_stats,demo,e2e_tests,push --concurrency=$${CELERY_AUTOSCALE_MAX} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1"
+      sh -c "chown -R celeryuser:celeryuser /vault-data && gosu celeryuser python -m celery -A backend.core.api.app.tasks.celery_config worker --loglevel=info --queues=persistence,health_check,server_stats,demo,e2e_tests,push,leaderboard --concurrency=$${CELERY_AUTOSCALE_MAX} --max-tasks-per-child=50 --max-memory-per-child=600000 --prefetch-multiplier=1"
 
   user-tasks-worker:
     <<: *openmates-worker-base

package/templates/core/monitoring/prometheus/alert_rules.yml

@@ -0,0 +1,164 @@
+# Prometheus alert rules for OpenMates.
+# These fire when monitored conditions breach thresholds, and Alertmanager
+# routes the resulting notifications (with throttling to prevent email floods).
+#
+# Metric names used here must match what the API actually exposes.
+# API metrics are defined in backend/core/api/app/services/metrics.py:
+#   - api_requests_total         (Counter, labels: method, endpoint, status_code)
+#   - api_request_duration_seconds (Histogram, labels: method, endpoint)
+#   - celery_task_failures_total (Counter, labels: task_name, queue)  — added in celery_config.py
+#
+# Architecture context: See docs/architecture/logging-and-monitoring.md
+
+groups:
+  # -------------------------------------------------------------------------
+  # API health
+  # -------------------------------------------------------------------------
+  - name: api_alerts
+    rules:
+      - alert: HighErrorRate
+        expr: |
+          (
+            sum(rate(api_requests_total{status_code=~"5.."}[5m]))
+            /
+            sum(rate(api_requests_total[5m]))
+          ) > 0.05
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "API 5xx error rate above 5%"
+          description: >-
+            Over the last 5 minutes, {{ $value | humanizePercentage }} of
+            requests returned 5xx errors.
+
+      - alert: HighRequestLatency
+        expr: |
+          histogram_quantile(0.95,
+            sum(rate(api_request_duration_seconds_bucket[5m])) by (le)
+          ) > 5
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "P95 request latency above 5 s"
+          description: >-
+            P95 latency is {{ $value | humanizeDuration }}.
+
+      - alert: APIDown
+        expr: up{job="api"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "API container is down"
+          description: "Prometheus cannot scrape the API metrics endpoint."
+
+  # -------------------------------------------------------------------------
+  # Celery task health
+  # Requires celery_task_failures_total counter — added in celery_config.py
+  # worker_process_init signal starts a prometheus_client HTTP server per worker.
+  # -------------------------------------------------------------------------
+  - name: celery_alerts
+    rules:
+      - alert: TaskFailureSpike
+        expr: |
+          sum(increase(celery_task_failures_total[15m])) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "More than 10 Celery task failures in 15 min"
+          description: >-
+            {{ $value }} task failures detected in the last 15 minutes.
+
+  # -------------------------------------------------------------------------
+  # Container / resource health (cAdvisor)
+  # -------------------------------------------------------------------------
+  - name: container_alerts
+    rules:
+      - alert: ContainerRestarting
+        expr: |
+          increase(container_restart_count[30m]) > 3
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container {{ $labels.name }} restarted >3 times in 30 min"
+          description: >-
+            Container {{ $labels.name }} has restarted {{ $value }} times.
+
+      - alert: HighMemoryUsage
+        expr: |
+          (
+            container_memory_usage_bytes{name!="",container_label_com_docker_compose_project="openmates"}
+            /
+            container_spec_memory_limit_bytes{name!="",container_label_com_docker_compose_project="openmates"}
+          ) > 0.90
+          and on(instance, id, name, environment, container_label_com_docker_compose_project)
+          container_spec_memory_limit_bytes{name!="",container_label_com_docker_compose_project="openmates"} > 0
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Container {{ $labels.name }} memory above 90%"
+          description: >-
+             Memory usage is at {{ $value | humanizePercentage }}.
+
+  - name: host_resource_alerts
+    rules:
+      - alert: HostDiskSpaceWarning
+        expr: |
+          node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          /
+          node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          < 0.15
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Host disk usage above 85% on {{ $labels.mountpoint }}"
+          description: "Disk usage is above 85% for {{ $labels.mountpoint }}."
+
+      - alert: HostDiskSpaceCritical
+        expr: |
+          node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          /
+          node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs"}
+          < 0.05
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Host disk usage above 95% on {{ $labels.mountpoint }}"
+          description: "Disk usage is above 95% for {{ $labels.mountpoint }}."
+
+      - alert: OperationalReportStale
+        expr: |
+          (time() - operational_report_last_success_timestamp_seconds > 93600)
+          or
+          (
+            (time() - operational_report_monitoring_started_timestamp_seconds > 93600)
+            unless on(environment) operational_report_last_success_timestamp_seconds
+          )
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "{{ $labels.environment }} operational report is stale"
+          description: "No accepted operational report has been recorded for more than 26 hours."
+
+  # -------------------------------------------------------------------------
+  # Prometheus self-monitoring
+  # -------------------------------------------------------------------------
+  - name: prometheus_self
+    rules:
+      - alert: PrometheusTargetDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Scrape target {{ $labels.job }} is down"
+          description: >-
+            Prometheus cannot reach {{ $labels.instance }} (job={{ $labels.job }}).

package/templates/core/monitoring/prometheus/prometheus.yml

@@ -0,0 +1,96 @@
+global:
+  scrape_interval: 30s
+  scrape_timeout: 10s
+  evaluation_interval: 30s
+  external_labels:
+    environment: "${SERVER_ENVIRONMENT}"
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+      scheme: http
+      timeout: 10s
+
+rule_files:
+  - "/etc/prometheus/alert_rules.yml"
+
+# Self-host image mode keeps metrics locally; remote write requires separately provisioned credentials.
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "api"
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["api:8000"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  - job_name: "cadvisor"
+    scrape_interval: 60s
+    scrape_timeout: 10s
+    static_configs:
+      - targets: ["cadvisor:8080"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  - job_name: "node"
+    scrape_interval: 60s
+    static_configs:
+      - targets: ["node-exporter:9100"]
+        labels:
+          environment: "${SERVER_ENVIRONMENT}"
+
+  # Celery worker Prometheus metrics servers.
+  # Each worker starts a prometheus_client HTTP server on CELERY_METRICS_PORT
+  # via the worker_process_init signal in celery_config.py.
+  # Port assignments (must match CELERY_METRICS_PORT in docker-compose.yml):
+  #   task-worker:        9101
+  #   user-init-worker:   9110
+  #   core-worker:        9109
+  #   user-tasks-worker:  9112
+  #   reminder-worker:    9111
+  #   app-ai-worker:      9102
+  #   app-images-worker:  9103
+  #   app-pdf-worker:     9104
+  - job_name: "celery-task-worker"
+    static_configs:
+      - targets: ["task-worker:9101"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-user-init-worker"
+    static_configs:
+      - targets: ["user-init-worker:9110"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-core-worker"
+    static_configs:
+      - targets: ["core-worker:9109"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-user-tasks-worker"
+    static_configs:
+      - targets: ["user-tasks-worker:9112"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-reminder-worker"
+    static_configs:
+      - targets: ["reminder-worker:9111"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-ai-worker"
+    static_configs:
+      - targets: ["app-ai-worker:9102"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-images-worker"
+    static_configs:
+      - targets: ["app-images-worker:9103"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}
+
+  - job_name: "celery-app-pdf-worker"
+    static_configs:
+      - targets: ["app-pdf-worker:9104"]
+        labels: {environment: "${SERVER_ENVIRONMENT}"}