@forgecart/flight-observability 0.202608220925.0 → 0.202608300703.0

package/dist/exporters/pg-custom-queries.yaml

@@ -25,6 +25,42 @@ pg_idle_in_tx:
         usage: GAUGE
         description: Number of idle-in-transaction backends
 
+# Attribution for the FcIdleInTxAge class (run 32930565055 forensics: a 9.7min
+# idle-in-tx squatter blocked a scope boot and nothing named the owner). Top-1
+# oldest idle-in-tx session WITH identity labels. Cardinality is safe by
+# construction: zero rows — and therefore ZERO series — whenever no squatter
+# exists; during an incident exactly one label tuple lives for the session's
+# lifetime. query_head is whitespace-collapsed and truncated so a label value
+# can never carry a full statement.
+pg_idle_in_tx_top:
+  query: >-
+    SELECT
+      COALESCE(datname, '') AS datname,
+      COALESCE(usename, '') AS usename,
+      COALESCE(application_name, '') AS application_name,
+      COALESCE(left(regexp_replace(query, '\s+', ' ', 'g'), 60), '') AS query_head,
+      EXTRACT(EPOCH FROM (now() - xact_start))::float AS age_seconds
+    FROM pg_stat_activity
+    WHERE backend_type = 'client backend' AND state = 'idle in transaction'
+    ORDER BY xact_start ASC
+    LIMIT 1
+  metrics:
+    - datname:
+        usage: LABEL
+        description: Database of the oldest idle-in-transaction session
+    - usename:
+        usage: LABEL
+        description: Role of the oldest idle-in-transaction session
+    - application_name:
+        usage: LABEL
+        description: application_name of the oldest idle-in-transaction session
+    - query_head:
+        usage: LABEL
+        description: First 60 chars of the session's last statement (whitespace-collapsed)
+    - age_seconds:
+        usage: GAUGE
+        description: Age of the oldest idle-in-transaction session (seconds)
+
 pg_citus_backends:
   query: >-
     SELECT
@@ -62,3 +98,25 @@ pg_outbox:
     - oldest_unsent_seconds:
         usage: GAUGE
         description: Age of the oldest due-but-unsent outbox row (seconds)
+
+# Marketing-event flush health (#1074) — the sensor run 32584944772 lacked: a
+# batch consumed with zero dispatch was invisible. WHERE-form ON PURPOSE, not
+# pg_outbox's FILTER shape: FILTER predicates are evaluated at the aggregate
+# node over an unqualified scan, so on the unreaped firehose table every
+# scrape would seq-scan ALL rows; the WHERE form is served by the partial
+# index IDX_marketing_event_outbox_unflushed and stays O(backlog) — the 097
+# shop_outbox_metrics() precedent. Column names quoted (TypeORM camelCase).
+pg_marketing_outbox:
+  query: >-
+    SELECT
+      count(*)::float AS backlog,
+      COALESCE(EXTRACT(EPOCH FROM (now() - min("ingestedAt"))), 0)::float AS oldest_pending_seconds
+    FROM marketing_event_outbox
+    WHERE "flushedAt" IS NULL
+  metrics:
+    - backlog:
+        usage: GAUGE
+        description: Pending (unflushed) marketing-event outbox rows
+    - oldest_pending_seconds:
+        usage: GAUGE
+        description: Age of the oldest pending marketing-event outbox row (seconds)

package/dist/rules/fc-alerts.yml

@@ -54,7 +54,7 @@ groups:
           severity: warning
         annotations:
           summary: 'Idle-in-transaction backend older than 120s'
-          description: 'Oldest idle-in-tx = {{ $value }}s — a held-open transaction pins a real coordinator connection through the pooler (tech-debt #10).'
+          description: 'Oldest idle-in-tx = {{ $value }}s — a held-open transaction pins a real coordinator connection through the pooler (tech-debt #10). Owner: db={{ with query "topk(1, pg_idle_in_tx_top_age_seconds)" }}{{ . | first | label "datname" }} user={{ . | first | label "usename" }} app={{ . | first | label "application_name" }} q="{{ . | first | label "query_head" }}"{{ end }}'
 
   - name: fc-alerts-pooler
     rules:
@@ -123,6 +123,20 @@ groups:
           summary: 'Transaction p99 above 80% of TRANSACTION_TIME_BUDGET_MS'
           description: 'p99 = {{ $value }}s vs the budget gauge — client-paced spans are drifting toward the kill window.'
 
+  - name: fc-alerts-model
+    rules:
+      # Run 32571110796's invisible class: account starvation fails live-model
+      # turns FAST behind generic fallbacks while every dashboard reads
+      # healthy — this is the loud surface. Fires on the FIRST rejection
+      # (usage limits persist for a billing window; there is no flap to damp).
+      - alert: FcModelUsageLimit
+        expr: sum(increase(fc_model_rate_limited_total{reason="usage_limit"}[10m])) > 0
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Model account usage limit — live-model turns are being rejected'
+          description: 'Agent turns are failing fast on provider usage limits; resetsAt rides the agent log "Claude Code cannot serve". Live-model specs will red until the window resets or the limit is raised.'
+
   - name: fc-alerts-probes
     rules:
       - alert: FcDnsProbeFailing
@@ -142,6 +156,14 @@ groups:
         annotations:
           summary: 'Oldest due-but-unsent outbox row over 120s'
           description: 'oldest = {{ $value }}s, backlog = {{ with query "fc:outbox:backlog" }}{{ . | first | value }}{{ end }} — the relay is stalled or the broker is refusing publishes.'
+      - alert: FcMarketingFlushLag
+        expr: fc:marketing:flush_lag_seconds > 60
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Oldest pending marketing-event outbox row over 60s'
+          description: 'oldest = {{ $value }}s, backlog = {{ with query "fc:marketing:backlog" }}{{ . | first | value }}{{ end }} — flush workers stalled, claim starvation, or every claimer erroring (#1074).'
 
   - name: fc-alerts-doctrine
     rules:

package/dist/rules/fc-recording.yml

@@ -242,6 +242,10 @@ groups:
         expr: sum by (service, target) (fc_grpc_inflight)
       - record: fc:grpc:transient_failure_rate
         expr: sum by (service, target) (rate(fc_grpc_transient_failure_seconds_total[5m]))
+      # Model-provider limit rejections (run 32571110796's invisible class):
+      # reasons are a bounded set by construction (deriveModelLimitReason).
+      - record: fc:model:rate_limited_rate
+        expr: sum by (reason) (rate(fc_model_rate_limited_total[5m]))
 
   - name: fc-pgbouncer
     interval: 15s
@@ -384,6 +388,23 @@ groups:
         expr: max(pg_outbox_oldest_unsent_seconds)
       - record: fc:outbox:backlog
         expr: max(pg_outbox_backlog)
+  - name: fc-marketing
+    interval: 15s
+    rules:
+      # Marketing-event flush health (#1074) — pg_marketing_outbox is the
+      # WHERE-form probe (partial-index-served; never the FILTER full-scan).
+      - record: fc:marketing:backlog
+        expr: max(pg_marketing_outbox_backlog)
+      - record: fc:marketing:flush_lag_seconds
+        expr: max(pg_marketing_outbox_oldest_pending_seconds)
+      # Planned-vs-fired imbalance panel — both counters at-least-once, so
+      # panels read rate trends, never exact equality (#1074).
+      - record: fc:automation:planned_match_rate
+        expr: sum(rate(fc_automation_dispatch_matches_total[5m]))
+      - record: fc:automation:fire_rate
+        expr: sum by (outcome) (rate(fc_automation_fire_total[5m]))
+      - record: fc:automation:rule_cache_rate
+        expr: sum by (outcome) (rate(fc_automation_rule_cache_total[5m]))
   - name: fc-workspace
     interval: 15s
     rules:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forgecart/flight-observability",
-  "version": "0.202608220925.0",
+  "version": "0.202608300703.0",
   "private": false,
   "description": "Flight Deck dashboards, fc: recording/alert rules, and the pg-custom-queries contract — the version-pinned sync artifact between shop-beta and forgecart/cloud (launch#46 P4).",
   "files": [