@forgecart/flight-observability 0.202608221935.0 → 0.202609190800.0

package/dist/exporters/pg-custom-queries.yaml

@@ -25,6 +25,42 @@ pg_idle_in_tx:
         usage: GAUGE
         description: Number of idle-in-transaction backends
 
+# Attribution for the FcIdleInTxAge class (run 32930565055 forensics: a 9.7min
+# idle-in-tx squatter blocked a scope boot and nothing named the owner). Top-1
+# oldest idle-in-tx session WITH identity labels. Cardinality is safe by
+# construction: zero rows — and therefore ZERO series — whenever no squatter
+# exists; during an incident exactly one label tuple lives for the session's
+# lifetime. query_head is whitespace-collapsed and truncated so a label value
+# can never carry a full statement.
+pg_idle_in_tx_top:
+  query: >-
+    SELECT
+      COALESCE(datname, '') AS datname,
+      COALESCE(usename, '') AS usename,
+      COALESCE(application_name, '') AS application_name,
+      COALESCE(left(regexp_replace(query, '\s+', ' ', 'g'), 60), '') AS query_head,
+      EXTRACT(EPOCH FROM (now() - xact_start))::float AS age_seconds
+    FROM pg_stat_activity
+    WHERE backend_type = 'client backend' AND state = 'idle in transaction'
+    ORDER BY xact_start ASC
+    LIMIT 1
+  metrics:
+    - datname:
+        usage: LABEL
+        description: Database of the oldest idle-in-transaction session
+    - usename:
+        usage: LABEL
+        description: Role of the oldest idle-in-transaction session
+    - application_name:
+        usage: LABEL
+        description: application_name of the oldest idle-in-transaction session
+    - query_head:
+        usage: LABEL
+        description: First 60 chars of the session's last statement (whitespace-collapsed)
+    - age_seconds:
+        usage: GAUGE
+        description: Age of the oldest idle-in-transaction session (seconds)
+
 pg_citus_backends:
   query: >-
     SELECT
@@ -62,3 +98,25 @@ pg_outbox:
     - oldest_unsent_seconds:
         usage: GAUGE
         description: Age of the oldest due-but-unsent outbox row (seconds)
+
+# Marketing-event flush health (#1074) — the sensor run 32584944772 lacked: a
+# batch consumed with zero dispatch was invisible. WHERE-form ON PURPOSE, not
+# pg_outbox's FILTER shape: FILTER predicates are evaluated at the aggregate
+# node over an unqualified scan, so on the unreaped firehose table every
+# scrape would seq-scan ALL rows; the WHERE form is served by the partial
+# index IDX_marketing_event_outbox_unflushed and stays O(backlog) — the 097
+# shop_outbox_metrics() precedent. Column names quoted (TypeORM camelCase).
+pg_marketing_outbox:
+  query: >-
+    SELECT
+      count(*)::float AS backlog,
+      COALESCE(EXTRACT(EPOCH FROM (now() - min("ingestedAt"))), 0)::float AS oldest_pending_seconds
+    FROM marketing_event_outbox
+    WHERE "flushedAt" IS NULL
+  metrics:
+    - backlog:
+        usage: GAUGE
+        description: Pending (unflushed) marketing-event outbox rows
+    - oldest_pending_seconds:
+        usage: GAUGE
+        description: Age of the oldest pending marketing-event outbox row (seconds)

package/dist/rules/fc-alerts.yml

@@ -54,7 +54,7 @@ groups:
           severity: warning
         annotations:
           summary: 'Idle-in-transaction backend older than 120s'
-          description: 'Oldest idle-in-tx = {{ $value }}s — a held-open transaction pins a real coordinator connection through the pooler (tech-debt #10).'
+          description: 'Oldest idle-in-tx = {{ $value }}s — a held-open transaction pins a real coordinator connection through the pooler (tech-debt #10). Owner: db={{ with query "topk(1, pg_idle_in_tx_top_age_seconds)" }}{{ . | first | label "datname" }} user={{ . | first | label "usename" }} app={{ . | first | label "application_name" }} q="{{ . | first | label "query_head" }}"{{ end }}'
 
   - name: fc-alerts-pooler
     rules:
@@ -156,6 +156,14 @@ groups:
         annotations:
           summary: 'Oldest due-but-unsent outbox row over 120s'
           description: 'oldest = {{ $value }}s, backlog = {{ with query "fc:outbox:backlog" }}{{ . | first | value }}{{ end }} — the relay is stalled or the broker is refusing publishes.'
+      - alert: FcMarketingFlushLag
+        expr: fc:marketing:flush_lag_seconds > 60
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Oldest pending marketing-event outbox row over 60s'
+          description: 'oldest = {{ $value }}s, backlog = {{ with query "fc:marketing:backlog" }}{{ . | first | value }}{{ end }} — flush workers stalled, claim starvation, or every claimer erroring (#1074).'
 
   - name: fc-alerts-doctrine
     rules:

package/dist/rules/fc-recording.yml

@@ -224,6 +224,12 @@ groups:
       # F4: the REAL effective prefetch — the HOL-utilisation divisor.
       - record: fc:mq:prefetch_effective
         expr: max by (service) (fc_mq_prefetch_effective)
+      # #1172: AMQP liveness per service — min, so ANY disconnected replica
+      # drags the series to 0 (the loud direction). The cause line under an
+      # outbox mute: eligible depth climbing at 0 = genuine broker mute;
+      # climbing at 1 = a claim/delivery defect.
+      - record: fc:mq:connected
+        expr: min by (service) (fc_mq_connected)
       # The register-trigger meter (debt #3 ≥1k, debt #2/Kafka ≥5-10k): the
       # 1k/5k reference lines live on the dashboard panel.
       - record: fc:ingest:events_per_second
@@ -240,6 +246,16 @@ groups:
         expr: max by (service, target) (fc_grpc_ready)
       - record: fc:grpc:inflight
         expr: sum by (service, target) (fc_grpc_inflight)
+      # Ads gateway closure (#933, register #10): staleness/lag/depth gauges + the
+      # re-auth counter rate — bounded provider label by construction.
+      - record: fc:ads:spend_sync_staleness_seconds
+        expr: max by (service, provider) (fc_ads_spend_sync_staleness_seconds)
+      - record: fc:ads:audience_watermark_lag_seconds
+        expr: max by (service, provider) (fc_ads_audience_watermark_lag_seconds)
+      - record: fc:ads:conversion_dlq_depth
+        expr: max by (service) (fc_ads_conversion_dlq_depth)
+      - record: fc:ads:token_refresh_failure_rate
+        expr: sum by (provider, reason) (rate(fc_ads_token_refresh_failures_total[5m]))
       - record: fc:grpc:transient_failure_rate
         expr: sum by (service, target) (rate(fc_grpc_transient_failure_seconds_total[5m]))
       # Model-provider limit rejections (run 32571110796's invisible class):
@@ -388,6 +404,40 @@ groups:
         expr: max(pg_outbox_oldest_unsent_seconds)
       - record: fc:outbox:backlog
         expr: max(pg_outbox_backlog)
+      # #1172: the relay's own lane-scoped census — the ELIGIBILITY-AWARE
+      # truth. The pg_outbox_* pair above is eligibility-blind by construction
+      # (it counts future-dated re-arm slots and foreign-prefix CI orphans,
+      # and ages rows from birth — the 2026-08-29 false "hard-mute"): keep it
+      # for exporter parity, but alert predicates belong on THESE series.
+      - record: fc:outbox:eligible_depth
+        expr: max by (service) (fc_outbox_eligible_depth)
+      - record: fc:outbox:scheduled_depth
+        expr: max by (service) (fc_outbox_scheduled_depth)
+      - record: fc:outbox:retrying_depth
+        expr: max by (service) (fc_outbox_retrying_depth)
+      - record: fc:outbox:dead_depth
+        expr: max by (service) (fc_outbox_dead_depth)
+      - record: fc:outbox:oldest_eligible_seconds
+        expr: max by (service) (fc_outbox_oldest_eligible_seconds)
+      - record: fc:outbox:dead_lettered_rate
+        expr: sum by (service) (rate(fc_outbox_dead_lettered_total[5m]))
+  - name: fc-marketing
+    interval: 15s
+    rules:
+      # Marketing-event flush health (#1074) — pg_marketing_outbox is the
+      # WHERE-form probe (partial-index-served; never the FILTER full-scan).
+      - record: fc:marketing:backlog
+        expr: max(pg_marketing_outbox_backlog)
+      - record: fc:marketing:flush_lag_seconds
+        expr: max(pg_marketing_outbox_oldest_pending_seconds)
+      # Planned-vs-fired imbalance panel — both counters at-least-once, so
+      # panels read rate trends, never exact equality (#1074).
+      - record: fc:automation:planned_match_rate
+        expr: sum(rate(fc_automation_dispatch_matches_total[5m]))
+      - record: fc:automation:fire_rate
+        expr: sum by (outcome) (rate(fc_automation_fire_total[5m]))
+      - record: fc:automation:rule_cache_rate
+        expr: sum by (outcome) (rate(fc_automation_rule_cache_total[5m]))
   - name: fc-workspace
     interval: 15s
     rules:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forgecart/flight-observability",
-  "version": "0.202608221935.0",
+  "version": "0.202609190800.0",
   "private": false,
   "description": "Flight Deck dashboards, fc: recording/alert rules, and the pg-custom-queries contract — the version-pinned sync artifact between shop-beta and forgecart/cloud (launch#46 P4).",
   "files": [