@forgecart/flight-observability 0.202608212205.0

package/dist/exporters/pg-custom-queries.yaml

@@ -0,0 +1,64 @@
+# pg-custom-queries.yaml v1 — the shared PG custom-metric contract (Flight
+# Deck P0 #967 → P4 #973). ONE dialect, two consumers: locally the shop-obs
+# postgres_exporter ×3 (`--extend.query-path`), in prod CNPG's
+# `monitoring.customQueriesConfigMap` renders THIS SAME file (architecture §7
+# item 3) — the metric names below are contract, never fork them per env.
+#
+# v1 scope (§3 rows 6): idle-in-transaction age/count (tech-debt #10's
+# left half — the FcIdleInTxAge alert consumes fc:pg:idle_in_tx_age_max_seconds)
+# and coordinator↔worker backend attribution. The P1c extensions (in-flight
+# age percentiles, connection ages, fan-out counts) append here — additive.
+
+pg_idle_in_tx:
+  query: >-
+    SELECT
+      COALESCE(EXTRACT(EPOCH FROM
+        max(now() - xact_start) FILTER (WHERE state = 'idle in transaction')), 0)::float AS age_max_seconds,
+      count(*) FILTER (WHERE state = 'idle in transaction')::float AS count
+    FROM pg_stat_activity
+    WHERE backend_type = 'client backend'
+  metrics:
+    - age_max_seconds:
+        usage: GAUGE
+        description: Age of the oldest idle-in-transaction backend (seconds)
+    - count:
+        usage: GAUGE
+        description: Number of idle-in-transaction backends
+
+pg_citus_backends:
+  query: >-
+    SELECT
+      CASE WHEN application_name LIKE 'citus%%' THEN 'citus' ELSE 'client' END AS source,
+      count(*)::float AS count
+    FROM pg_stat_activity
+    WHERE backend_type = 'client backend'
+    GROUP BY 1
+  metrics:
+    - source:
+        usage: LABEL
+        description: Backend origin (citus = coordinator fan-out, client = direct)
+    - count:
+        usage: GAUGE
+        description: Client backends by origin
+
+# Outbox staleness (P4 #973 — FcOutboxOldest's source). Oldest UNSENT row that
+# is actually DUE (deliver_at in the future is a scheduled send, not a stall).
+pg_outbox:
+  # Column names are TypeORM camelCase and MUST be quoted — the unquoted
+  # snake_case draft failed live on the Q12 verification pass
+  # (`pg_outbox pq: column "created_at" does not exist`, exporter served
+  # nothing and FcOutboxOldest was input-starved).
+  query: >-
+    SELECT
+      count(*) FILTER (WHERE NOT sent)::float AS backlog,
+      COALESCE(EXTRACT(EPOCH FROM (now() - min("createdAt") FILTER (
+        WHERE NOT sent AND ("deliverAt" IS NULL OR "deliverAt" <= now())
+      ))), 0)::float AS oldest_unsent_seconds
+    FROM queue_outbox
+  metrics:
+    - backlog:
+        usage: GAUGE
+        description: Unsent outbox rows
+    - oldest_unsent_seconds:
+        usage: GAUGE
+        description: Age of the oldest due-but-unsent outbox row (seconds)

package/dist/rules/fc-alerts.yml

@@ -0,0 +1,188 @@
+# fc-alerts.yml — the ONE alert-rules file both worlds evaluate (architecture
+# §7 item 5): locally the shop-obs vmalert fires into the BLACKHOLE notifier,
+# prod (P4 #973) routes the identical file to Discord via alertmanager. Never
+# fork per env — env-specific thresholds select on the `env` external label.
+#
+# P1c seeded the cardinality self-watch; P4 lands the full set. Thresholds
+# marked [Q9-PROPOSED] ratify with the operator at final acceptance (E3) —
+# they alert from day one so the one-week forgecart-test dry-run (E2) can
+# judge them against reality. `CitusBackendsHigh` already exists in
+# forgecart/cloud — displayed on Row 4, deliberately NOT duplicated here.
+groups:
+  - name: fc-alerts-self
+    rules:
+      # C5 — the series budget is a hard ceiling per environment: 60k local,
+      # 150k prod. Breach = a cardinality leak (per-run/per-test label escape)
+      # until proven otherwise; the offender census starts at
+      # /api/v1/status/tsdb on the breaching store. F1's ~1k-workspace-pod
+      # trigger (tech-debt #1) surfaces HERE first by design.
+      - alert: FcCardinalityBudget
+        expr: >-
+          fc:self:active_series{env="local"} > 60000
+          or
+          fc:self:active_series{env!="local"} > 150000
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'TSDB active series over the {{ $labels.env }} budget'
+          description: 'fc:self:active_series = {{ $value }} — cardinality leak until proven otherwise (architecture §2.2/C5).'
+
+  # ── tech-debt #10, closed verbatim (P4 #973 item 5) ─────────────────────────
+  - name: fc-alerts-held-tx
+    rules:
+      - alert: FcHeldTxPoolWait
+        expr: fc:convoy:heldtx == 1
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Held-tx convoy sustained 2m'
+          description: 'Waiters queue on the held-transaction lane while it sits pinned — the pool-convoy class is invisible to CPU/latency dashboards by construction (tech-debt #10).'
+      - alert: FcHeldTxAcquireWaitCritical
+        expr: fc:pool:acquire_wait_p99_seconds{lane="held-tx"} > 5
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: 'Held-tx acquire wait p99 over 5s'
+          description: 'p99 = {{ $value }}s — editing transactions are starving; check fc:pgbouncer:pinned_share and the coordinator connection budget (raise-together rule).'
+      - alert: FcIdleInTxAge
+        expr: fc:pg:idle_in_tx_age_max_seconds > 120
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Idle-in-transaction backend older than 120s'
+          description: 'Oldest idle-in-tx = {{ $value }}s — a held-open transaction pins a real coordinator connection through the pooler (tech-debt #10).'
+
+  - name: fc-alerts-pooler
+    rules:
+      - alert: FcPoolerMaxwait
+        expr: fc:pgbouncer:maxwait_seconds > 2
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'PgBouncer pool maxwait over 2s'
+          description: 'db_family={{ $labels.db_family }} maxwait = {{ $value }}s — clients queue at the pooler; the F5 pinned-share panel names the culprit side.'
+      # F5's invariant as an alert: the held-lane share of standing server
+      # connections. [Q9-PROPOSED] 0.8.
+      - alert: FcPinnedShareHigh
+        expr: fc:pgbouncer:pinned_share > 0.8
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Held-tx lane consumes >80% of standing pooler servers'
+          description: 'pinned_share = {{ $value }} — client-paced transactions are crowding out multiplexed traffic (§3.4).'
+
+  - name: fc-alerts-leak
+    rules:
+      - alert: FcCloseWaitLeak
+        expr: fc:leak:close_wait > 50
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'CLOSE_WAIT sockets accumulating on {{ $labels.actor }}'
+          description: '{{ $value }} CLOSE_WAIT sockets held 15m+ — the peer closed, the app never reaped (Row 8; application_name from #969 names the PG-side twin).'
+      # ~1 MB/min sustained for an hour. [Q9-PROPOSED]
+      - alert: FcRssLeak
+        expr: fc:leak:rss_slope_bytes_per_s > 17476
+        for: 60m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'RSS climbing ~1MB/min for 1h on {{ $labels.actor }}'
+          description: '30m slope = {{ $value }} B/s sustained — leak candidate; replay the window and read the actor heat bands.'
+
+  - name: fc-alerts-latency
+    rules:
+      # [Q9-PROPOSED] 200ms.
+      - alert: FcEventLoopLag
+        expr: fc:loop:lag_p99_seconds > 0.2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Event-loop lag p99 over 200ms on {{ $labels.service }}'
+          description: 'p99 = {{ $value }}s — the serve tier is starving; correlate with fc:actor:psi_cpu_some_ratio and the melee lanes.'
+      # [Q9-PROPOSED] tx p99 above 80% of the live budget gauge (F9).
+      - alert: FcTxNearBudget
+        expr: fc:tx:p99_seconds > 0.8 * fc:tx:budget_seconds
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Transaction p99 above 80% of TRANSACTION_TIME_BUDGET_MS'
+          description: 'p99 = {{ $value }}s vs the budget gauge — client-paced spans are drifting toward the kill window.'
+
+  - name: fc-alerts-probes
+    rules:
+      - alert: FcDnsProbeFailing
+        expr: fc:dns:probe_success == 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'DNS probe {{ $labels.probe }} failing 5m'
+          description: 'system-fail + control-ok is the router rebind-protection signature; both-fail is an egress/DNS outage (§3 row 17).'
+      # [Q9-PROPOSED] 120s.
+      - alert: FcOutboxOldest
+        expr: fc:outbox:oldest_seconds > 120
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Oldest due-but-unsent outbox row over 120s'
+          description: 'oldest = {{ $value }}s, backlog = {{ with query "fc:outbox:backlog" }}{{ . | first | value }}{{ end }} — the relay is stalled or the broker is refusing publishes.'
+
+  - name: fc-alerts-doctrine
+    rules:
+      # G5's tripwire in alert form: weights never quotas — nonzero throttling
+      # means someone added a CPU limit (a namespace LimitRange injecting
+      # defaults is the usual prod offender, §6.8).
+      - alert: FcThrottleNonZero
+        expr: fc:actor:throttled_periods_delta > 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: 'CPU throttling on {{ $labels.actor }} — a quota exists'
+          description: 'nr_throttled delta = {{ $value }} — the limits doctrine is weights NEVER quotas (§6.1/G5); find and remove the quota, never tune it.'
+
+  - name: fc-alerts-convoys
+    rules:
+      - alert: FcConvoyPgbouncer
+        expr: fc:convoy:pgbouncer == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Pooler convoy: clients waiting while servers sit active'
+          description: 'The multiplexing layer is the bottleneck — read the Multiplexing page top-down (§4.2).'
+      - alert: FcConvoyCitusFanout
+        expr: fc:convoy:citus_fanout == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Citus fan-out convoy: coordinator backends saturated by distributed queries'
+          description: 'Distributed-query fan-out is crowding the coordinator (>256 citus-source backends sustained).'
+      - alert: FcConvoyGrpcParked
+        expr: fc:convoy:grpc_parked == 1
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'gRPC channel parked with in-flight RPCs ({{ $labels.service }}→{{ $labels.target }})'
+          description: 'RPCs issued against a channel that is not READY — the park signature from the dual7/dual12 forensics (#969).'
+      - alert: FcConvoyPrefetchHol
+        expr: fc:convoy:prefetch_hol == 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'MQ prefetch head-of-line: unacked pinned at consumers × effective prefetch'
+          description: 'Consumers hold their full prefetch window unacked (F4 divisor) — slow-consumer HOL, not backlog.'

package/dist/rules/fc-recording.yml

@@ -0,0 +1,409 @@
+# fc-recording.yml v1 — THE `fc:` recording-rule contract (Flight Deck P0,
+# launch#46 / #967; architecture §2.1/§3). This ONE file is the local↔prod
+# parity seam: dashboards query `fc:*` ONLY, and every exporter-label
+# difference between the two worlds is absorbed HERE. The same file is
+# evaluated by the local shop-obs vmalert (blackhole notifier) and by the prod
+# victoria-metrics-k8s-stack vmalert (P4 #973) — never fork it per env.
+#
+# Cardinality doctrine (§2.2): every label on a rule below comes from a BOUNDED
+# set (actor, db_family, pg_node, state, service, probe target). No per-test,
+# per-channel, per-run, or per-op-name label may ever appear in this file.
+#
+# v1 rows (P0): host weather, per-actor CPU/RSS/PSI/throttle, PG backends by
+# db_family + idle-in-tx, MQ transport basics, sockets, DNS, victim canary,
+# redis/es/ch health, TSDB self-watch. The gRPC row is deliberately ABSENT
+# until #969 lands the generic package/grpc seam (F3 honesty rule — its only
+# P0 truth is Plane-B rpc.* markers); `fc:workspace:*` (F1 bounded families)
+# arrives with kube-state-metrics in prod (P4) — local per-pod truth lives in
+# Plane B / replay.
+groups:
+  - name: fc-host
+    interval: 15s
+    rules:
+      - record: fc:host:cpu_busy_ratio
+        expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m]))
+      - record: fc:host:cpu_steal_ratio
+        expr: avg(rate(node_cpu_seconds_total{mode="steal"}[1m]))
+      - record: fc:host:psi_cpu_some_ratio
+        expr: rate(node_pressure_cpu_waiting_seconds_total[1m])
+      - record: fc:host:psi_mem_some_ratio
+        expr: rate(node_pressure_memory_waiting_seconds_total[1m])
+      - record: fc:host:psi_io_some_ratio
+        expr: rate(node_pressure_io_waiting_seconds_total[1m])
+      - record: fc:host:load1
+        expr: node_load1
+      - record: fc:host:mem_available_ratio
+        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes
+      # The red-weather stamp (§3 row 3): swap-out while memory looks "fine"
+      # is the melee-killing signature steal alone cannot explain.
+      - record: fc:host:pswpout_rate
+        expr: rate(node_vmstat_pswpout[1m])
+
+  - name: fc-actor
+    interval: 15s
+    rules:
+      # Actor identity is BOUNDED. Docker containers are discovered via the
+      # system containerd (moby namespace — this host's docker runs the
+      # containerd-snapshotter layout, and docker propagates NO compose labels
+      # to containerd), so the stable bounded key is the IMAGE basename
+      # (postgresql-citus, rabbitmq-delayed, redis, k3s, …) — container-hash
+      # `name`s recycle on every recreate and must never become actors.
+      # `fc-*` systemd scopes (P2 governance, raw whitelist) map from `id`.
+      - record: fc:actor:cpu_ratio
+        expr: >-
+          sum by (actor) (
+            label_replace(rate(container_cpu_usage_seconds_total{image=~".+"}[1m]), "actor", "$1", "image", ".*/([^/:]+):[^/]+")
+            or
+            label_replace(rate(container_cpu_usage_seconds_total{image="", id=~".*/fc-[^/]+\\.(scope|slice)"}[1m]), "actor", "$1", "id", ".*/(fc-[^/]+)\\.(?:scope|slice)")
+          )
+      - record: fc:actor:rss_bytes
+        expr: >-
+          sum by (actor) (
+            label_replace(container_memory_rss{image=~".+"}, "actor", "$1", "image", ".*/([^/:]+):[^/]+")
+            or
+            label_replace(container_memory_rss{image="", id=~".*/fc-[^/]+\\.(scope|slice)"}, "actor", "$1", "id", ".*/(fc-[^/]+)\\.(?:scope|slice)")
+          )
+      - record: fc:actor:psi_cpu_some_ratio
+        expr: >-
+          sum by (actor) (
+            label_replace(rate(container_pressure_cpu_waiting_seconds_total{image=~".+"}[1m]), "actor", "$1", "image", ".*/([^/:]+):[^/]+")
+            or
+            label_replace(rate(container_pressure_cpu_waiting_seconds_total{image="", id=~".*/fc-[^/]+\\.(scope|slice)"}[1m]), "actor", "$1", "id", ".*/(fc-[^/]+)\\.(?:scope|slice)")
+          )
+      # The limits-doctrine tripwire (§6, G5): weights never quotas, so this
+      # must read 0 on EVERY cgroup — any increase means someone added a quota.
+      - record: fc:actor:throttled_periods_delta
+        expr: >-
+          sum by (actor) (
+            label_replace(increase(container_cpu_cfs_throttled_periods_total{image=~".+"}[1m]), "actor", "$1", "image", ".*/([^/:]+):[^/]+")
+            or
+            label_replace(increase(container_cpu_cfs_throttled_periods_total{image="", id=~".*/fc-[^/]+\\.(scope|slice)"}[1m]), "actor", "$1", "id", ".*/(fc-[^/]+)\\.(?:scope|slice)")
+          )
+
+  - name: fc-pg
+    interval: 15s
+    rules:
+      # db_family collapse (§3 row 6 / P1c F1-awareness): per-run DBs fold to a
+      # bounded family — a per-run datname must never survive into fc:*.
+      - record: fc:pg:backends
+        expr: >-
+          sum by (pg_node, db_family) (
+            label_replace(
+              label_replace(
+                label_replace(
+                  label_replace(pg_stat_database_numbackends{datname!~"template.*|postgres|^$"}, "db_family", "other", "datname", ".*"),
+                  "db_family", "e2e", "datname", "e2e_.*"),
+                "db_family", "integration", "datname", "integration_.*"),
+              "db_family", "standing", "datname", "forgecart")
+          )
+      - record: fc:pg:idle_in_tx_age_max_seconds
+        expr: max(pg_idle_in_tx_age_max_seconds)
+      - record: fc:pg:idle_in_tx_count
+        expr: sum(pg_idle_in_tx_count)
+      - record: fc:pg:backends_total
+        expr: sum by (pg_node) (pg_stat_database_numbackends)
+      # Cluster or-branch (#973 rollout): CNPG's NATIVE exporter serves this
+      # family as cnpg_backends_total (its own metric, not our custom-queries
+      # file, so the scrape-time cnpg_-prefix strip cannot claim it without
+      # colliding with the pg_ exporter namespace). The cloud vmagent attaches
+      # {pg_node,source} at scrape; per-env the two arms are disjoint.
+      - record: fc:pg:citus_backends
+        expr: >-
+          sum by (pg_node, source) (pg_citus_backends_count or cnpg_backends_total)
+
+  - name: fc-mq
+    interval: 15s
+    rules:
+      # Aggregated `/metrics` basics only in v1 (POC §3.2): per-queue_class
+      # series exist ONLY on the detailed endpoint and land with P1c/P4 (F4).
+      # Dark until the Q12 root-compose event publishes :15692.
+      - record: fc:mq:ready_total
+        expr: rabbitmq_queue_messages_ready
+      - record: fc:mq:unacked_total
+        expr: rabbitmq_queue_messages_unacked
+      - record: fc:mq:connections
+        expr: rabbitmq_connections
+      - record: fc:mq:channels
+        expr: rabbitmq_channels
+      - record: fc:mq:consumers
+        expr: rabbitmq_consumers
+
+  - name: fc-sock
+    interval: 15s
+    rules:
+      - record: fc:sock:tcp_states
+        expr: node_tcp_connection_states
+      - record: fc:sock:sockets_used
+        expr: node_sockstat_sockets_used
+
+  - name: fc-dns
+    interval: 15s
+    rules:
+      # probe=system rides getaddrinfo (HTTP probe of the nip host); control
+      # asks 1.1.1.1 directly. system-fail + control-success = router
+      # rebind-protection signature (§3 row 17).
+      - record: fc:dns:probe_success
+        expr: >-
+          label_replace(probe_success{job="dns-system"}, "probe", "system", "job", ".*")
+          or
+          label_replace(probe_success{job="dns-control"}, "probe", "control", "job", ".*")
+      - record: fc:dns:probe_duration_seconds
+        expr: >-
+          label_replace(probe_duration_seconds{job="dns-system"}, "probe", "system", "job", ".*")
+          or
+          label_replace(probe_duration_seconds{job="dns-control"}, "probe", "control", "job", ".*")
+
+  - name: fc-canary
+    interval: 15s
+    rules:
+      # The victim canary (G2's instrument): /health = liveness floor,
+      # GraphQL = the load-bearing latency gate.
+      - record: fc:canary:health_success
+        expr: probe_success{job="canary-health"}
+      - record: fc:canary:health_duration_seconds
+        expr: probe_duration_seconds{job="canary-health"}
+      - record: fc:canary:graphql_success
+        expr: probe_success{job="canary-graphql"}
+      - record: fc:canary:graphql_duration_seconds
+        expr: probe_duration_seconds{job="canary-graphql"}
+
+  - name: fc-backing
+    interval: 15s
+    rules:
+      - record: fc:redis:connected_clients
+        expr: redis_connected_clients
+      - record: fc:redis:blocked_clients
+        expr: redis_blocked_clients
+      - record: fc:es:up
+        expr: up{job="elasticsearch"}
+      - record: fc:ch:up
+        expr: up{job="clickhouse"}
+
+  - name: fc-app
+    interval: 15s
+    rules:
+      # P1a (#968) — the package/flight runtime instruments, scraped off the
+      # per-service /metrics listeners (:9464-68; `service` comes from the
+      # scrape target labels). fc_* app series → fc:* dashboard contract.
+      - record: fc:tx:p50_seconds
+        expr: histogram_quantile(0.5, sum by (le, op_class) (rate(fc_tx_duration_seconds_bucket[1m])))
+      - record: fc:tx:p99_seconds
+        expr: histogram_quantile(0.99, sum by (le, op_class) (rate(fc_tx_duration_seconds_bucket[1m])))
+      # F9: the live budget gauge, per service — never a rules-file constant.
+      - record: fc:tx:budget_seconds
+        expr: max by (service) (fc_tx_budget_seconds)
+      - record: fc:tx:ring_min_seconds
+        expr: max by (service) (fc_tx_ring_min_seconds)
+      - record: fc:tx:ring_max_seconds
+        expr: max by (service) (fc_tx_ring_max_seconds)
+      # F2: the all-traffic distribution (op_class=query|mutation|tx_mutation).
+      - record: fc:gql:p50_seconds
+        expr: histogram_quantile(0.5, sum by (le, op_class) (rate(fc_gql_duration_seconds_bucket[1m])))
+      - record: fc:gql:p99_seconds
+        expr: histogram_quantile(0.99, sum by (le, op_class) (rate(fc_gql_duration_seconds_bucket[1m])))
+      - record: fc:gql:rate
+        expr: sum by (op_class) (rate(fc_gql_duration_seconds_count[1m]))
+      - record: fc:pool:size
+        expr: sum by (service, lane) (fc_pool_size)
+      - record: fc:pool:in_use
+        expr: sum by (service, lane) (fc_pool_in_use)
+      - record: fc:pool:idle
+        expr: sum by (service, lane) (fc_pool_idle)
+      - record: fc:pool:waiting
+        expr: sum by (service, lane) (fc_pool_waiting)
+      - record: fc:pool:acquire_wait_p99_seconds
+        expr: histogram_quantile(0.99, sum by (le, lane) (rate(fc_pool_acquire_wait_seconds_bucket[1m])))
+      - record: fc:heldtx:open
+        expr: sum by (service) (fc_heldtx_open)
+      - record: fc:heldtx:oldest_age_seconds
+        expr: max by (service) (fc_heldtx_oldest_age_seconds)
+      - record: fc:mq:confirm_p99_seconds
+        expr: histogram_quantile(0.99, sum by (le) (rate(fc_mq_confirm_duration_seconds_bucket[1m])))
+      # F4: the REAL effective prefetch — the HOL-utilisation divisor.
+      - record: fc:mq:prefetch_effective
+        expr: max by (service) (fc_mq_prefetch_effective)
+      # The register-trigger meter (debt #3 ≥1k, debt #2/Kafka ≥5-10k): the
+      # 1k/5k reference lines live on the dashboard panel.
+      - record: fc:ingest:events_per_second
+        expr: sum(rate(fc_mq_publish_total[1m]))
+      - record: fc:ingest:events_per_second_by_class
+        expr: sum by (queue_class) (rate(fc_mq_publish_total[1m]))
+      # Event-loop lag off prom-client's default monitorEventLoopDelay.
+      - record: fc:loop:lag_p99_seconds
+        expr: max by (service) (nodejs_eventloop_lag_p99_seconds)
+      # gRPC per target — the generic package/grpc seam (P1b #969, F3):
+      # channel readiness from the 5s state poll, in-flight from the
+      # per-attempt wrap (issue-without-settle = the park signature).
+      - record: fc:grpc:ready
+        expr: max by (service, target) (fc_grpc_ready)
+      - record: fc:grpc:inflight
+        expr: sum by (service, target) (fc_grpc_inflight)
+      - record: fc:grpc:transient_failure_rate
+        expr: sum by (service, target) (rate(fc_grpc_transient_failure_seconds_total[5m]))
+
+  - name: fc-pgbouncer
+    interval: 15s
+    rules:
+      # pgbouncer_exporter families with the db_family collapse (P1c #970).
+      # Dark locally until the Q12 event lands the stats console; live in CI
+      # (the per-run compose carries the committed STATS_USERS edit).
+      - record: fc:pgbouncer:cl_active
+        expr: >-
+          sum by (db_family) (
+            label_replace(
+              label_replace(
+                label_replace(
+                  label_replace(pgbouncer_pools_client_active_connections, "db_family", "other", "database", ".*"),
+                  "db_family", "e2e", "database", "e2e_.*"),
+                "db_family", "integration", "database", "integration_.*"),
+              "db_family", "standing", "database", "forgecart")
+          )
+      - record: fc:pgbouncer:cl_waiting
+        expr: >-
+          sum by (db_family) (
+            label_replace(
+              label_replace(
+                label_replace(
+                  label_replace(pgbouncer_pools_client_waiting_connections, "db_family", "other", "database", ".*"),
+                  "db_family", "e2e", "database", "e2e_.*"),
+                "db_family", "integration", "database", "integration_.*"),
+              "db_family", "standing", "database", "forgecart")
+          )
+      # sv_active carries the same cluster or-branch (cnpg_pgbouncer_pools_sv_active).
+      - record: fc:pgbouncer:sv_active
+        expr: >-
+          sum by (db_family) (
+            label_replace(
+              label_replace(
+                label_replace(
+                  label_replace(
+                    (pgbouncer_pools_server_active_connections or cnpg_pgbouncer_pools_sv_active),
+                    "db_family", "other", "database", ".*"),
+                  "db_family", "e2e", "database", "e2e_.*"),
+                "db_family", "integration", "database", "integration_.*"),
+              "db_family", "standing", "database", "forgecart")
+          )
+      - record: fc:pgbouncer:sv_idle
+        expr: >-
+          sum by (db_family) (
+            label_replace(
+              label_replace(
+                label_replace(
+                  label_replace(pgbouncer_pools_server_idle_connections, "db_family", "other", "database", ".*"),
+                  "db_family", "e2e", "database", "e2e_.*"),
+                "db_family", "integration", "database", "integration_.*"),
+              "db_family", "standing", "database", "forgecart")
+          )
+      # Cluster or-branch (#973): CNPG's managed pooler exports its pool stats
+      # under cnpg_pgbouncer_pools_* — same strip-collision as backends above.
+      # CNPG splits maxwait into whole seconds + a micros remainder (live-scrape
+      # verified 2026-08-21; there is NO *_maxwait_seconds family) — recompose
+      # for local-parity precision.
+      - record: fc:pgbouncer:maxwait_seconds
+        expr: >-
+          max(
+            pgbouncer_pools_client_maxwait_seconds
+            or (cnpg_pgbouncer_pools_maxwait + cnpg_pgbouncer_pools_maxwait_us / 1e6)
+          )
+      # F5 — the raise-together invariant panel's rule: BOTH sides summed,
+      # denominator SCOPED (standing locally; prod aggregates across the two
+      # pooler replicas + per-DB pools at P4). PromQL's default matching
+      # returns EMPTY if either side isn't aggregated, and on() without sum()
+      # silently overstates the share — this exact rule is the audit's F5.
+      - record: fc:pgbouncer:pinned_share
+        expr: >-
+          sum(fc:pool:in_use{lane="held-tx"})
+          /
+          sum(fc:pgbouncer:sv_active{db_family="standing"})
+
+  - name: fc-convoy
+    interval: 15s
+    rules:
+      # §4.2 — one NAMED, legible boolean per multiplexing layer. The same
+      # file drives the panels AND the prod alerts (P4): sustained-ness lives
+      # in the alert `for:`, the booleans stay instantaneous truth.
+      - record: fc:convoy:heldtx
+        expr: max(fc:pool:waiting{lane="held-tx"}) > bool 0
+      - record: fc:convoy:pgbouncer
+        expr: max(fc:pgbouncer:cl_waiting) > bool 0
+      # Citus fan-out pressure against the compose CITUS_MAX_SHARED_POOL_SIZE
+      # (320) — 80% is the convoy line; P4 parameterizes per env.
+      - record: fc:convoy:citus_fanout
+        expr: max(sum by (pg_node) (fc:pg:citus_backends{source="citus"})) > bool 256
+      # Park signature (F3): RPCs in flight into a channel that is not ready.
+      - record: fc:convoy:grpc_parked
+        expr: >-
+          max by (service, target) (
+            (fc:grpc:inflight > bool 0) * (fc:grpc:ready == bool 0)
+          )
+      # Prefetch head-of-line: unacked pinned at consumers × the REAL
+      # effective prefetch (F4's divisor, never a hardcoded ×10). The second
+      # factor requires live consumers — zero-consumer backlog is an outbox
+      # signal, not HOL. Empty until prefetch_effective exists (honest gap).
+      - record: fc:convoy:prefetch_hol
+        expr: >-
+          (
+            max(fc:mq:unacked_total)
+            >= bool
+            clamp_min(max(fc:mq:consumers) * max(fc:mq:prefetch_effective), 1)
+          )
+          * (max(fc:mq:consumers) > bool 0)
+
+  # ── P4 (#973): leak row + outbox + the F1 workspace family ─────────────────
+  - name: fc-leak
+    interval: 15s
+    rules:
+      # Row 8 (leak detection, prod-first and identical locally): CLOSE_WAIT
+      # by actor from cAdvisor's tcp keep-list — a held-open peer socket the
+      # app never reaped; a climbing per-actor count is the classic fd leak.
+      - record: fc:leak:close_wait
+        expr: >-
+          max by (actor) (
+            label_replace(container_network_tcp_usage_total{tcp_state="close_wait"},
+              "actor", "$1", "name", "(.+)")
+          )
+      # RSS slope per actor (bytes/s over 30m) — the leak view that catches
+      # slow growth long before any fence: > ~1 MB/min sustained is a leak
+      # candidate; pair with fc:host:mem_available_ratio for urgency.
+      - record: fc:leak:rss_slope_bytes_per_s
+        expr: deriv(fc:actor:rss_bytes[30m])
+  - name: fc-outbox
+    interval: 15s
+    rules:
+      # FcOutboxOldest's feed — from the SAME pg-custom-queries.yaml the prod
+      # CNPG ConfigMap renders (pg_outbox query, #973).
+      - record: fc:outbox:oldest_seconds
+        expr: max(pg_outbox_oldest_unsent_seconds)
+      - record: fc:outbox:backlog
+        expr: max(pg_outbox_backlog)
+  - name: fc-workspace
+    interval: 15s
+    rules:
+      # F1 (CRITICAL) — bounded workspace-pod FAMILY series: no channel-code
+      # label EVER (per-tenant labels are the one tenant-count-proportional
+      # axis). tier derives from NAMESPACE at scrape time (prod vmagent
+      # relabel: shop-workspaces-* → workspace-pod, runner ns → ci-runner);
+      # locally the pods nest inside the k3s container and these evaluate
+      # empty — the honest local state; per-pod truth lives in Plane B/replay.
+      # The tenant-count counter survives as the pods_count sum (debt #1's
+      # ~1k trigger line rides FcCardinalityBudget).
+      - record: fc:workspace:pods_count
+        expr: count by (tier) (container_memory_working_set_bytes{tier=~"workspace-pod|ci-runner"})
+      - record: fc:workspace:rss_bytes
+        expr: >-
+          sum by (tier) (container_memory_working_set_bytes{tier=~"workspace-pod|ci-runner"})
+      - record: fc:workspace:rss_bytes_max
+        expr: >-
+          max by (tier) (container_memory_working_set_bytes{tier=~"workspace-pod|ci-runner"})
+      - record: fc:workspace:cpu_cores
+        expr: >-
+          sum by (tier) (rate(container_cpu_usage_seconds_total{tier=~"workspace-pod|ci-runner"}[1m]))
+
+  - name: fc-self
+    interval: 15s
+    rules:
+      # Cardinality self-watch (C5): active series vs the 60k-local ceiling.
+      # The FcCardinalityBudget alert + the formal budget panel land in P1c;
+      # this rule is the placeholder feed (#967 acceptance, C5 partial).
+      - record: fc:self:active_series
+        expr: vm_cache_entries{type="storage/tsid"}

package/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@forgecart/flight-observability",
+  "version": "0.202608212205.0",
+  "private": false,
+  "description": "Flight Deck dashboards, fc: recording/alert rules, and the pg-custom-queries contract — the version-pinned sync artifact between shop-beta and forgecart/cloud (launch#46 P4).",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  }
+}