@damiangilgonzalez/harness 1.3.7

package/skills/notion-connector/scripts/verify.sh

@@ -0,0 +1,162 @@
+#!/usr/bin/env bash
+#
+# verify.sh — static lint for Notion connector code (outbound Notion API).
+#
+# WHAT IT DOES (read-only; never edits, never hits the network, no token needed)
+#   Greps target .py/.ts/.tsx/.js/.jsx files that touch the Notion API and flags
+#   the classic post-2025-09-03 mistakes:
+#     1. No pinned Notion-Version / notionVersion -> FAIL (rides the moving
+#        default; behavior drifts across versions).
+#     2. A deprecated databases/:id/query query path -> FAIL (that endpoint 404s
+#        on 2025-09-03+; query against /v1/data_sources/:id/query instead).
+#     3. A data-source query with no pagination loop (no has_more AND no
+#        next_cursor) -> FAIL (silently drops every row past 100).
+#     4. No 429 / Retry-After handling anywhere in a file that queries -> WARN
+#        (the integration is capped at ~3 req/s; over-limit returns 429).
+#     5. A hardcoded Notion token literal (ntn_.../secret_...) -> FAIL
+#        (committed bearer secret = full workspace access; read it from env).
+#
+# EXIT CODES
+#   0  clean, or no relevant file to inspect (empty/clean target is NOT a failure)
+#   1  at least one [fail] finding
+#   2  bad usage
+#
+# HOW TO RUN (point it at YOUR code, not the skills repo)
+#   ./verify.sh app.ts                 # lint one file
+#   ./verify.sh --path src/            # lint every supported file under a dir
+#   ./verify.sh                        # scan ./ ; if nothing matches, skip + exit 0
+#
+# Runs on stock macOS bash 3.2: no mapfile, no associative arrays.
+
+set -euo pipefail
+
+if [ -t 1 ]; then
+  RED=$'\033[31m'; GREEN=$'\033[32m'; YELLOW=$'\033[33m'; NC=$'\033[0m'
+else
+  RED=''; GREEN=''; YELLOW=''; NC=''
+fi
+
+ok()   { printf '%s[ ok ]%s %s\n' "$GREEN"  "$NC" "$*"; }
+skip() { printf '%s[skip]%s %s\n' "$YELLOW" "$NC" "$*"; }
+warn() { printf '%s[warn]%s %s\n' "$YELLOW" "$NC" "$*"; }
+fail() { printf '%s[fail]%s %s\n' "$RED"    "$NC" "$*"; }
+
+usage() { sed -n '2,38p' "$0" | sed 's/^# \{0,1\}//'; }
+
+# --- arg parse --------------------------------------------------------------
+SCAN_PATH=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --path)    SCAN_PATH="${2:?--path needs a value}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    -*)        printf '%sUnknown argument: %s%s\n\n' "$RED" "$1" "$NC"; usage; exit 2 ;;
+    *)         SCAN_PATH="$1"; shift ;;
+  esac
+done
+[ -z "$SCAN_PATH" ] && SCAN_PATH="."
+
+if [ ! -e "$SCAN_PATH" ]; then
+  printf '%sPath not found: %s%s\n' "$RED" "$SCAN_PATH" "$NC"; exit 2
+fi
+
+# --- collect candidate source files ----------------------------------------
+if [ -f "$SCAN_PATH" ]; then
+  FILES="$SCAN_PATH"
+else
+  FILES="$(find "$SCAN_PATH" -type f \
+    \( -name '*.py' -o -name '*.ts' -o -name '*.tsx' -o -name '*.js' -o -name '*.jsx' \) \
+    2>/dev/null || true)"
+fi
+
+# Keep only files that actually touch the Notion API (so we never false-fail on
+# unrelated code).
+RELEVANT=""
+if [ -n "$FILES" ]; then
+  while IFS= read -r f; do
+    [ -z "$f" ] && continue
+    if grep -iEq '@notionhq/client|api\.notion\.com|notionVersion|Notion-Version|data_sources|NOTION_TOKEN' "$f" 2>/dev/null; then
+      RELEVANT="$RELEVANT$f
+"
+    fi
+  done <<EOF
+$FILES
+EOF
+fi
+
+# Empty / clean target is NOT a failure.
+if [ -z "$RELEVANT" ]; then
+  skip "no Notion connector code found under '$SCAN_PATH' — nothing to lint"
+  exit 0
+fi
+
+n_files="$(printf '%s' "$RELEVANT" | grep -c . || true)"
+printf 'Linting %s Notion connector file(s) under: %s\n\n' "$n_files" "$SCAN_PATH"
+
+fail_total=0
+
+while IFS= read -r f; do
+  [ -z "$f" ] && continue
+  printf '%s— %s%s\n' "$YELLOW" "$f" "$NC"
+
+  # 1. pinned version present? (notionVersion option or Notion-Version header)
+  if grep -iEq 'notionVersion|Notion-Version' "$f" 2>/dev/null; then
+    ok "  Notion version is pinned"
+  else
+    fail "  no pinned Notion-Version / notionVersion — pin it (e.g. 2025-09-03):"
+    fail_total=$((fail_total + 1))
+  fi
+
+  # 2. deprecated databases/:id/query path -> FAIL
+  if grep -inE 'databases/[^"'\'' ]*/query|databases\.query' "$f" 2>/dev/null | grep -vq 'data_sources'; then
+    hits="$(grep -inE 'databases/[^"'\'' ]*/query|databases\.query' "$f" 2>/dev/null | grep -v 'data_sources' || true)"
+    fail "  deprecated databases/:id/query path — 404s on 2025-09-03+; use data_sources/:id/query:"
+    printf '%s\n' "$hits" | head -n 5 | sed 's/^/        /'
+    fail_total=$((fail_total + 1))
+  else
+    ok "  no deprecated databases/:id/query path"
+  fi
+
+  # Does this file query a data source at all?
+  queries=0
+  grep -iEq 'data_sources/[^"'\'' ]*/query|dataSources\.query|/v1/data_sources' "$f" 2>/dev/null && queries=1
+
+  if [ "$queries" -eq 1 ]; then
+    # 3. pagination loop present? need has_more AND next_cursor referenced
+    if grep -q 'has_more' "$f" 2>/dev/null && grep -q 'next_cursor' "$f" 2>/dev/null; then
+      ok "  query paginates (has_more + next_cursor)"
+    else
+      fail "  data-source query without a pagination loop — drops rows past 100; loop on has_more + next_cursor:"
+      fail_total=$((fail_total + 1))
+    fi
+
+    # 4. 429 / Retry-After handling -> WARN
+    if grep -iEq '429|Retry-After|retry-after|retryAfter' "$f" 2>/dev/null; then
+      ok "  handles 429 / Retry-After"
+    else
+      warn "  no 429 / Retry-After handling found — the integration is capped at ~3 req/s"
+    fi
+  fi
+
+  # 5. hardcoded Notion token literal -> FAIL
+  hard="$(grep -inE '["'\''](ntn_|secret_)[A-Za-z0-9]{8,}' "$f" 2>/dev/null \
+            | grep -viE 'os\.environ|process\.env|getenv|(^|[[:space:]])//|(^|[[:space:]])#' || true)"
+  if [ -n "$hard" ]; then
+    fail "  hardcoded Notion token literal — read it from an env var instead:"
+    printf '%s\n' "$hard" | head -n 5 | sed 's/^/        /'
+    fail_total=$((fail_total + 1))
+  else
+    ok "  Notion token is read from an env var (no inline literal)"
+  fi
+
+  printf '\n'
+done <<EOF
+$RELEVANT
+EOF
+
+cat <<'NOTE'
+Note: [fail] = a bug that breaks the call, drops data, or leaks a secret (fix it).
+      [warn] = resilience gap (confirm 429/Retry-After is handled somewhere upstream).
+NOTE
+
+if [ "$fail_total" -gt 0 ]; then exit 1; fi
+exit 0

package/skills/observability/SKILL.md

@@ -0,0 +1,235 @@
+---
+name: observability
+description: "Use when instrumenting a service from the inside so an incident can be explained from telemetry alone — wiring OpenTelemetry logs, metrics and traces, standing up a Collector, exporting via OTLP, and defining telemetry-driven alerts. NOT outside-in uptime probes, on-call rotation, or who-gets-paged (that is `monitoring`)."
+tags: [observability, opentelemetry, tracing, metrics, structured-logging]
+recommends: [monitoring, error-handling, performance, cost-tracking, docker]
+origin: risco
+---
+
+# Observability
+
+You are wiring the *inside* view of a service: when something breaks at 3am, an engineer must be able to answer "what happened, where, and why" from telemetry alone — without adding a `console.log` and redeploying into the fire. This skill emits a concrete artifact: SDK init code, instrumentation (spans/metrics/structured logs), a Collector config, and alert rules that the instrumentation makes possible. The outside-in half — is it up, who gets paged — is `../monitoring/SKILL.md`.
+
+## The one rule
+
+**Every signal carries the same correlation identity: `trace_id`, `service.name`, `deployment.environment`.** A log line you cannot pivot to its trace, or a spiking metric you cannot pivot to an exemplar span, doubles your mean-time-to-resolution — you are back to grepping. Three signals that don't share keys are three disconnected tools; three that do are one queryable system. Set the resource once at SDK init, inject `trace_id`/`span_id` into every log, and never emit a metric you can't tie back to a service and environment.
+
+## Start from the operator's questions
+
+Before choosing signals, write **two to four questions on-call must answer during the likely incident**. For example: “Are payment retries recovering?”, “Which dependency and failure class drives exhaustion?”, “Can one payment be charged twice?”, “Which customer-visible operations need intervention now?” Then assign the cheapest signal: metrics say **that/how much**, traces show **where/causal path**, logs explain **why for this event**. If a proposed event or label answers none of the questions, do not emit it.
+
+## The three pillars — when each earns its place
+
+Don't emit all three of everything. Each signal answers a different question at a different cost.
+
+| Signal | Answers | Cost | Alert on it? | Main gotcha |
+|---|---|---|---|---|
+| **Logs** | "what exactly happened in this one event" | high per-event, cheap to skip | rarely (noisy) | high-cardinality fields belong in the *body*, not in stream labels |
+| **Metrics** | "what's the rate/aggregate over time" | cheap, pre-aggregated | yes — this is your alert source | cardinality explosion if a label is unbounded |
+| **Traces** | "what was the causal path across hops, and where did time go" | medium; sample it | indirectly (via derived RED metrics) | one giant span = no causality; sample or you pay for noise |
+
+The "fourth pillar," **continuous profiling** (CPU/heap flame graphs over time), is now a first-class OTel signal — reach for it only when traces say "the time is inside *this* function" and you need to know which line.
+
+## Architecture
+
+Instrument once, route anywhere. The app talks OTLP to a Collector; the Collector fans the firehose out to backends.
+
+```text
+  ┌─────────────┐   OTLP/gRPC :4317        ┌───────────────────────────┐
+  │  app + SDK  │   OTLP/HTTP :4318  ───▶   │       OTel Collector       │
+  │ (resource:  │   /v1/traces             │  receivers → processors    │
+  │ service.name│   /v1/metrics            │  → exporters (per signal)  │
+  │  +env+ver)  │   /v1/logs               │  wired in service.pipelines│
+  └─────────────┘                          └───────────────────────────┘
+                                                 │        │        │
+                                            logs │ traces │ metrics│
+                                                 ▼        ▼        ▼
+                                              Loki     Tempo   Mimir/Prom   (+ Grafana to view)
+                                                  └── or a single vendor: Datadog / Honeycomb ──┘
+```
+
+OTLP is the wire format: **gRPC on 4317** (TLS + gzip by default), **HTTP on 4318** with per-signal paths `/v1/traces`, `/v1/metrics`, `/v1/logs`. Always export to a Collector, never straight to the vendor. Why: the Collector gives you one place to **batch** (fewer round-trips), **retry** (survive a backend blip), **redact** PII, and **swap or add a backend without redeploying the app**. App SDKs should be dumb pipes; policy lives in the Collector.
+
+## Instrument in the right order: auto first, manual second
+
+**Never hand-roll a span for something auto-instrumentation already covers** (HTTP servers, DB clients, queues). You will miss edges and waste effort. Turn on zero-code instrumentation, confirm traces flow, *then* add manual spans only where your business logic lives.
+
+```bash
+# Node — zero-code, no app changes. SDK is 2.0+; the register hook is compatible.
+npm i @opentelemetry/api @opentelemetry/auto-instrumentations-node
+OTEL_SERVICE_NAME=checkout-api \
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=prod,service.version=1.4.2 \
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+node --require '@opentelemetry/auto-instrumentations-node/register' app.js
+```
+
+```bash
+# Python — zero-code via the launcher; it patches known libraries on import.
+pip install opentelemetry-distro opentelemetry-exporter-otlp
+opentelemetry-bootstrap -a install
+OTEL_SERVICE_NAME=checkout-api \
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=prod,service.version=1.4.2 \
+opentelemetry-instrument python app.py
+```
+
+Then add a manual span only around a meaningful business operation — and give it attributes and a status, or it tells you nothing:
+
+```javascript
+// Bad — a span with no attributes and no status. You learn that "something ran."
+const span = tracer.startSpan('work');
+await chargeCard(order);
+span.end();
+```
+
+```javascript
+// Good — named for the business op, carries the inputs you'd filter by, records outcome.
+const { trace, SpanStatusCode } = require('@opentelemetry/api');
+const tracer = trace.getTracer('checkout');
+await tracer.startActiveSpan('charge_card', async (span) => {
+  span.setAttribute('order.id', order.id);          // searchable dimension
+  span.setAttribute('payment.provider', 'stripe');
+  try {
+    await chargeCard(order);
+    span.setStatus({ code: SpanStatusCode.OK });
+  } catch (err) {
+    span.recordException(err);                        // attaches the stack as an event
+    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+    throw err;
+  } finally {
+    span.end();                                       // a span you never end leaks forever
+  }
+});
+```
+
+Per-stack init (Node SDK 2.0 manual setup, Go SDK, context propagation across HTTP/queue hops, the GenAI span template) lives in `references/instrumentation-recipes.md`.
+
+## Resource and semantic conventions
+
+**Use the standard attribute names; never invent bespoke keys.** The whole correlation story and every prebuilt backend dashboard assume `http.*`, `db.*`, `gen_ai.*`, `service.*`. A homegrown `mycompany.endpoint` attribute is invisible to every tool that expects `http.route`.
+
+- Set on the resource, once: `service.name` (required — unset means telemetry lands as `unknown_service`), `service.version`, `deployment.environment`.
+- Kubernetes attributes (`k8s.*`) reached release candidate (2026-03); DB conventions are on their 2nd RC. Prefer the standard names even while an area is still stabilizing.
+- For LLM calls, **GenAI conventions exited experimental for client spans (early 2026)** — use `gen_ai.*` so the same span feeds your spend view:
+
+```javascript
+// LLM call as a span: model + token attrs. These attributes feed ../cost-tracking/SKILL.md.
+await tracer.startActiveSpan('chat gpt-4o', async (span) => {
+  span.setAttribute('gen_ai.system', 'openai');
+  span.setAttribute('gen_ai.request.model', 'gpt-4o');
+  const res = await openai.chat.completions.create({ /* ... */ });
+  span.setAttribute('gen_ai.usage.input_tokens', res.usage.prompt_tokens);
+  span.setAttribute('gen_ai.usage.output_tokens', res.usage.completion_tokens);
+  span.end();
+});
+```
+
+This skill emits the *signal* (token attributes on a span). Turning those tokens into a dollar figure and a budget is `../cost-tracking/SKILL.md`.
+
+## Structured, correlated logs
+
+**Logs are JSON, carry the trace context, and never carry PII.** Free-text `print` lines cost you twice: you can't query them, and you can't jump from the log to its trace.
+
+```python
+# Bad — unstructured, unsearchable, unlinkable to a trace.
+print("charged user " + email + " amount " + str(amount))
+```
+
+```python
+# Good — structured, leveled, correlated, no PII (id not email).
+import logging, json
+from opentelemetry import trace
+
+def log_charge(order_id, amount):
+    ctx = trace.get_current_span().get_span_context()
+    logging.info(json.dumps({
+        "event": "charge.succeeded",
+        "level": "info",
+        "order_id": order_id,            # an opaque id, not the customer's email
+        "amount_cents": amount,
+        "trace_id": format(ctx.trace_id, "032x"),  # ← the pivot back to the trace
+        "span_id": format(ctx.span_id, "016x"),
+    }))
+```
+
+Level discipline: `error` = a human should look, `warn` = degraded but handled, `info` = business milestones, `debug` = off in prod. If everything is `error`, nothing is.
+
+## Metrics that earn an alert
+
+**RED for request-driven services, USE for finite resources.** Alerts come from metrics — logs and traces are for *investigating* the alert, not firing it.
+
+- **RED** (per service/endpoint): **R**ate (requests/s), **E**rrors (failed/s), **D**uration (latency as a *histogram*, so you can read p50/p95/p99 — never a single average, which hides the tail).
+- **USE** (per resource — CPU, pool, disk): **U**tilization, **S**aturation (queue depth / waiting), **E**rrors.
+
+**The cardinality rule — this is the #1 way an observability stack falls over.** A metric's total series count is the product of its label cardinalities. Put an unbounded value on a label and you create a near-infinite series count; the time-series database (Prometheus/Mimir) is most often restarted because of exactly this. Loki indexes *labels only, not log contents* — so the same rule binds its stream labels.
+
+```text
+# Bad — user_id is unbounded; 5M users = 5M series per metric. OOMs the TSDB.
+http_requests_total{route="/checkout", user_id="u_8f3a...", status="200"}
+```
+
+```text
+# Good — only bounded, low-cardinality dimensions on the metric.
+http_requests_total{route="/checkout", method="POST", status="200"}
+# Need to slice by user? That's a trace attribute or a log field, never a metric label.
+```
+
+## The Collector config
+
+A minimal valid pipeline. An exporter is **inert until it appears in a pipeline** — defining one under `exporters:` does nothing on its own.
+
+```yaml
+# otel-collector.yaml — Collector v0.153.0 shape
+receivers:
+  otlp:
+    protocols:
+      grpc: { endpoint: 0.0.0.0:4317 }
+      http: { endpoint: 0.0.0.0:4318 }
+
+processors:
+  memory_limiter:                 # first line of defense: shed load before OOM
+    check_interval: 1s
+    limit_percentage: 80
+  batch: {}                       # batch before export — fewer, bigger round-trips
+  redaction:                      # strip PII before it leaves your network
+    allow_all_keys: true
+    blocked_values: ["[0-9]{13,16}", "\\b[\\w.]+@[\\w.]+\\b"]  # PANs, emails
+
+exporters:
+  otlphttp/traces:  { endpoint: http://tempo:4318 }
+  otlphttp/logs:    { endpoint: http://loki:3100/otlp }
+  otlphttp/metrics: { endpoint: http://mimir:9009/otlp }
+
+service:
+  pipelines:
+    traces:  { receivers: [otlp], processors: [memory_limiter, redaction, batch], exporters: [otlphttp/traces] }
+    logs:    { receivers: [otlp], processors: [memory_limiter, redaction, batch], exporters: [otlphttp/logs] }
+    metrics: { receivers: [otlp], processors: [memory_limiter, batch],            exporters: [otlphttp/metrics] }
+```
+
+Tail sampling, gateway-vs-agent topology, multi-backend fan-out (LGTM **and** a vendor in parallel), and `resourcedetection` live in `references/collector-config.md`. Shipping the Collector as a container or in CI is `../docker/SKILL.md`.
+
+## Alerts the telemetry now enables
+
+The instrumentation above makes these *possible* — defining them is your job; routing the page to a human is `../monitoring/SKILL.md`.
+
+- **SLO burn-rate (multi-window)** — alert when you're spending the error budget too fast, using a fast window (e.g. 5m) AND a slow window (e.g. 1h) so a brief blip doesn't page but a sustained burn does. This fires on *impact*, not on traffic.
+- **Latency SLO** — p99 of the duration histogram over budget for a sustained window.
+- **Saturation** — the USE "S": queue depth / pool waiters climbing, the leading indicator before errors appear.
+
+## Anti-patterns
+
+| Anti-pattern | Why it bites | Do instead |
+|---|---|---|
+| Unbounded label (`user_id`, `request_id`, `email`) on a metric or Loki stream | Cardinality explosion → TSDB OOM/restart, cost blowup | Keep labels low-cardinality; put the high-cardinality field on a span attribute or log body |
+| Logging PII / secrets (email, card, token) | Compliance breach + the leak is now in every log backend | Log opaque ids; redact in the Collector before export |
+| 100% trace sampling in prod, no policy | Pay to store noise; backend throttles and drops the traces you needed | Head/tail sampling — keep all errors + slow traces, sample the rest |
+| App exports straight to the vendor, no Collector | Can't batch, retry, redact, or swap backends without a redeploy | Always route through a Collector |
+| Instrument everything before deciding the question | Noise with no signal; nobody opens the dashboard | Start from "what would I ask during an incident," instrument that path |
+| Alert on a raw error *count* | Fires on traffic spikes, silent during a low-traffic outage | Alert on error *rate* / SLO burn |
+| One giant span per request (or a thousand contentless ones) | No causality, or context with no detail — both useless | Span per meaningful operation, each with attributes + status |
+
+## Verify
+
+Run `scripts/verify.sh` against the directory holding your Collector config + SDK init. It checks the config is valid, that every defined exporter is actually wired into a pipeline (the classic "defined but unused" footgun), that `service.name` is set, and warns on high-cardinality metric labels. It is read-only and exits 0 when there's nothing to check.
+
+Then prove the wire with one safe induced failure in a test/staging path. Confirm the expected error-rate metric changes, the trace records the failing operation and error status, and the structured log carries the same `trace_id` without PII. Query the backend/Collector output; “the instrumentation code ran” is not evidence that usable telemetry arrived. Live paging and escalation proof remains `../monitoring/SKILL.md`.

package/skills/observability/evals/README.md

@@ -0,0 +1,3 @@
+# Evals for observability
+
+These cases are eyeballed, not automated. Read `cases.yaml` and check three things by hand. First, every `should_trigger` prompt should make this skill the obvious pick — pay special attention to the non-obvious ones (the "can't see where the time goes across services" latency-symptom line, and the "LLM calls as spans with token counts" line, since those are where a thinner trigger set would miss). Second, every `should_not_trigger` prompt must route cleanly to its named sibling — the sharp boundary is `monitoring` (it emits no signals, it decides the page) and `cost-tracking` (it prices the gen_ai.* tokens this skill emits); if either of those feels like it could legitimately land here, tighten the description. Third, run the `capability` scenario against a generated answer and confirm each `must_include` item is actually present, not hand-waved — especially that every Collector exporter is wired into a pipeline, that the manual span sets status and attributes, that logs carry trace_id/span_id with no PII, and that no metric uses a high-cardinality label. Note that `scripts/verify.sh` checks the emitted artifact (Collector config + SDK init), not the prose: a great write-up that emits no wired-up config has not finished the job.

package/skills/observability/evals/cases.yaml

@@ -0,0 +1,51 @@
+skill: observability
+
+should_trigger:
+  - prompt: "Add OpenTelemetry traces, metrics and logs to our API."
+    why: Core trigger — wire the three signals through OTel. The skill's central job.
+  - prompt: "Requests are slow and we can't see where the time goes across our services."
+    why: Non-obvious wording (no mention of OTel/tracing). The symptom maps to distributed tracing — spans across hops to attribute latency.
+  - prompt: "Replace our print/console.log statements with structured logs correlated to traces."
+    why: The structured-logging pillar — JSON logs injected with trace_id/span_id so you can pivot log → trace.
+  - prompt: "Instrument our LLM calls so each request is a span with token counts."
+    why: Non-obvious. GenAI semantic conventions (gen_ai.* token attrs) on spans; the bridge that feeds cost-tracking, but the instrumentation itself lives here.
+  - prompt: "Stand up an OTel Collector and export our telemetry to Grafana."
+    why: Collector config + OTLP export + per-signal pipelines — the architecture core.
+  - prompt: "Define RED metrics and a burn-rate alert from the telemetry we emit."
+    why: Telemetry-driven alerting that the instrumentation makes possible (rate/errors/duration + multi-window SLO burn).
+  - prompt: "Instrumenta el servei amb OpenTelemetry i exporta a Grafana."
+    why: Catalan trigger for the core instrument-and-export flow.
+
+should_not_trigger:
+  - prompt: "Alert me when the site goes down and set up an on-call rotation."
+    route_to: monitoring
+    why: Outside-in detection + who-gets-paged. Observability emits the signals; monitoring decides the page and the human.
+  - prompt: "Design our retry strategy, circuit breaker, and error taxonomy."
+    route_to: error-handling
+    why: How the code handles failure, not how failure is recorded as telemetry.
+  - prompt: "This endpoint takes 3 seconds — make it fast."
+    route_to: performance
+    why: Fixing slowness. Observability makes slowness visible/attributable; performance does the actual optimization.
+  - prompt: "Track user signups and our activation funnel conversion."
+    route_to: analytics
+    why: Product/business events for growth, not system/operational signals.
+  - prompt: "Build a dashboard of our monthly LLM spend by model and team."
+    route_to: cost-tracking
+    why: Turning token usage into dollars and budgets. Observability emits the gen_ai.* token span; cost-tracking prices it.
+
+capability:
+  - scenario: >
+      I have a Node service that uses console.log for everything and has no metrics.
+      Make it observable: I want to explain an incident from telemetry alone.
+    must_include:
+      - SDK init that sets resource attributes service.name, service.version, and deployment.environment
+      - auto-instrumentation enabled (zero-code register hook or getNodeAutoInstrumentations) before any manual spans
+      - one manual span around a business operation that carries attributes AND sets span status (OK/ERROR) and records exceptions
+      - structured JSON logs that inject trace_id and span_id (not free-text print/console.log)
+      - a RED metric set — request rate, error rate, and latency as a histogram (not a single average)
+      - a Collector config where every exporter under exporters is referenced in a service.pipelines entry
+      - a multi-window SLO burn-rate alert defined from the emitted metrics
+      - respects the cardinality rule (no user_id/request_id/email as a metric label) and logs no PII (opaque ids, redaction in the Collector)
+      - exports through a Collector via OTLP, not straight to a vendor
+      - starts from two to four concrete on-call questions and maps metrics to that/how much, traces to where, and logs to why
+      - induces one safe failure and proves the correlated metric, error trace, and structured log actually arrive without PII

package/skills/observability/references/collector-config.md

@@ -0,0 +1,98 @@
+# Collector config — fuller recipes
+
+Branch-specific Collector depth offloaded from `SKILL.md`. Shapes target Collector **v0.153.0**. An exporter or processor is inert until it is named in a `service.pipelines` entry.
+
+## Gateway vs agent topology
+
+Two deployment shapes; most production setups run both as a tiered pipeline.
+
+- **Agent** — one Collector per host/pod (sidecar or DaemonSet), close to the app. Cheap network hop, does fast local work: `memory_limiter`, `batch`, `resourcedetection`. Forwards via OTLP to the gateway.
+- **Gateway** — a horizontally-scaled standalone tier. Does the expensive, stateful work that needs a *whole-trace* view: `tail_sampling`, cross-service `redaction`, per-tenant routing, backend fan-out. Scale it independently of your apps.
+
+Rule: do per-span/per-host work in the agent, do whole-trace and policy work in the gateway. Tail sampling in an agent is wrong — an agent never sees a full trace.
+
+```yaml
+# agent (DaemonSet) → forwards everything to the gateway
+exporters:
+  otlp/gateway: { endpoint: otel-gateway:4317, tls: { insecure: false } }
+service:
+  pipelines:
+    traces:  { receivers: [otlp], processors: [memory_limiter, resourcedetection, batch], exporters: [otlp/gateway] }
+    metrics: { receivers: [otlp], processors: [memory_limiter, batch],                     exporters: [otlp/gateway] }
+    logs:    { receivers: [otlp], processors: [memory_limiter, batch],                     exporters: [otlp/gateway] }
+```
+
+## Tail sampling (gateway only)
+
+Head sampling decides at span start, before you know if the request was interesting. **Tail sampling** buffers the whole trace and decides at the end — so you keep every error and every slow trace, and sample the boring fast ones. It must run on the gateway, after a `groupbytrace`-style stage so all spans of a trace land on the same Collector.
+
+```yaml
+processors:
+  tail_sampling:
+    decision_wait: 10s            # how long to buffer a trace before deciding
+    num_traces: 100000
+    policies:
+      - name: keep-errors
+        type: status_code
+        status_code: { status_codes: [ERROR] }     # never drop a failed trace
+      - name: keep-slow
+        type: latency
+        latency: { threshold_ms: 1000 }             # never drop a slow trace
+      - name: sample-the-rest
+        type: probabilistic
+        probabilistic: { sampling_percentage: 5 }   # 5% of the boring ones
+```
+
+## Redaction / attributes for PII
+
+Strip sensitive data before it leaves your network. `redaction` masks values by regex; `attributes` deletes or hashes specific keys.
+
+```yaml
+processors:
+  redaction:
+    allow_all_keys: true
+    blocked_values:
+      - "\\b[0-9]{13,16}\\b"                # card PANs
+      - "\\b[\\w.+-]+@[\\w-]+\\.[\\w.-]+\\b" # emails
+    summary: debug                          # report what it masked, at debug
+  attributes/pii:
+    actions:
+      - { key: user.email, action: delete }
+      - { key: user.id,    action: hash }   # keep joinable, not readable
+```
+
+Wire both into the pipeline's `processors` list before the exporter, or they do nothing.
+
+## Multi-backend fan-out (LGTM + a vendor in parallel)
+
+A pipeline's `exporters` list is a fan-out — list two and every item goes to both. Useful for migrating off a vendor, or keeping a cheap self-hosted copy beside a managed one.
+
+```yaml
+exporters:
+  otlphttp/tempo:    { endpoint: http://tempo:4318 }
+  otlphttp/honeycomb:
+    endpoint: https://api.honeycomb.io
+    headers: { "x-honeycomb-team": "${env:HONEYCOMB_API_KEY}" }
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlphttp/tempo, otlphttp/honeycomb]   # both, in parallel
+```
+
+LGTM split for reference: **Loki** logs, **Tempo** traces, **Mimir/Prometheus** metrics, **Grafana** to view.
+
+## resourcedetection + env-var overrides
+
+`resourcedetection` auto-fills `host.*`, `cloud.*`, `k8s.*`, `container.*` resource attributes so you don't hand-set them per deploy. Combine with env-var interpolation (`${env:VAR}`) so the same config file works across environments — never bake secrets or per-env endpoints into the file.
+
+```yaml
+processors:
+  resourcedetection:
+    detectors: [env, system, gcp, eks]    # order = precedence
+    timeout: 2s
+exporters:
+  otlphttp/metrics:
+    endpoint: "${env:METRICS_ENDPOINT}"   # differs per environment, injected at runtime
+```