@dsalazardev/ai-hub-cli 0.0.0-stage → 0.2.0

package/skills/shared/nodejs/scripts/verify.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# verify.sh — static lint for a plain Node.js / Express service scaffold.
+# Read-only. No install or run required. Heuristic and advisory.
+#
+# HARD-FAILS (exit 1) only on:
+#   - app.ts calls .listen( (app construction not split from server)
+#   - no 4-arg (err,req,res,next) error middleware found anywhere
+# Everything else is a WARNING. Exits 0 on a clean/empty target (no false failure).
+#
+# Usage: scripts/verify.sh [dir]   (defaults to current directory)
+set -euo pipefail
+
+DIR="${1:-.}"
+HARD_FAIL=0
+WARN=0
+
+if [ ! -d "$DIR" ]; then
+  echo "verify.sh: '$DIR' is not a directory" >&2
+  exit 2
+fi
+
+warn() { echo "WARN: $*"; WARN=$((WARN + 1)); }
+fail() { echo "FAIL: $*"; HARD_FAIL=$((HARD_FAIL + 1)); }
+
+# Collect JS/TS source files, skipping deps, build output and test files.
+# bash 3.2 (macOS) friendly: newline-delimited, no mapfile.
+SRC_FILES="$(
+  find "$DIR" -type f \( -name '*.ts' -o -name '*.js' -o -name '*.mjs' \) \
+    -not -path '*/node_modules/*' \
+    -not -path '*/dist/*' \
+    -not -path '*/build/*' \
+    -not -path '*/.git/*' \
+    2>/dev/null
+)"
+
+if [ -z "$SRC_FILES" ]; then
+  echo "verify.sh: no JS/TS source files under '$DIR' — nothing to check."
+  exit 0
+fi
+
+is_test_file() {
+  case "$1" in
+    *.spec.*|*.test.*|*/test/*|*/tests/*|*/__tests__/*) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+ERROR_HANDLER_FOUND=0
+ERROR_HANDLER_LAST=1   # assume ok until a route appears after it in any app file
+
+while IFS= read -r f; do
+  [ -z "$f" ] && continue
+  is_test_file "$f" && continue
+
+  base="$(basename "$f")"
+
+  # 1. HARD: app.ts (or app.js/mjs) must not call .listen(
+  case "$base" in
+    app.ts|app.js|app.mjs)
+      if grep -Eq '\.listen\(' "$f"; then
+        fail "$f calls .listen() — split app construction from server.ts (server should listen, not app)."
+      fi
+      ;;
+  esac
+
+  # 2. Detect a 4-arg error middleware: (err, req, res, next) used with app.use.
+  if grep -Eq '\(\s*err\s*,\s*[A-Za-z_]+\s*,\s*[A-Za-z_]+\s*,\s*[A-Za-z_]+\s*\)' "$f"; then
+    ERROR_HANDLER_FOUND=1
+    # Heuristic: warn if a 4-arg error handler line precedes a route registration
+    # in the same file (app.get/post/.../use("/...")) — handler should be LAST.
+    err_line="$(grep -nE 'use\(\s*\(\s*err\s*,' "$f" | head -1 | cut -d: -f1 || true)"
+    if [ -n "${err_line:-}" ]; then
+      route_after="$(grep -nE 'app\.(get|post|put|patch|delete|use\("/)' "$f" \
+        | awk -F: -v e="$err_line" '$1 > e {print; exit}' || true)"
+      if [ -n "${route_after:-}" ]; then
+        warn "$f registers a route AFTER the 4-arg error handler — the error handler must be LAST."
+      fi
+    fi
+  fi
+
+  # 3. WARN: global unhandledRejection that does not exit/shutdown (used as control flow).
+  if grep -Eq "on\(\s*['\"]unhandledRejection['\"]" "$f"; then
+    if ! grep -Eq "(process\.exit|shutdown)" "$f"; then
+      warn "$f handles 'unhandledRejection' but never exits/shuts down — do not use it as control flow."
+    fi
+  fi
+
+  # 4. WARN: floating-promise heuristic in controllers/routes/services — a statement-position
+  # call to an async-looking name with no await/return/.then/.catch on the line.
+  case "$f" in
+    */controllers/*|*/routes/*|*/services/*)
+      if grep -nE '^[[:space:]]*[A-Za-z_][A-Za-z0-9_.]*\(' "$f" \
+        | grep -vE '(await|return|=>|\.then|\.catch|=|const|let|var|if|for|while|res\.|console\.|logger\.|app\.|router\.)' \
+        | grep -qE '(save|fetch|find|create|update|delete|send|publish|notify|query)[A-Za-z]*\('; then
+        warn "$f has a statement-position call to an async-looking function with no await/return — possible floating promise."
+      fi
+      ;;
+  esac
+done <<EOF
+$SRC_FILES
+EOF
+
+if [ "$ERROR_HANDLER_FOUND" -eq 0 ]; then
+  fail "no 4-arg (err,req,res,next) error middleware found — register exactly one, LAST."
+fi
+
+# 5. WARN: engines.node should pin a supported LTS (>=22).
+PKG="$(find "$DIR" -maxdepth 2 -name package.json -not -path '*/node_modules/*' 2>/dev/null | head -1 || true)"
+if [ -n "${PKG:-}" ]; then
+  if grep -Eq '"engines"' "$PKG"; then
+    if ! grep -Eq '"node"\s*:\s*"[^"]*(2[2-9]|[3-9][0-9])' "$PKG"; then
+      warn "$PKG engines.node does not clearly pin a supported LTS (>=22; prefer 24)."
+    fi
+  else
+    warn "$PKG has no engines.node — pin a supported Node LTS (>=24)."
+  fi
+fi
+
+echo
+echo "verify.sh: $HARD_FAIL hard failure(s), $WARN warning(s)."
+[ "$HARD_FAIL" -gt 0 ] && exit 1
+exit 0

package/skills/shared/observability/SKILL.md

@@ -0,0 +1,235 @@
+---
+name: observability
+description: "Use when instrumenting a service from the inside so an incident can be explained from telemetry alone — wiring OpenTelemetry logs, metrics and traces, standing up a Collector, exporting via OTLP, and defining telemetry-driven alerts. NOT outside-in uptime probes, on-call rotation, or who-gets-paged (that is `monitoring`)."
+tags: [observability, opentelemetry, tracing, metrics, structured-logging]
+recommends: [monitoring, error-handling, performance, cost-tracking, docker]
+origin: ai-hub
+---
+
+# Observability
+
+You are wiring the *inside* view of a service: when something breaks at 3am, an engineer must be able to answer "what happened, where, and why" from telemetry alone — without adding a `console.log` and redeploying into the fire. This skill emits a concrete artifact: SDK init code, instrumentation (spans/metrics/structured logs), a Collector config, and alert rules that the instrumentation makes possible. The outside-in half — is it up, who gets paged — is `../monitoring/SKILL.md`.
+
+## The one rule
+
+**Every signal carries the same correlation identity: `trace_id`, `service.name`, `deployment.environment`.** A log line you cannot pivot to its trace, or a spiking metric you cannot pivot to an exemplar span, doubles your mean-time-to-resolution — you are back to grepping. Three signals that don't share keys are three disconnected tools; three that do are one queryable system. Set the resource once at SDK init, inject `trace_id`/`span_id` into every log, and never emit a metric you can't tie back to a service and environment.
+
+## Start from the operator's questions
+
+Before choosing signals, write **two to four questions on-call must answer during the likely incident**. For example: “Are payment retries recovering?”, “Which dependency and failure class drives exhaustion?”, “Can one payment be charged twice?”, “Which customer-visible operations need intervention now?” Then assign the cheapest signal: metrics say **that/how much**, traces show **where/causal path**, logs explain **why for this event**. If a proposed event or label answers none of the questions, do not emit it.
+
+## The three pillars — when each earns its place
+
+Don't emit all three of everything. Each signal answers a different question at a different cost.
+
+| Signal | Answers | Cost | Alert on it? | Main gotcha |
+|---|---|---|---|---|
+| **Logs** | "what exactly happened in this one event" | high per-event, cheap to skip | rarely (noisy) | high-cardinality fields belong in the *body*, not in stream labels |
+| **Metrics** | "what's the rate/aggregate over time" | cheap, pre-aggregated | yes — this is your alert source | cardinality explosion if a label is unbounded |
+| **Traces** | "what was the causal path across hops, and where did time go" | medium; sample it | indirectly (via derived RED metrics) | one giant span = no causality; sample or you pay for noise |
+
+The "fourth pillar," **continuous profiling** (CPU/heap flame graphs over time), is now a first-class OTel signal — reach for it only when traces say "the time is inside *this* function" and you need to know which line.
+
+## Architecture
+
+Instrument once, route anywhere. The app talks OTLP to a Collector; the Collector fans the firehose out to backends.
+
+```text
+  ┌─────────────┐   OTLP/gRPC :4317        ┌───────────────────────────┐
+  │  app + SDK  │   OTLP/HTTP :4318  ───▶   │       OTel Collector       │
+  │ (resource:  │   /v1/traces             │  receivers → processors    │
+  │ service.name│   /v1/metrics            │  → exporters (per signal)  │
+  │  +env+ver)  │   /v1/logs               │  wired in service.pipelines│
+  └─────────────┘                          └───────────────────────────┘
+                                                 │        │        │
+                                            logs │ traces │ metrics│
+                                                 ▼        ▼        ▼
+                                              Loki     Tempo   Mimir/Prom   (+ Grafana to view)
+                                                  └── or a single vendor: Datadog / Honeycomb ──┘
+```
+
+OTLP is the wire format: **gRPC on 4317** (TLS + gzip by default), **HTTP on 4318** with per-signal paths `/v1/traces`, `/v1/metrics`, `/v1/logs`. Always export to a Collector, never straight to the vendor. Why: the Collector gives you one place to **batch** (fewer round-trips), **retry** (survive a backend blip), **redact** PII, and **swap or add a backend without redeploying the app**. App SDKs should be dumb pipes; policy lives in the Collector.
+
+## Instrument in the right order: auto first, manual second
+
+**Never hand-roll a span for something auto-instrumentation already covers** (HTTP servers, DB clients, queues). You will miss edges and waste effort. Turn on zero-code instrumentation, confirm traces flow, *then* add manual spans only where your business logic lives.
+
+```bash
+# Node — zero-code, no app changes. SDK is 2.0+; the register hook is compatible.
+npm i @opentelemetry/api @opentelemetry/auto-instrumentations-node
+OTEL_SERVICE_NAME=checkout-api \
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=prod,service.version=1.4.2 \
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
+node --require '@opentelemetry/auto-instrumentations-node/register' app.js
+```
+
+```bash
+# Python — zero-code via the launcher; it patches known libraries on import.
+pip install opentelemetry-distro opentelemetry-exporter-otlp
+opentelemetry-bootstrap -a install
+OTEL_SERVICE_NAME=checkout-api \
+OTEL_RESOURCE_ATTRIBUTES=deployment.environment=prod,service.version=1.4.2 \
+opentelemetry-instrument python app.py
+```
+
+Then add a manual span only around a meaningful business operation — and give it attributes and a status, or it tells you nothing:
+
+```javascript
+// Bad — a span with no attributes and no status. You learn that "something ran."
+const span = tracer.startSpan('work');
+await chargeCard(order);
+span.end();
+```
+
+```javascript
+// Good — named for the business op, carries the inputs you'd filter by, records outcome.
+const { trace, SpanStatusCode } = require('@opentelemetry/api');
+const tracer = trace.getTracer('checkout');
+await tracer.startActiveSpan('charge_card', async (span) => {
+  span.setAttribute('order.id', order.id);          // searchable dimension
+  span.setAttribute('payment.provider', 'stripe');
+  try {
+    await chargeCard(order);
+    span.setStatus({ code: SpanStatusCode.OK });
+  } catch (err) {
+    span.recordException(err);                        // attaches the stack as an event
+    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+    throw err;
+  } finally {
+    span.end();                                       // a span you never end leaks forever
+  }
+});
+```
+
+Per-stack init (Node SDK 2.0 manual setup, Go SDK, context propagation across HTTP/queue hops, the GenAI span template) lives in `references/instrumentation-recipes.md`.
+
+## Resource and semantic conventions
+
+**Use the standard attribute names; never invent bespoke keys.** The whole correlation story and every prebuilt backend dashboard assume `http.*`, `db.*`, `gen_ai.*`, `service.*`. A homegrown `mycompany.endpoint` attribute is invisible to every tool that expects `http.route`.
+
+- Set on the resource, once: `service.name` (required — unset means telemetry lands as `unknown_service`), `service.version`, `deployment.environment`.
+- Kubernetes attributes (`k8s.*`) reached release candidate (2026-03); DB conventions are on their 2nd RC. Prefer the standard names even while an area is still stabilizing.
+- For LLM calls, **GenAI conventions exited experimental for client spans (early 2026)** — use `gen_ai.*` so the same span feeds your spend view:
+
+```javascript
+// LLM call as a span: model + token attrs. These attributes feed ../cost-tracking/SKILL.md.
+await tracer.startActiveSpan('chat gpt-4o', async (span) => {
+  span.setAttribute('gen_ai.system', 'openai');
+  span.setAttribute('gen_ai.request.model', 'gpt-4o');
+  const res = await openai.chat.completions.create({ /* ... */ });
+  span.setAttribute('gen_ai.usage.input_tokens', res.usage.prompt_tokens);
+  span.setAttribute('gen_ai.usage.output_tokens', res.usage.completion_tokens);
+  span.end();
+});
+```
+
+This skill emits the *signal* (token attributes on a span). Turning those tokens into a dollar figure and a budget is `../cost-tracking/SKILL.md`.
+
+## Structured, correlated logs
+
+**Logs are JSON, carry the trace context, and never carry PII.** Free-text `print` lines cost you twice: you can't query them, and you can't jump from the log to its trace.
+
+```python
+# Bad — unstructured, unsearchable, unlinkable to a trace.
+print("charged user " + email + " amount " + str(amount))
+```
+
+```python
+# Good — structured, leveled, correlated, no PII (id not email).
+import logging, json
+from opentelemetry import trace
+
+def log_charge(order_id, amount):
+    ctx = trace.get_current_span().get_span_context()
+    logging.info(json.dumps({
+        "event": "charge.succeeded",
+        "level": "info",
+        "order_id": order_id,            # an opaque id, not the customer's email
+        "amount_cents": amount,
+        "trace_id": format(ctx.trace_id, "032x"),  # ← the pivot back to the trace
+        "span_id": format(ctx.span_id, "016x"),
+    }))
+```
+
+Level discipline: `error` = a human should look, `warn` = degraded but handled, `info` = business milestones, `debug` = off in prod. If everything is `error`, nothing is.
+
+## Metrics that earn an alert
+
+**RED for request-driven services, USE for finite resources.** Alerts come from metrics — logs and traces are for *investigating* the alert, not firing it.
+
+- **RED** (per service/endpoint): **R**ate (requests/s), **E**rrors (failed/s), **D**uration (latency as a *histogram*, so you can read p50/p95/p99 — never a single average, which hides the tail).
+- **USE** (per resource — CPU, pool, disk): **U**tilization, **S**aturation (queue depth / waiting), **E**rrors.
+
+**The cardinality rule — this is the #1 way an observability stack falls over.** A metric's total series count is the product of its label cardinalities. Put an unbounded value on a label and you create a near-infinite series count; the time-series database (Prometheus/Mimir) is most often restarted because of exactly this. Loki indexes *labels only, not log contents* — so the same rule binds its stream labels.
+
+```text
+# Bad — user_id is unbounded; 5M users = 5M series per metric. OOMs the TSDB.
+http_requests_total{route="/checkout", user_id="u_8f3a...", status="200"}
+```
+
+```text
+# Good — only bounded, low-cardinality dimensions on the metric.
+http_requests_total{route="/checkout", method="POST", status="200"}
+# Need to slice by user? That's a trace attribute or a log field, never a metric label.
+```
+
+## The Collector config
+
+A minimal valid pipeline. An exporter is **inert until it appears in a pipeline** — defining one under `exporters:` does nothing on its own.
+
+```yaml
+# otel-collector.yaml — Collector v0.153.0 shape
+receivers:
+  otlp:
+    protocols:
+      grpc: { endpoint: 0.0.0.0:4317 }
+      http: { endpoint: 0.0.0.0:4318 }
+
+processors:
+  memory_limiter:                 # first line of defense: shed load before OOM
+    check_interval: 1s
+    limit_percentage: 80
+  batch: {}                       # batch before export — fewer, bigger round-trips
+  redaction:                      # strip PII before it leaves your network
+    allow_all_keys: true
+    blocked_values: ["[0-9]{13,16}", "\\b[\\w.]+@[\\w.]+\\b"]  # PANs, emails
+
+exporters:
+  otlphttp/traces:  { endpoint: http://tempo:4318 }
+  otlphttp/logs:    { endpoint: http://loki:3100/otlp }
+  otlphttp/metrics: { endpoint: http://mimir:9009/otlp }
+
+service:
+  pipelines:
+    traces:  { receivers: [otlp], processors: [memory_limiter, redaction, batch], exporters: [otlphttp/traces] }
+    logs:    { receivers: [otlp], processors: [memory_limiter, redaction, batch], exporters: [otlphttp/logs] }
+    metrics: { receivers: [otlp], processors: [memory_limiter, batch],            exporters: [otlphttp/metrics] }
+```
+
+Tail sampling, gateway-vs-agent topology, multi-backend fan-out (LGTM **and** a vendor in parallel), and `resourcedetection` live in `references/collector-config.md`. Shipping the Collector as a container or in CI is `../docker/SKILL.md`.
+
+## Alerts the telemetry now enables
+
+The instrumentation above makes these *possible* — defining them is your job; routing the page to a human is `../monitoring/SKILL.md`.
+
+- **SLO burn-rate (multi-window)** — alert when you're spending the error budget too fast, using a fast window (e.g. 5m) AND a slow window (e.g. 1h) so a brief blip doesn't page but a sustained burn does. This fires on *impact*, not on traffic.
+- **Latency SLO** — p99 of the duration histogram over budget for a sustained window.
+- **Saturation** — the USE "S": queue depth / pool waiters climbing, the leading indicator before errors appear.
+
+## Anti-patterns
+
+| Anti-pattern | Why it bites | Do instead |
+|---|---|---|
+| Unbounded label (`user_id`, `request_id`, `email`) on a metric or Loki stream | Cardinality explosion → TSDB OOM/restart, cost blowup | Keep labels low-cardinality; put the high-cardinality field on a span attribute or log body |
+| Logging PII / secrets (email, card, token) | Compliance breach + the leak is now in every log backend | Log opaque ids; redact in the Collector before export |
+| 100% trace sampling in prod, no policy | Pay to store noise; backend throttles and drops the traces you needed | Head/tail sampling — keep all errors + slow traces, sample the rest |
+| App exports straight to the vendor, no Collector | Can't batch, retry, redact, or swap backends without a redeploy | Always route through a Collector |
+| Instrument everything before deciding the question | Noise with no signal; nobody opens the dashboard | Start from "what would I ask during an incident," instrument that path |
+| Alert on a raw error *count* | Fires on traffic spikes, silent during a low-traffic outage | Alert on error *rate* / SLO burn |
+| One giant span per request (or a thousand contentless ones) | No causality, or context with no detail — both useless | Span per meaningful operation, each with attributes + status |
+
+## Verify
+
+Run `scripts/verify.sh` against the directory holding your Collector config + SDK init. It checks the config is valid, that every defined exporter is actually wired into a pipeline (the classic "defined but unused" footgun), that `service.name` is set, and warns on high-cardinality metric labels. It is read-only and exits 0 when there's nothing to check.
+
+Then prove the wire with one safe induced failure in a test/staging path. Confirm the expected error-rate metric changes, the trace records the failing operation and error status, and the structured log carries the same `trace_id` without PII. Query the backend/Collector output; “the instrumentation code ran” is not evidence that usable telemetry arrived. Live paging and escalation proof remains `../monitoring/SKILL.md`.

package/skills/shared/observability/evals/README.md

@@ -0,0 +1,3 @@
+# Evals for observability
+
+These cases are eyeballed, not automated. Read `cases.yaml` and check three things by hand. First, every `should_trigger` prompt should make this skill the obvious pick — pay special attention to the non-obvious ones (the "can't see where the time goes across services" latency-symptom line, and the "LLM calls as spans with token counts" line, since those are where a thinner trigger set would miss). Second, every `should_not_trigger` prompt must route cleanly to its named sibling — the sharp boundary is `monitoring` (it emits no signals, it decides the page) and `cost-tracking` (it prices the gen_ai.* tokens this skill emits); if either of those feels like it could legitimately land here, tighten the description. Third, run the `capability` scenario against a generated answer and confirm each `must_include` item is actually present, not hand-waved — especially that every Collector exporter is wired into a pipeline, that the manual span sets status and attributes, that logs carry trace_id/span_id with no PII, and that no metric uses a high-cardinality label. Note that `scripts/verify.sh` checks the emitted artifact (Collector config + SDK init), not the prose: a great write-up that emits no wired-up config has not finished the job.

package/skills/shared/observability/evals/cases.yaml

@@ -0,0 +1,51 @@
+skill: observability
+
+should_trigger:
+  - prompt: "Add OpenTelemetry traces, metrics and logs to our API."
+    why: Core trigger — wire the three signals through OTel. The skill's central job.
+  - prompt: "Requests are slow and we can't see where the time goes across our services."
+    why: Non-obvious wording (no mention of OTel/tracing). The symptom maps to distributed tracing — spans across hops to attribute latency.
+  - prompt: "Replace our print/console.log statements with structured logs correlated to traces."
+    why: The structured-logging pillar — JSON logs injected with trace_id/span_id so you can pivot log → trace.
+  - prompt: "Instrument our LLM calls so each request is a span with token counts."
+    why: Non-obvious. GenAI semantic conventions (gen_ai.* token attrs) on spans; the bridge that feeds cost-tracking, but the instrumentation itself lives here.
+  - prompt: "Stand up an OTel Collector and export our telemetry to Grafana."
+    why: Collector config + OTLP export + per-signal pipelines — the architecture core.
+  - prompt: "Define RED metrics and a burn-rate alert from the telemetry we emit."
+    why: Telemetry-driven alerting that the instrumentation makes possible (rate/errors/duration + multi-window SLO burn).
+  - prompt: "Instrumenta el servei amb OpenTelemetry i exporta a Grafana."
+    why: Catalan trigger for the core instrument-and-export flow.
+
+should_not_trigger:
+  - prompt: "Alert me when the site goes down and set up an on-call rotation."
+    route_to: monitoring
+    why: Outside-in detection + who-gets-paged. Observability emits the signals; monitoring decides the page and the human.
+  - prompt: "Design our retry strategy, circuit breaker, and error taxonomy."
+    route_to: error-handling
+    why: How the code handles failure, not how failure is recorded as telemetry.
+  - prompt: "This endpoint takes 3 seconds — make it fast."
+    route_to: performance
+    why: Fixing slowness. Observability makes slowness visible/attributable; performance does the actual optimization.
+  - prompt: "Track user signups and our activation funnel conversion."
+    route_to: analytics
+    why: Product/business events for growth, not system/operational signals.
+  - prompt: "Build a dashboard of our monthly LLM spend by model and team."
+    route_to: cost-tracking
+    why: Turning token usage into dollars and budgets. Observability emits the gen_ai.* token span; cost-tracking prices it.
+
+capability:
+  - scenario: >
+      I have a Node service that uses console.log for everything and has no metrics.
+      Make it observable: I want to explain an incident from telemetry alone.
+    must_include:
+      - SDK init that sets resource attributes service.name, service.version, and deployment.environment
+      - auto-instrumentation enabled (zero-code register hook or getNodeAutoInstrumentations) before any manual spans
+      - one manual span around a business operation that carries attributes AND sets span status (OK/ERROR) and records exceptions
+      - structured JSON logs that inject trace_id and span_id (not free-text print/console.log)
+      - a RED metric set — request rate, error rate, and latency as a histogram (not a single average)
+      - a Collector config where every exporter under exporters is referenced in a service.pipelines entry
+      - a multi-window SLO burn-rate alert defined from the emitted metrics
+      - respects the cardinality rule (no user_id/request_id/email as a metric label) and logs no PII (opaque ids, redaction in the Collector)
+      - exports through a Collector via OTLP, not straight to a vendor
+      - starts from two to four concrete on-call questions and maps metrics to that/how much, traces to where, and logs to why
+      - induces one safe failure and proves the correlated metric, error trace, and structured log actually arrive without PII

package/skills/shared/observability/references/collector-config.md

@@ -0,0 +1,98 @@
+# Collector config — fuller recipes
+
+Branch-specific Collector depth offloaded from `SKILL.md`. Shapes target Collector **v0.153.0**. An exporter or processor is inert until it is named in a `service.pipelines` entry.
+
+## Gateway vs agent topology
+
+Two deployment shapes; most production setups run both as a tiered pipeline.
+
+- **Agent** — one Collector per host/pod (sidecar or DaemonSet), close to the app. Cheap network hop, does fast local work: `memory_limiter`, `batch`, `resourcedetection`. Forwards via OTLP to the gateway.
+- **Gateway** — a horizontally-scaled standalone tier. Does the expensive, stateful work that needs a *whole-trace* view: `tail_sampling`, cross-service `redaction`, per-tenant routing, backend fan-out. Scale it independently of your apps.
+
+Rule: do per-span/per-host work in the agent, do whole-trace and policy work in the gateway. Tail sampling in an agent is wrong — an agent never sees a full trace.
+
+```yaml
+# agent (DaemonSet) → forwards everything to the gateway
+exporters:
+  otlp/gateway: { endpoint: otel-gateway:4317, tls: { insecure: false } }
+service:
+  pipelines:
+    traces:  { receivers: [otlp], processors: [memory_limiter, resourcedetection, batch], exporters: [otlp/gateway] }
+    metrics: { receivers: [otlp], processors: [memory_limiter, batch],                     exporters: [otlp/gateway] }
+    logs:    { receivers: [otlp], processors: [memory_limiter, batch],                     exporters: [otlp/gateway] }
+```
+
+## Tail sampling (gateway only)
+
+Head sampling decides at span start, before you know if the request was interesting. **Tail sampling** buffers the whole trace and decides at the end — so you keep every error and every slow trace, and sample the boring fast ones. It must run on the gateway, after a `groupbytrace`-style stage so all spans of a trace land on the same Collector.
+
+```yaml
+processors:
+  tail_sampling:
+    decision_wait: 10s            # how long to buffer a trace before deciding
+    num_traces: 100000
+    policies:
+      - name: keep-errors
+        type: status_code
+        status_code: { status_codes: [ERROR] }     # never drop a failed trace
+      - name: keep-slow
+        type: latency
+        latency: { threshold_ms: 1000 }             # never drop a slow trace
+      - name: sample-the-rest
+        type: probabilistic
+        probabilistic: { sampling_percentage: 5 }   # 5% of the boring ones
+```
+
+## Redaction / attributes for PII
+
+Strip sensitive data before it leaves your network. `redaction` masks values by regex; `attributes` deletes or hashes specific keys.
+
+```yaml
+processors:
+  redaction:
+    allow_all_keys: true
+    blocked_values:
+      - "\\b[0-9]{13,16}\\b"                # card PANs
+      - "\\b[\\w.+-]+@[\\w-]+\\.[\\w.-]+\\b" # emails
+    summary: debug                          # report what it masked, at debug
+  attributes/pii:
+    actions:
+      - { key: user.email, action: delete }
+      - { key: user.id,    action: hash }   # keep joinable, not readable
+```
+
+Wire both into the pipeline's `processors` list before the exporter, or they do nothing.
+
+## Multi-backend fan-out (LGTM + a vendor in parallel)
+
+A pipeline's `exporters` list is a fan-out — list two and every item goes to both. Useful for migrating off a vendor, or keeping a cheap self-hosted copy beside a managed one.
+
+```yaml
+exporters:
+  otlphttp/tempo:    { endpoint: http://tempo:4318 }
+  otlphttp/honeycomb:
+    endpoint: https://api.honeycomb.io
+    headers: { "x-honeycomb-team": "${env:HONEYCOMB_API_KEY}" }
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [otlphttp/tempo, otlphttp/honeycomb]   # both, in parallel
+```
+
+LGTM split for reference: **Loki** logs, **Tempo** traces, **Mimir/Prometheus** metrics, **Grafana** to view.
+
+## resourcedetection + env-var overrides
+
+`resourcedetection` auto-fills `host.*`, `cloud.*`, `k8s.*`, `container.*` resource attributes so you don't hand-set them per deploy. Combine with env-var interpolation (`${env:VAR}`) so the same config file works across environments — never bake secrets or per-env endpoints into the file.
+
+```yaml
+processors:
+  resourcedetection:
+    detectors: [env, system, gcp, eks]    # order = precedence
+    timeout: 2s
+exporters:
+  otlphttp/metrics:
+    endpoint: "${env:METRICS_ENDPOINT}"   # differs per environment, injected at runtime
+```

package/skills/shared/observability/references/instrumentation-recipes.md

@@ -0,0 +1,115 @@
+# Instrumentation recipes — per stack
+
+Stack-specific snippets offloaded from `SKILL.md`. Reach for these only after zero-code auto-instrumentation is flowing; manual code is for business logic the auto layer can't see. APIs target **OpenTelemetry JS SDK 2.0+** and current Python/Go SDKs.
+
+## Node / JS — manual SDK init
+
+Use this only if you need programmatic control beyond the `--require` register hook (custom processors, multiple exporters, sampler tuning). Otherwise prefer the zero-code path in `SKILL.md`.
+
+```javascript
+// instrument.js — load FIRST, before any other require (node -r ./instrument.js app.js)
+const { NodeSDK } = require('@opentelemetry/sdk-node');
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');
+const { resourceFromAttributes } = require('@opentelemetry/resources');
+
+const sdk = new NodeSDK({
+  resource: resourceFromAttributes({
+    'service.name': 'checkout-api',
+    'service.version': '1.4.2',
+    'deployment.environment': 'prod',
+  }),
+  traceExporter: new OTLPTraceExporter({ url: 'http://localhost:4318/v1/traces' }),
+  instrumentations: [getNodeAutoInstrumentations()],
+});
+sdk.start();
+process.on('SIGTERM', () => sdk.shutdown());  // flush buffered spans on shutdown
+```
+
+## Python — launcher + manual span
+
+`opentelemetry-instrument` patches known libraries on import. Add manual spans only around your own logic.
+
+```python
+from opentelemetry import trace
+tracer = trace.get_tracer("checkout")
+
+def settle_order(order_id):
+    with tracer.start_as_current_span("settle_order") as span:
+        span.set_attribute("order.id", order_id)
+        try:
+            do_settlement(order_id)
+            span.set_status(trace.StatusCode.OK)
+        except Exception as err:
+            span.record_exception(err)
+            span.set_status(trace.StatusCode.ERROR, str(err))
+            raise
+```
+
+## Go — SDK init
+
+Go has no zero-code agent; init the provider in `main` and use `otelhttp`/`otelgrpc` middleware for transport-level spans.
+
+```go
+import (
+    "go.opentelemetry.io/otel"
+    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+    "go.opentelemetry.io/otel/sdk/resource"
+    sdktrace "go.opentelemetry.io/otel/sdk/trace"
+    semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+)
+
+func initTracer(ctx context.Context) (*sdktrace.TracerProvider, error) {
+    exp, err := otlptracehttp.New(ctx) // reads OTEL_EXPORTER_OTLP_ENDPOINT
+    if err != nil { return nil, err }
+    res, _ := resource.New(ctx, resource.WithAttributes(
+        semconv.ServiceName("checkout-api"),
+        semconv.ServiceVersion("1.4.2"),
+        semconv.DeploymentEnvironment("prod"),
+    ))
+    tp := sdktrace.NewTracerProvider(
+        sdktrace.WithBatcher(exp),
+        sdktrace.WithResource(res),
+    )
+    otel.SetTracerProvider(tp)
+    return tp, nil // defer tp.Shutdown(ctx) in main
+}
+```
+
+## Context propagation across boundaries
+
+A trace only spans services if the `traceparent` (W3C Trace Context) is carried across each hop. Auto-instrumentation handles HTTP for you; you must do it by hand across **message queues**, which have no built-in carrier.
+
+```javascript
+// Producer — inject the active context into message headers.
+const { propagation, context } = require('@opentelemetry/api');
+const headers = {};
+propagation.inject(context.active(), headers);
+await queue.send({ body, headers });          // ship headers alongside the payload
+
+// Consumer — extract it, then run work inside the parent context.
+const parentCtx = propagation.extract(context.active(), msg.headers);
+context.with(parentCtx, () => {
+  tracer.startActiveSpan('process_message', (span) => { /* ... */ span.end(); });
+});
+```
+
+Without this, the consumer starts a brand-new root trace and you lose the causal link from producer to consumer.
+
+## GenAI / LLM span template
+
+Use `gen_ai.*` semantic conventions (client spans exited experimental, early 2026). Span name convention is `{operation} {model}`. These token attributes are the bridge that `../cost-tracking/SKILL.md` consumes to compute spend.
+
+```python
+with tracer.start_as_current_span("chat gpt-4o") as span:
+    span.set_attribute("gen_ai.system", "openai")
+    span.set_attribute("gen_ai.operation.name", "chat")
+    span.set_attribute("gen_ai.request.model", "gpt-4o")
+    span.set_attribute("gen_ai.request.temperature", 0.2)
+    resp = client.chat.completions.create(model="gpt-4o", messages=msgs, temperature=0.2)
+    span.set_attribute("gen_ai.response.model", resp.model)
+    span.set_attribute("gen_ai.usage.input_tokens", resp.usage.prompt_tokens)
+    span.set_attribute("gen_ai.usage.output_tokens", resp.usage.completion_tokens)
+```
+
+Do not put the prompt or completion *text* on the span by default — it is high-cardinality and frequently PII. Capture it only behind an explicit, redacted, opt-in content-capture flag.