@harness-engineering/cli 1.23.1 → 1.24.0

package/dist/agents/skills/codex/otel-custom-instrumentation/SKILL.md

@@ -0,0 +1,166 @@
+# OpenTelemetry Custom Instrumentation
+
+> Add custom spans, attributes, and events to business-critical code paths beyond auto-instrumentation
+
+## When to Use
+
+- Auto-instrumentation covers HTTP and database but not your business logic
+- Need to trace specific operations: payment processing, recommendation generation, report building
+- Adding business context (order ID, user tier, feature flag) to traces
+- Wrapping third-party SDK calls that lack OpenTelemetry support
+
+## Instructions
+
+1. Import the API only (`@opentelemetry/api`), not the SDK, in application code. The SDK is initialized in `instrumentation.ts`.
+2. Get a tracer once per module: `const tracer = trace.getTracer('module-name')`.
+3. Use `tracer.startActiveSpan` for most cases — it sets the span as active context automatically.
+4. Add attributes that help debugging: IDs, counts, flags, operation types.
+5. Record exceptions with `span.recordException(error)` before setting error status.
+6. For utility functions called from many places, create a reusable tracing wrapper.
+
+```typescript
+// decorators/traced.ts — reusable tracing wrapper
+import { trace, SpanKind, SpanStatusCode, Span } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('app');
+
+export function traced<T extends (...args: any[]) => Promise<any>>(
+  name: string,
+  fn: T,
+  options?: {
+    kind?: SpanKind;
+    attributes?: Record<string, string | number | boolean>;
+  }
+): T {
+  return (async (...args: any[]) => {
+    return tracer.startActiveSpan(
+      name,
+      {
+        kind: options?.kind ?? SpanKind.INTERNAL,
+        attributes: options?.attributes,
+      },
+      async (span: Span) => {
+        try {
+          const result = await fn(...args);
+          span.setStatus({ code: SpanStatusCode.OK });
+          return result;
+        } catch (error) {
+          span.recordException(error as Error);
+          span.setStatus({ code: SpanStatusCode.ERROR, message: (error as Error).message });
+          throw error;
+        } finally {
+          span.end();
+        }
+      }
+    );
+  }) as T;
+}
+
+// Usage
+const processPayment = traced('payment.process', async (orderId: string, amount: number) => {
+  const span = trace.getActiveSpan();
+  span?.setAttribute('payment.order_id', orderId);
+  span?.setAttribute('payment.amount', amount);
+
+  const result = await paymentGateway.charge(orderId, amount);
+
+  span?.setAttribute('payment.transaction_id', result.transactionId);
+  span?.addEvent('payment.charged', { 'payment.method': result.method });
+
+  return result;
+});
+```
+
+```typescript
+// services/recommendation-service.ts — inline custom spans
+import { trace, SpanStatusCode } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('recommendation-service');
+
+export async function getRecommendations(userId: string, limit: number): Promise<Product[]> {
+  return tracer.startActiveSpan('recommendations.generate', async (span) => {
+    span.setAttribute('user.id', userId);
+    span.setAttribute('recommendations.limit', limit);
+
+    // Step 1: Get user profile
+    const profile = await tracer.startActiveSpan(
+      'recommendations.loadProfile',
+      async (profileSpan) => {
+        const result = await userService.getProfile(userId);
+        profileSpan.setAttribute('user.segment', result.segment);
+        profileSpan.end();
+        return result;
+      }
+    );
+
+    // Step 2: Score candidates
+    const scored = await tracer.startActiveSpan('recommendations.score', async (scoreSpan) => {
+      const candidates = await productService.getCandidates(profile.segment);
+      scoreSpan.setAttribute('candidates.count', candidates.length);
+      const results = await mlService.score(candidates, profile);
+      scoreSpan.addEvent('scoring.complete', { top_score: results[0]?.score ?? 0 });
+      scoreSpan.end();
+      return results;
+    });
+
+    span.setAttribute('recommendations.returned', Math.min(scored.length, limit));
+    span.end();
+    return scored.slice(0, limit);
+  });
+}
+```
+
+## Details
+
+**API vs SDK separation:** Application code imports `@opentelemetry/api` (zero-cost no-op if no SDK is registered). The SDK is configured once in `instrumentation.ts`. This means your business code has no dependency on the SDK — if OpenTelemetry is not initialized, spans are silently discarded.
+
+**Getting the active span (without creating a new one):**
+
+```typescript
+import { trace } from '@opentelemetry/api';
+
+function enrichSpan(orderId: string) {
+  const span = trace.getActiveSpan();
+  if (span) {
+    span.setAttribute('order.id', orderId);
+  }
+}
+```
+
+**Span links:** Connect related traces that are not parent-child (e.g., a batch job processing items from different requests):
+
+```typescript
+const span = tracer.startSpan('batch.process', {
+  links: items.map((item) => ({
+    context: item.spanContext,
+    attributes: { 'batch.item_id': item.id },
+  })),
+});
+```
+
+**What to instrument (rule of thumb):**
+
+- Business operations (order creation, payment, fulfillment) — always
+- Service boundaries (incoming requests, outgoing calls) — auto-instrumentation handles this
+- Significant decision points (feature flag evaluation, A/B test assignment) — as events
+- Every function call — never (too much overhead, noisy traces)
+
+## Source
+
+https://opentelemetry.io/docs/languages/js/instrumentation/
+
+## Process
+
+1. Read the instructions and examples in this document.
+2. Apply the patterns to your implementation, adapting to your specific context.
+3. Verify your implementation against the details and edge cases listed above.
+
+## Harness Integration
+
+- **Type:** knowledge — this skill is a reference document, not a procedural workflow.
+- **No tools or state** — consumed as context by other skills and agents.
+
+## Success Criteria
+
+- The patterns described in this document are applied correctly in the implementation.
+- Edge cases and anti-patterns listed in this document are avoided.

package/dist/agents/skills/codex/otel-custom-instrumentation/skill.yaml

@@ -0,0 +1,39 @@
+name: otel-custom-instrumentation
+version: '1.0.0'
+description: Add custom spans and attributes to business-critical code paths
+cognitive_mode: advisory-guide
+type: knowledge
+tier: 3
+triggers:
+  - manual
+platforms:
+  - claude-code
+  - gemini-cli
+  - cursor
+  - codex
+tools: []
+paths:
+  - '**/instrumentation/**'
+  - '**/services/**'
+  - '**/telemetry/**'
+related_skills:
+  - otel-tracing-pattern
+  - otel-context-propagation
+  - otel-sdk-setup
+stack_signals:
+  - opentelemetry
+  - nodejs
+  - typescript
+keywords:
+  - custom-spans
+  - manual-instrumentation
+  - span-attributes
+  - business-tracing
+  - decorator
+metadata:
+  author: community
+  upstream: 'opentelemetry/docs/instrumentation/js/manual'
+state:
+  persistent: false
+  files: []
+depends_on: []

package/dist/agents/skills/codex/otel-error-tracking/SKILL.md

@@ -0,0 +1,163 @@
+# OpenTelemetry Error Tracking
+
+> Track and correlate errors across services with span exceptions, status codes, and error events
+
+## When to Use
+
+- Tracking errors across distributed services and correlating them to specific traces
+- Replacing or supplementing Sentry/Bugsnag with OpenTelemetry-native error tracking
+- Understanding error propagation — which service caused a cascading failure
+- Building error rate SLIs from trace data
+
+## Instructions
+
+1. Record exceptions with `span.recordException(error)` — this adds an exception event with stack trace, type, and message.
+2. Always set span status to ERROR after recording an exception: `span.setStatus({ code: SpanStatusCode.ERROR, message })`.
+3. Add error context as span attributes: error code, affected resource, user context.
+4. Use span events for non-fatal errors (warnings, handled exceptions, fallback activations).
+5. Create error metrics from trace data: error rate by service, error type distribution, error count by endpoint.
+6. Build error-aware sampling: always keep traces that contain errors.
+
+```typescript
+// Comprehensive error recording
+import { trace, SpanStatusCode } from '@opentelemetry/api';
+
+const tracer = trace.getTracer('order-service');
+
+export async function processOrder(orderId: string): Promise<Order> {
+  return tracer.startActiveSpan('order.process', async (span) => {
+    span.setAttribute('order.id', orderId);
+
+    try {
+      const order = await db.orders.findById(orderId);
+      if (!order) {
+        // Business error — record as exception with context
+        const error = new NotFoundError(`Order ${orderId} not found`);
+        span.recordException(error);
+        span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+        span.setAttribute('error.type', 'NotFoundError');
+        span.setAttribute('error.category', 'business');
+        throw error;
+      }
+
+      try {
+        await paymentService.charge(order);
+      } catch (paymentError) {
+        // Handled error — using fallback
+        span.addEvent('payment.fallback', {
+          'error.message': (paymentError as Error).message,
+          'fallback.strategy': 'retry-queue',
+        });
+        await retryQueue.enqueue(order);
+        span.setAttribute('order.payment_status', 'queued');
+        return { ...order, paymentStatus: 'queued' };
+      }
+
+      span.setStatus({ code: SpanStatusCode.OK });
+      return order;
+    } catch (error) {
+      if (!span.isRecording()) return; // Span already ended
+      span.recordException(error as Error);
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: (error as Error).message,
+      });
+      span.setAttribute('error.type', (error as Error).constructor.name);
+      throw error;
+    } finally {
+      span.end();
+    }
+  });
+}
+```
+
+```typescript
+// Global error handler that enriches the active span
+export function handleUnexpectedError(error: Error, context?: Record<string, string>): void {
+  const span = trace.getActiveSpan();
+  if (span) {
+    span.recordException(error);
+    span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+    span.setAttribute('error.unexpected', true);
+    if (context) {
+      Object.entries(context).forEach(([key, value]) => {
+        span.setAttribute(`error.context.${key}`, value);
+      });
+    }
+  }
+  // Also log for non-trace-aware systems
+  console.error('Unexpected error', { error, context, traceId: span?.spanContext().traceId });
+}
+```
+
+```typescript
+// Express error middleware with tracing
+app.use((err: Error, req: Request, res: Response, next: NextFunction) => {
+  const span = trace.getActiveSpan();
+  if (span) {
+    span.recordException(err);
+    span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
+    span.setAttribute('http.status_code', err instanceof HttpError ? err.status : 500);
+  }
+
+  const status = err instanceof HttpError ? err.status : 500;
+  res.status(status).json({
+    error: err.message,
+    traceId: span?.spanContext().traceId, // Include in response for client-side correlation
+  });
+});
+```
+
+## Details
+
+**recordException vs setStatus:** Both are needed. `recordException` adds an event with the stack trace and error details. `setStatus(ERROR)` marks the span as failed in the trace viewer. Without `setStatus`, the span appears as successful even though an exception was recorded.
+
+**Exception event attributes** (set automatically by `recordException`):
+
+- `exception.type` — error class name
+- `exception.message` — error message
+- `exception.stacktrace` — full stack trace
+
+**Error categorization:** Add custom attributes to distinguish error types in dashboards:
+
+```typescript
+span.setAttribute('error.category', 'business'); // vs 'infrastructure', 'validation', 'external'
+span.setAttribute('error.retryable', true);
+span.setAttribute('error.severity', 'critical'); // vs 'warning', 'info'
+```
+
+**Trace ID in API responses:** Include the trace ID in error responses so users or support teams can reference it:
+
+```json
+{
+  "error": "Payment failed",
+  "traceId": "abc123def456",
+  "message": "Please contact support with this trace ID"
+}
+```
+
+**Error-based alerting from traces:** Most observability backends support alerts on:
+
+- Error rate exceeding threshold (>1% of traces have ERROR status)
+- Specific error types appearing (new `NullPointerException` in production)
+- Error count spike detection (anomaly detection on error volume)
+
+## Source
+
+https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/
+
+## Process
+
+1. Read the instructions and examples in this document.
+2. Apply the patterns to your implementation, adapting to your specific context.
+3. Verify your implementation against the details and edge cases listed above.
+
+## Harness Integration
+
+- **Type:** knowledge — this skill is a reference document, not a procedural workflow.
+- **No tools or state** — consumed as context by other skills and agents.
+
+## Success Criteria
+
+- The patterns described in this document are applied correctly in the implementation.
+- Edge cases and anti-patterns listed in this document are avoided.

package/dist/agents/skills/codex/otel-error-tracking/skill.yaml

@@ -0,0 +1,39 @@
+name: otel-error-tracking
+version: '1.0.0'
+description: Track and correlate errors across services with OpenTelemetry span exceptions and status
+cognitive_mode: advisory-guide
+type: knowledge
+tier: 3
+triggers:
+  - manual
+platforms:
+  - claude-code
+  - gemini-cli
+  - cursor
+  - codex
+tools: []
+paths:
+  - '**/instrumentation/**'
+  - '**/telemetry/**'
+  - '**/error/**'
+related_skills:
+  - otel-tracing-pattern
+  - otel-logging-pattern
+  - otel-custom-instrumentation
+stack_signals:
+  - opentelemetry
+  - nodejs
+  - typescript
+keywords:
+  - error-tracking
+  - exceptions
+  - span-status
+  - error-correlation
+  - sentry-migration
+metadata:
+  author: community
+  upstream: 'opentelemetry/docs/error-handling'
+state:
+  persistent: false
+  files: []
+depends_on: []

package/dist/agents/skills/codex/otel-exporter-config/SKILL.md

@@ -0,0 +1,168 @@
+# OpenTelemetry Exporter Configuration
+
+> Configure OTLP exporters to send traces, metrics, and logs to observability backends and collectors
+
+## When to Use
+
+- Connecting your OpenTelemetry SDK to a backend (Jaeger, Grafana Tempo, Datadog, Honeycomb)
+- Setting up an OpenTelemetry Collector as a telemetry pipeline
+- Choosing between direct export and collector-mediated export
+- Configuring export batching, compression, and retry behavior
+
+## Instructions
+
+1. Prefer OTLP (OpenTelemetry Protocol) exporters — they work with all major backends.
+2. Choose HTTP (`@opentelemetry/exporter-trace-otlp-http`) or gRPC (`@opentelemetry/exporter-trace-otlp-grpc`). HTTP is simpler; gRPC has better performance for high-volume telemetry.
+3. In production, export to an OpenTelemetry Collector, not directly to the backend. The Collector handles batching, retries, and routing.
+4. Configure exporters via environment variables for deployment flexibility.
+5. Set appropriate batch sizes and export intervals to balance latency and resource usage.
+
+```typescript
+// Direct export to backend (simple setup)
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
+import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base';
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+
+// Traces
+const traceExporter = new OTLPTraceExporter({
+  url: process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || 'http://localhost:4318/v1/traces',
+  headers: {
+    Authorization: `Bearer ${process.env.OTEL_EXPORTER_API_KEY}`,
+  },
+  compression: 'gzip',
+});
+
+const spanProcessor = new BatchSpanProcessor(traceExporter, {
+  maxQueueSize: 2048,
+  maxExportBatchSize: 512,
+  scheduledDelayMillis: 5000,
+  exportTimeoutMillis: 30000,
+});
+
+// Metrics
+const metricReader = new PeriodicExportingMetricReader({
+  exporter: new OTLPMetricExporter({
+    url: process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT || 'http://localhost:4318/v1/metrics',
+  }),
+  exportIntervalMillis: 15000,
+});
+```
+
+```yaml
+# otel-collector-config.yaml — Collector as telemetry pipeline
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+    timeout: 5s
+    send_batch_size: 1024
+  memory_limiter:
+    check_interval: 1s
+    limit_mib: 512
+  attributes:
+    actions:
+      - key: environment
+        value: production
+        action: upsert
+
+exporters:
+  otlphttp/grafana:
+    endpoint: https://tempo.grafana.net
+    headers:
+      Authorization: 'Basic ${GRAFANA_API_KEY}'
+  prometheus:
+    endpoint: 0.0.0.0:8889
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch, attributes]
+      exporters: [otlphttp/grafana]
+    metrics:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [prometheus]
+```
+
+## Details
+
+**Direct export vs Collector:**
+
+- **Direct export:** Simpler setup, fewer moving parts. Good for small deployments and development.
+- **Collector:** Decouples app from backend, handles retries/batching/sampling, can route to multiple backends, filters sensitive data. Recommended for production.
+
+**Environment variables (standard):**
+
+```bash
+OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4318
+OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
+OTEL_EXPORTER_OTLP_HEADERS=Authorization=Bearer token
+OTEL_EXPORTER_OTLP_COMPRESSION=gzip
+OTEL_EXPORTER_OTLP_TIMEOUT=10000
+```
+
+**Backend-specific configurations:**
+
+```typescript
+// Honeycomb
+new OTLPTraceExporter({
+  url: 'https://api.honeycomb.io/v1/traces',
+  headers: { 'x-honeycomb-team': process.env.HONEYCOMB_API_KEY },
+});
+
+// Grafana Cloud
+new OTLPTraceExporter({
+  url: 'https://tempo-us-central1.grafana.net/tempo',
+  headers: { Authorization: `Basic ${btoa(`${instanceId}:${apiKey}`)}` },
+});
+
+// Datadog (via OTLP)
+new OTLPTraceExporter({
+  url: 'http://datadog-agent:4318/v1/traces',
+});
+```
+
+**Console exporter for development:**
+
+```typescript
+import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-base';
+
+const sdk = new NodeSDK({
+  traceExporter: new ConsoleSpanExporter(), // Prints traces to stdout
+});
+```
+
+**Batch processor tuning:**
+
+- `maxQueueSize: 2048` — max spans held in memory before dropping
+- `maxExportBatchSize: 512` — spans sent per export call
+- `scheduledDelayMillis: 5000` — export interval
+- Increase batch size for high-throughput services; decrease delay for lower latency
+
+## Source
+
+https://opentelemetry.io/docs/languages/js/exporters/
+
+## Process
+
+1. Read the instructions and examples in this document.
+2. Apply the patterns to your implementation, adapting to your specific context.
+3. Verify your implementation against the details and edge cases listed above.
+
+## Harness Integration
+
+- **Type:** knowledge — this skill is a reference document, not a procedural workflow.
+- **No tools or state** — consumed as context by other skills and agents.
+
+## Success Criteria
+
+- The patterns described in this document are applied correctly in the implementation.
+- Edge cases and anti-patterns listed in this document are avoided.

package/dist/agents/skills/codex/otel-exporter-config/skill.yaml

@@ -0,0 +1,39 @@
+name: otel-exporter-config
+version: '1.0.0'
+description: Configure OTLP exporters for traces, metrics, and logs to observability backends
+cognitive_mode: advisory-guide
+type: knowledge
+tier: 3
+triggers:
+  - manual
+platforms:
+  - claude-code
+  - gemini-cli
+  - cursor
+  - codex
+tools: []
+paths:
+  - '**/instrumentation/**'
+  - '**/telemetry/**'
+related_skills:
+  - otel-sdk-setup
+  - otel-sampling-strategies
+  - otel-tracing-pattern
+stack_signals:
+  - opentelemetry
+  - nodejs
+  - typescript
+keywords:
+  - OTLP
+  - exporter
+  - collector
+  - Jaeger
+  - Grafana
+  - Datadog
+metadata:
+  author: community
+  upstream: 'opentelemetry/docs/exporters'
+state:
+  persistent: false
+  files: []
+depends_on: []