@vzn/vx-otel 0.0.518 → 0.0.520

package/README.md

@@ -105,7 +105,17 @@ otel({
   a sandboxed one its violation count (`vx.task.sandbox_violations`), on a
   persistent one that never became ready why (`vx.task.not_ready`). A
   failed task sets span status
-  `ERROR`.
+  `ERROR`. A task span links to the spans of the tasks it waited on (a
+  group seen through to the tasks behind it), and carries an event per
+  attempt that failed and was run again (`vx.task.retry`, with
+  `vx.task.attempt` and `vx.task.exit_code`) and one when vx's own
+  timeout killed it (`vx.task.timeout`);
+- a child span per stage of the run, named by the stage (`startup`,
+  `load configs`, `git enumeration`, `classify + probe`, `run graph`,
+  `record history`, …, the stages `VX_TIMING` prints), with
+  `vx.stage.name`: where the time before the first task and after the
+  last went. The stages ahead of the run lock end before the `vx.run`
+  span starts.
 
 **Metrics per run**: `vx.tasks.total`, `vx.tasks.failed`,
 `vx.tasks.cache_hits{source=local|remote}`, what those hits did to the disk as
@@ -116,6 +126,15 @@ and the `vx.run.duration_ms` gauge. The `vx.run` span carries the same counts
 The counts are DELTA sums over the run's own interval (start to end), so a
 backend adds runs rather than reading each as the series' new total.
 
+**Metrics per task**, as gauges keyed by `cicd.pipeline.task.name`,
+`vx.task.project` and `vx.task.task`: at each task's end
+`vx.task.duration` (ms, with `vx.cache.source`), and for a task the runner
+measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes); a
+skipped task sends none. While a task runs, its process tree is sampled
+each second: `vx.task.cpu_usage` (cores busy since the last sample, 1 =
+one core) and `vx.task.memory` (resident bytes). Sampling runs only while
+metrics export; a remote task, or a task under 1 s, has no samples.
+
 **Logs per run** (on by default): the captured output tail of each executed
 task, as one log record linked to its task span (unlinked when traces are
 off: the span is never exported). Build output can hold

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-otel",
-  "version": "0.0.518",
+  "version": "0.0.520",
   "license": "MIT",
   "description": "OpenTelemetry exporter for vx — a telemetry plugin mapping runs to OTLP traces + metrics (CI/CD + VCS semantic conventions).",
   "keywords": [
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.518"
+    "@vzn/vx": "^0.0.520"
   },
   "repository": {
     "type": "git",

package/src/otlp.ts

@@ -6,7 +6,9 @@
 // of SDK-version drift. Maps a vx run to:
 //   - a TRACE: one root `vx.run` span + one child `vx.task` span per task,
 //     with CI/CD + VCS semantic-convention attributes;
-//   - METRICS: task/run counters + a run-duration gauge;
+//   - METRICS: task/run counters + a run-duration gauge; per task its
+//     duration, CPU time and peak memory, and while it runs its process
+//     tree's CPU usage and memory, sampled each second;
 //   - LOGS: one record per executed task carrying its captured output tail.
 //
 // References: OpenTelemetry CI/CD + VCS semantic conventions; OTLP/JSON
@@ -124,6 +126,8 @@ const VX_ATTR = {
   taskOutputs: 'vx.task.outputs',
   peakRssBytes: 'vx.peak_rss_bytes',
   taskAttempts: 'vx.task.attempts',
+  taskAttempt: 'vx.task.attempt',
+  stageName: 'vx.stage.name',
   wallclockStartNs: 'vx.task.wallclock_start_ns',
   wallclockEndNs: 'vx.task.wallclock_end_ns',
   // task log records (the OTel Logs signal)
@@ -149,6 +153,76 @@ export interface OtlpSpan {
   endTimeUnixNano: string
   attributes: KeyValue[]
   status: { code: number }
+  events?: OtlpSpanEvent[]
+  links?: OtlpSpanLink[]
+}
+
+export interface OtlpSpanEvent {
+  timeUnixNano: string
+  name: string
+  attributes: KeyValue[]
+}
+
+export interface OtlpSpanLink {
+  traceId: string
+  spanId: string
+  attributes: KeyValue[]
+}
+
+/**
+ * A task span's events: each attempt that failed and was run again
+ * (`vx.task.retry`, at its end), and vx's own deadline killing the task
+ * (`vx.task.timeout`, at the span's end).
+ */
+export function taskSpanEvents(t: TaskTelemetry, endUnixNano: string): OtlpSpanEvent[] {
+  const events: OtlpSpanEvent[] = (t.failedAttempts ?? []).map((a, i) => ({
+    timeUnixNano: String(Math.trunc(a.endedAt) * 1_000_000),
+    name: 'vx.task.retry',
+    attributes: [
+      intAttr(VX_ATTR.taskAttempt, i + 1),
+      intAttr(VX_ATTR.taskExitCode, a.exitCode),
+      ...(a.timedOut === true ? [boolAttr(VX_ATTR.taskTimedOut, true)] : []),
+    ],
+  }))
+  if (t.timedOut === true) {
+    events.push({ timeUnixNano: endUnixNano, name: 'vx.task.timeout', attributes: [] })
+  }
+  return events
+}
+
+/**
+ * A child span of the run for one of its stages (`VX_TIMING`'s marks):
+ * where the time before the first task and after the last one went. Named
+ * by the stage, a fixed set. The stages ahead of the run lock end before
+ * the run span starts.
+ */
+export function stageSpan(
+  traceId: string,
+  parentSpanId: string,
+  stage: { name: string; startedAt: number; endedAt: number },
+  spanId: string,
+): OtlpSpan {
+  return {
+    traceId,
+    spanId,
+    parentSpanId,
+    name: stage.name,
+    kind: SPAN_KIND_INTERNAL,
+    startTimeUnixNano: exactNanos(stage.startedAt),
+    endTimeUnixNano: exactNanos(stage.endedAt),
+    attributes: [strAttr(VX_ATTR.stageName, stage.name)],
+    status: { code: STATUS_UNSET },
+  }
+}
+
+/** Epoch ms with a fraction, as OTLP's int64 nanoseconds: a stage can be under 1 ms. */
+function exactNanos(ms: number): string {
+  return String(BigInt(Math.round(ms * 1000)) * 1000n)
+}
+
+/** A link from a task span to the span of a task it waited on. */
+export function dependencyLink(traceId: string, spanId: string, taskId: string): OtlpSpanLink {
+  return { traceId, spanId, attributes: [strAttr(SEMCONV.taskName, taskId)] }
 }
 
 // --- attribute mapping (pure) ------------------------------------------
@@ -367,6 +441,7 @@ export function buildMetricsRequest(
   nowUnixNano: string,
   startUnixNano: string,
   resource?: Readonly<Record<string, string>>,
+  points: readonly TaskMetricPoint[] = [],
 ): unknown {
   const point = (value: number, attrs: KeyValue[] = []) => ({
     asInt: String(value),
@@ -408,6 +483,7 @@ export function buildMetricsRequest(
               ]),
               sum('vx.tasks.cache_up_to_date', [point(summary.upToDateCount)]),
               gauge('vx.run.duration_ms', summary.totalDurationMs),
+              ...taskMetrics(points),
             ],
           },
         ],
@@ -416,6 +492,90 @@ export function buildMetricsRequest(
   }
 }
 
+/**
+ * A per-task metric point: a task's totals at its end, or one live sample
+ * of its process tree while it runs.
+ */
+export type TaskMetricPoint =
+  | { kind: 'end'; task: TaskTelemetry; startUnixNano: string; endUnixNano: string }
+  | {
+      kind: 'sample'
+      taskId: string
+      project: string
+      task: string
+      timeUnixNano: string
+      /** CPU cores busy since the previous sample (1 = one core). */
+      cpuUsage: number
+      rssBytes: number
+    }
+
+/**
+ * An ExportMetricsServiceRequest of per-task gauges alone: the points past
+ * what fit beside the run's own metrics (`buildMetricsRequest`).
+ */
+export function buildTaskMetricsRequest(
+  serviceName: string,
+  vxVersion: string,
+  points: readonly TaskMetricPoint[],
+  resource?: Readonly<Record<string, string>>,
+): unknown {
+  return {
+    resourceMetrics: [
+      {
+        resource: { attributes: resourceAttributes(serviceName, vxVersion, resource) },
+        scopeMetrics: [{ scope: { name: 'vx', version: vxVersion }, metrics: taskMetrics(points) }],
+      },
+    ],
+  }
+}
+
+/**
+ * Per-task gauges, keyed by task. At each task's end: its duration, CPU time and peak memory (CPU and memory as
+ * the runner measured them at exit; a task it did not measure, a cache hit or
+ * a remote run, sends no point for them). While a task runs: its process
+ * tree's CPU usage and memory, once per sample. A task's numbers otherwise
+ * live only on its span, which no metrics backend charts.
+ */
+function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
+  const series = new Map<string, { unit: string; dataPoints: unknown[] }>()
+  const add = (name: string, unit: string, point: unknown): void => {
+    const s = series.get(name)
+    if (s === undefined) series.set(name, { unit, dataPoints: [point] })
+    else s.dataPoints.push(point)
+  }
+  const taskAttrs = (taskId: string, project: string, task: string): KeyValue[] => [
+    strAttr(SEMCONV.taskName, taskId),
+    strAttr(VX_ATTR.taskProject, project),
+    strAttr(VX_ATTR.taskTask, task),
+  ]
+  for (const p of points) {
+    if (p.kind === 'sample') {
+      const at = (value: number) => ({
+        asDouble: value,
+        timeUnixNano: p.timeUnixNano,
+        attributes: taskAttrs(p.taskId, p.project, p.task),
+      })
+      add('vx.task.cpu_usage', '1', at(p.cpuUsage))
+      add('vx.task.memory', 'By', at(p.rssBytes))
+      continue
+    }
+    const t = p.task
+    const at = (value: number) => ({
+      asDouble: value,
+      startTimeUnixNano: p.startUnixNano,
+      timeUnixNano: p.endUnixNano,
+      attributes: [
+        ...taskAttrs(t.taskId, t.project, t.task),
+        strAttr(VX_ATTR.cacheSource, t.cacheSource),
+      ],
+    })
+    add('vx.task.duration', 'ms', at(t.durationMs))
+    if (t.cpuMs !== undefined) add('vx.task.cpu_time', 'ms', at(t.cpuMs))
+    if (t.peakRssBytes !== undefined) add('vx.task.peak_memory', 'By', at(t.peakRssBytes))
+  }
+  return [...series].map(([name, { unit, dataPoints }]) => ({ name, unit, gauge: { dataPoints } }))
+}
+
 // --- logs ---------------------------------------------------------------
 
 // OTLP severity numbers: 9 INFO, 17 ERROR.

package/src/sink.ts

@@ -16,10 +16,15 @@ import type {
 import {
   buildLogsRequest,
   buildMetricsRequest,
+  buildTaskMetricsRequest,
   buildTraceRequest,
   type OtlpSpan,
+  type TaskMetricPoint,
   runSpanAttributes,
   SPAN_KIND_INTERNAL,
+  dependencyLink,
+  stageSpan,
+  taskSpanEvents,
   runStatusCode,
   taskSpanAttributes,
   taskStatusCode,
@@ -269,8 +274,12 @@ export class OtelSink implements TelemetrySink {
   private rootStartNano = '0'
   private rootEndNano = '0'
   private readonly spans: OtlpSpan[] = []
+  private readonly taskPoints: TaskMetricPoint[] = []
+  private readonly taskNames = new Map<string, { project: string; task: string; startMs: number }>()
+  private readonly lastSample = new Map<string, { ts: number; cpuMs: number }>()
   private readonly taskSpanId = new Map<string, string>()
   private readonly taskStartNano = new Map<string, string>()
+  private readonly taskDeps = new Map<string, readonly string[]>()
   private summary: RunSummaryRecord | undefined
   private uploaded = false
   // Core's own bounded capture buffer, so which task's output survives a
@@ -301,9 +310,14 @@ export class OtelSink implements TelemetrySink {
       post: config.post ?? defaultPost(config.timeoutMs),
       ...(config.warn ? { warn: config.warn } : {}),
     }
-    this.wants = config.logsEnabled
-      ? ['run.start', 'task.start', 'task.log', 'task.end', 'run.end']
-      : ['run.start', 'task.start', 'task.end', 'run.end']
+    this.wants = [
+      'run.start',
+      'task.start',
+      ...(config.logsEnabled ? (['task.log'] as const) : []),
+      ...(config.metricsEnabled ? (['task.sample'] as const) : []),
+      'task.end',
+      'run.end',
+    ]
   }
 
   onRecord(record: TelemetryRecord): void {
@@ -321,12 +335,50 @@ export class OtelSink implements TelemetrySink {
       case 'task.start':
         this.taskSpanId.set(record.taskId, genId(8))
         this.taskStartNano.set(record.taskId, nanos(record.ts))
+        if (record.dependsOn !== undefined) this.taskDeps.set(record.taskId, record.dependsOn)
+        this.taskNames.set(record.taskId, {
+          project: record.project,
+          task: record.task,
+          startMs: record.ts,
+        })
+        return
+      case 'task.sample': {
+        // Usage is the CPU the tree spent since the last look, over the
+        // wall time between: the first look counts from the task's start.
+        const names = this.taskNames.get(record.taskId)
+        if (names === undefined) return
+        const prev = this.lastSample.get(record.taskId) ?? { ts: names.startMs, cpuMs: 0 }
+        const wallMs = record.ts - prev.ts
+        // A descendant that exited takes its CPU out of the sum: no negative usage.
+        const cpuUsage = wallMs > 0 ? Math.max(0, record.cpuMs - prev.cpuMs) / wallMs : 0
+        this.lastSample.set(record.taskId, { ts: record.ts, cpuMs: record.cpuMs })
+        this.taskPoints.push({
+          kind: 'sample',
+          taskId: record.taskId,
+          project: names.project,
+          task: names.task,
+          timeUnixNano: nanos(record.ts),
+          cpuUsage,
+          rssBytes: record.rssBytes,
+        })
         return
+      }
       case 'task.log':
         this.logs.append(record.taskId, record.chunk)
         return
       case 'task.end': {
-        const spanId = this.taskSpanId.get(record.taskId) ?? genId(8)
+        // A task that never started (a skip) still gets a span, and a later
+        // task that waited on it links to it.
+        let spanId = this.taskSpanId.get(record.taskId)
+        if (spanId === undefined) {
+          spanId = genId(8)
+          this.taskSpanId.set(record.taskId, spanId)
+        }
+        const links = (this.taskDeps.get(record.taskId) ?? []).flatMap((dep) => {
+          const depSpan = this.taskSpanId.get(dep)
+          return depSpan === undefined ? [] : [dependencyLink(this.traceId, depSpan, dep)]
+        })
+        const events = taskSpanEvents(record, nanos(record.ts))
         const startNano =
           this.taskStartNano.get(record.taskId) ?? nanos(record.ts - record.durationMs)
         const t: TaskTelemetry = record
@@ -344,7 +396,19 @@ export class OtelSink implements TelemetrySink {
             startedAt: this.runStartedAt,
           }),
           status: { code: taskStatusCode(t) },
+          ...(events.length > 0 ? { events } : {}),
+          ...(links.length > 0 ? { links } : {}),
         })
+        // A skipped task never ran: a zero duration would read as a fast one.
+        if (record.status !== 'skipped') {
+          this.taskPoints.push({
+            kind: 'end',
+            task: t,
+            startUnixNano: startNano,
+            endUnixNano: nanos(record.ts),
+          })
+        }
+        this.lastSample.delete(record.taskId)
         // Decides retention: a cache hit's bytes belong to the run that
         // executed them, so only an executed success/failure keeps a tail.
         if (this.cfg.logsEnabled) {
@@ -391,6 +455,10 @@ export class OtelSink implements TelemetrySink {
         attributes: runSpanAttributes(this.run, this.summary),
         status: { code: runStatusCode(this.summary) },
       })
+      const stages = (this.summary?.stages ?? []).map((stage) =>
+        stageSpan(this.traceId, this.rootSpanId, stage, genId(8)),
+      )
+      this.spans.splice(1, 0, ...stages)
     }
     const vxVersion = this.run?.vxVersion ?? '0.0.0'
     await Promise.all([this.shipTraces(vxVersion), this.shipMetrics(), this.shipLogs(vxVersion)])
@@ -406,16 +474,39 @@ export class OtelSink implements TelemetrySink {
 
   private async shipMetrics(): Promise<void> {
     if (!this.cfg.metricsEnabled || this.summary === undefined) return
-    const body = JSON.stringify(
+    const summary = this.summary
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const first = this.taskPoints[0]
+    // The run's own metrics ride with the first batch of per-task points.
+    const bodies =
+      first === undefined
+        ? [this.runMetricsBody(summary, [])]
+        : requestBodies(this.taskPoints, (points) =>
+            points[0] === first
+              ? this.runMetricsBody(summary, points)
+              : JSON.stringify(
+                  buildTaskMetricsRequest(
+                    this.cfg.serviceName,
+                    vxVersion,
+                    points,
+                    this.cfg.resource,
+                  ),
+                ),
+          )
+    await this.send('metrics', this.cfg.metricsUrl, bodies)
+  }
+
+  private runMetricsBody(summary: RunSummaryRecord, points: TaskMetricPoint[]): string {
+    return JSON.stringify(
       buildMetricsRequest(
         this.cfg.serviceName,
-        this.summary,
-        nanos(this.summary.endedAt),
-        nanos(this.summary.startedAt),
+        summary,
+        nanos(summary.endedAt),
+        nanos(summary.startedAt),
         this.cfg.resource,
+        points,
       ),
     )
-    await this.send('metrics', this.cfg.metricsUrl, [body])
   }
 
   private async shipLogs(vxVersion: string): Promise<void> {