@vzn/vx-otel 0.0.517 → 0.0.519

package/README.md

@@ -116,6 +116,15 @@ and the `vx.run.duration_ms` gauge. The `vx.run` span carries the same counts
 The counts are DELTA sums over the run's own interval (start to end), so a
 backend adds runs rather than reading each as the series' new total.
 
+**Metrics per task**, as gauges keyed by `cicd.pipeline.task.name`,
+`vx.task.project` and `vx.task.task`: at each task's end
+`vx.task.duration` (ms, with `vx.cache.source`), and for a task the runner
+measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes); a
+skipped task sends none. While a task runs, its process tree is sampled
+each second: `vx.task.cpu_usage` (cores busy since the last sample, 1 =
+one core) and `vx.task.memory` (resident bytes). Sampling runs only while
+metrics export; a remote task, or a task under 1 s, has no samples.
+
 **Logs per run** (on by default): the captured output tail of each executed
 task, as one log record linked to its task span (unlinked when traces are
 off: the span is never exported). Build output can hold

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-otel",
-  "version": "0.0.517",
+  "version": "0.0.519",
   "license": "MIT",
   "description": "OpenTelemetry exporter for vx — a telemetry plugin mapping runs to OTLP traces + metrics (CI/CD + VCS semantic conventions).",
   "keywords": [
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.517"
+    "@vzn/vx": "^0.0.519"
   },
   "repository": {
     "type": "git",

package/src/otlp.ts

@@ -6,7 +6,9 @@
 // of SDK-version drift. Maps a vx run to:
 //   - a TRACE: one root `vx.run` span + one child `vx.task` span per task,
 //     with CI/CD + VCS semantic-convention attributes;
-//   - METRICS: task/run counters + a run-duration gauge;
+//   - METRICS: task/run counters + a run-duration gauge; per task its
+//     duration, CPU time and peak memory, and while it runs its process
+//     tree's CPU usage and memory, sampled each second;
 //   - LOGS: one record per executed task carrying its captured output tail.
 //
 // References: OpenTelemetry CI/CD + VCS semantic conventions; OTLP/JSON
@@ -367,6 +369,7 @@ export function buildMetricsRequest(
   nowUnixNano: string,
   startUnixNano: string,
   resource?: Readonly<Record<string, string>>,
+  points: readonly TaskMetricPoint[] = [],
 ): unknown {
   const point = (value: number, attrs: KeyValue[] = []) => ({
     asInt: String(value),
@@ -408,6 +411,7 @@ export function buildMetricsRequest(
               ]),
               sum('vx.tasks.cache_up_to_date', [point(summary.upToDateCount)]),
               gauge('vx.run.duration_ms', summary.totalDurationMs),
+              ...taskMetrics(points),
             ],
           },
         ],
@@ -416,6 +420,90 @@ export function buildMetricsRequest(
   }
 }
 
+/**
+ * A per-task metric point: a task's totals at its end, or one live sample
+ * of its process tree while it runs.
+ */
+export type TaskMetricPoint =
+  | { kind: 'end'; task: TaskTelemetry; startUnixNano: string; endUnixNano: string }
+  | {
+      kind: 'sample'
+      taskId: string
+      project: string
+      task: string
+      timeUnixNano: string
+      /** CPU cores busy since the previous sample (1 = one core). */
+      cpuUsage: number
+      rssBytes: number
+    }
+
+/**
+ * An ExportMetricsServiceRequest of per-task gauges alone: the points past
+ * what fit beside the run's own metrics (`buildMetricsRequest`).
+ */
+export function buildTaskMetricsRequest(
+  serviceName: string,
+  vxVersion: string,
+  points: readonly TaskMetricPoint[],
+  resource?: Readonly<Record<string, string>>,
+): unknown {
+  return {
+    resourceMetrics: [
+      {
+        resource: { attributes: resourceAttributes(serviceName, vxVersion, resource) },
+        scopeMetrics: [{ scope: { name: 'vx', version: vxVersion }, metrics: taskMetrics(points) }],
+      },
+    ],
+  }
+}
+
+/**
+ * Per-task gauges, keyed by task. At each task's end: its duration, CPU time and peak memory (CPU and memory as
+ * the runner measured them at exit; a task it did not measure, a cache hit or
+ * a remote run, sends no point for them). While a task runs: its process
+ * tree's CPU usage and memory, once per sample. A task's numbers otherwise
+ * live only on its span, which no metrics backend charts.
+ */
+function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
+  const series = new Map<string, { unit: string; dataPoints: unknown[] }>()
+  const add = (name: string, unit: string, point: unknown): void => {
+    const s = series.get(name)
+    if (s === undefined) series.set(name, { unit, dataPoints: [point] })
+    else s.dataPoints.push(point)
+  }
+  const taskAttrs = (taskId: string, project: string, task: string): KeyValue[] => [
+    strAttr(SEMCONV.taskName, taskId),
+    strAttr(VX_ATTR.taskProject, project),
+    strAttr(VX_ATTR.taskTask, task),
+  ]
+  for (const p of points) {
+    if (p.kind === 'sample') {
+      const at = (value: number) => ({
+        asDouble: value,
+        timeUnixNano: p.timeUnixNano,
+        attributes: taskAttrs(p.taskId, p.project, p.task),
+      })
+      add('vx.task.cpu_usage', '1', at(p.cpuUsage))
+      add('vx.task.memory', 'By', at(p.rssBytes))
+      continue
+    }
+    const t = p.task
+    const at = (value: number) => ({
+      asDouble: value,
+      startTimeUnixNano: p.startUnixNano,
+      timeUnixNano: p.endUnixNano,
+      attributes: [
+        ...taskAttrs(t.taskId, t.project, t.task),
+        strAttr(VX_ATTR.cacheSource, t.cacheSource),
+      ],
+    })
+    add('vx.task.duration', 'ms', at(t.durationMs))
+    if (t.cpuMs !== undefined) add('vx.task.cpu_time', 'ms', at(t.cpuMs))
+    if (t.peakRssBytes !== undefined) add('vx.task.peak_memory', 'By', at(t.peakRssBytes))
+  }
+  return [...series].map(([name, { unit, dataPoints }]) => ({ name, unit, gauge: { dataPoints } }))
+}
+
 // --- logs ---------------------------------------------------------------
 
 // OTLP severity numbers: 9 INFO, 17 ERROR.

package/src/sink.ts

@@ -16,8 +16,10 @@ import type {
 import {
   buildLogsRequest,
   buildMetricsRequest,
+  buildTaskMetricsRequest,
   buildTraceRequest,
   type OtlpSpan,
+  type TaskMetricPoint,
   runSpanAttributes,
   SPAN_KIND_INTERNAL,
   runStatusCode,
@@ -269,6 +271,9 @@ export class OtelSink implements TelemetrySink {
   private rootStartNano = '0'
   private rootEndNano = '0'
   private readonly spans: OtlpSpan[] = []
+  private readonly taskPoints: TaskMetricPoint[] = []
+  private readonly taskNames = new Map<string, { project: string; task: string; startMs: number }>()
+  private readonly lastSample = new Map<string, { ts: number; cpuMs: number }>()
   private readonly taskSpanId = new Map<string, string>()
   private readonly taskStartNano = new Map<string, string>()
   private summary: RunSummaryRecord | undefined
@@ -301,9 +306,14 @@ export class OtelSink implements TelemetrySink {
       post: config.post ?? defaultPost(config.timeoutMs),
       ...(config.warn ? { warn: config.warn } : {}),
     }
-    this.wants = config.logsEnabled
-      ? ['run.start', 'task.start', 'task.log', 'task.end', 'run.end']
-      : ['run.start', 'task.start', 'task.end', 'run.end']
+    this.wants = [
+      'run.start',
+      'task.start',
+      ...(config.logsEnabled ? (['task.log'] as const) : []),
+      ...(config.metricsEnabled ? (['task.sample'] as const) : []),
+      'task.end',
+      'run.end',
+    ]
   }
 
   onRecord(record: TelemetryRecord): void {
@@ -321,7 +331,33 @@ export class OtelSink implements TelemetrySink {
       case 'task.start':
         this.taskSpanId.set(record.taskId, genId(8))
         this.taskStartNano.set(record.taskId, nanos(record.ts))
+        this.taskNames.set(record.taskId, {
+          project: record.project,
+          task: record.task,
+          startMs: record.ts,
+        })
+        return
+      case 'task.sample': {
+        // Usage is the CPU the tree spent since the last look, over the
+        // wall time between: the first look counts from the task's start.
+        const names = this.taskNames.get(record.taskId)
+        if (names === undefined) return
+        const prev = this.lastSample.get(record.taskId) ?? { ts: names.startMs, cpuMs: 0 }
+        const wallMs = record.ts - prev.ts
+        // A descendant that exited takes its CPU out of the sum: no negative usage.
+        const cpuUsage = wallMs > 0 ? Math.max(0, record.cpuMs - prev.cpuMs) / wallMs : 0
+        this.lastSample.set(record.taskId, { ts: record.ts, cpuMs: record.cpuMs })
+        this.taskPoints.push({
+          kind: 'sample',
+          taskId: record.taskId,
+          project: names.project,
+          task: names.task,
+          timeUnixNano: nanos(record.ts),
+          cpuUsage,
+          rssBytes: record.rssBytes,
+        })
         return
+      }
       case 'task.log':
         this.logs.append(record.taskId, record.chunk)
         return
@@ -345,6 +381,16 @@ export class OtelSink implements TelemetrySink {
           }),
           status: { code: taskStatusCode(t) },
         })
+        // A skipped task never ran: a zero duration would read as a fast one.
+        if (record.status !== 'skipped') {
+          this.taskPoints.push({
+            kind: 'end',
+            task: t,
+            startUnixNano: startNano,
+            endUnixNano: nanos(record.ts),
+          })
+        }
+        this.lastSample.delete(record.taskId)
         // Decides retention: a cache hit's bytes belong to the run that
         // executed them, so only an executed success/failure keeps a tail.
         if (this.cfg.logsEnabled) {
@@ -406,16 +452,39 @@ export class OtelSink implements TelemetrySink {
 
   private async shipMetrics(): Promise<void> {
     if (!this.cfg.metricsEnabled || this.summary === undefined) return
-    const body = JSON.stringify(
+    const summary = this.summary
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const first = this.taskPoints[0]
+    // The run's own metrics ride with the first batch of per-task points.
+    const bodies =
+      first === undefined
+        ? [this.runMetricsBody(summary, [])]
+        : requestBodies(this.taskPoints, (points) =>
+            points[0] === first
+              ? this.runMetricsBody(summary, points)
+              : JSON.stringify(
+                  buildTaskMetricsRequest(
+                    this.cfg.serviceName,
+                    vxVersion,
+                    points,
+                    this.cfg.resource,
+                  ),
+                ),
+          )
+    await this.send('metrics', this.cfg.metricsUrl, bodies)
+  }
+
+  private runMetricsBody(summary: RunSummaryRecord, points: TaskMetricPoint[]): string {
+    return JSON.stringify(
       buildMetricsRequest(
         this.cfg.serviceName,
-        this.summary,
-        nanos(this.summary.endedAt),
-        nanos(this.summary.startedAt),
+        summary,
+        nanos(summary.endedAt),
+        nanos(summary.startedAt),
         this.cfg.resource,
+        points,
       ),
     )
-    await this.send('metrics', this.cfg.metricsUrl, [body])
   }
 
   private async shipLogs(vxVersion: string): Promise<void> {