@vzn/vx-otel 0.0.519 → 0.0.521

package/README.md

@@ -103,14 +103,32 @@ otel({
   `vx.task.hash`, duration, CPU ms, peak RSS, and on a skipped task its root
   blocker (`vx.task.blocked_by`), on a timed-out one `vx.task.timed_out`, on
   a sandboxed one its violation count (`vx.task.sandbox_violations`), on a
-  persistent one that never became ready why (`vx.task.not_ready`). A
+  persistent one that never became ready why (`vx.task.not_ready`), its
+  command (`vx.task.command`, secrets masked), on a flaky one its record
+  (`vx.task.flaky.passes`, `vx.task.flaky.failures`), on a hit what the
+  stored run took (`vx.cache.stored_duration_ms`, `vx.cache.stored_cpu_ms`,
+  `vx.cache.stored_peak_rss_bytes`), and how long an `admit` policy held it
+  (`vx.task.admission_held_ms`). A
   failed task sets span status
-  `ERROR`.
+  `ERROR`. A task span links to the spans of the tasks it waited on (a
+  group seen through to the tasks behind it), and carries an event per
+  attempt that failed and was run again (`vx.task.retry`, with
+  `vx.task.attempt` and `vx.task.exit_code`), one when vx's own
+  timeout killed it (`vx.task.timeout`), and one per sandbox violation
+  (`vx.sandbox.violation`, its line; past 100, `vx.sandbox.violations_dropped`
+  says how many more);
+- a child span per stage of the run, named by the stage (`startup`,
+  `load configs`, `git enumeration`, `classify + probe`, `run graph`,
+  `record history`, …, the stages `VX_TIMING` prints), with
+  `vx.stage.name`: where the time before the first task and after the
+  last went. The stages ahead of the run lock end before the `vx.run`
+  span starts.
 
 **Metrics per run**: `vx.tasks.total`, `vx.tasks.failed`,
 `vx.tasks.cache_hits{source=local|remote}`, what those hits did to the disk as
 `vx.tasks.cache_restored{source=local|remote}` and `vx.tasks.cache_up_to_date`,
-and the `vx.run.duration_ms` gauge. The `vx.run` span carries the same counts
+and the `vx.run.duration_ms` and `vx.run.time_saved_ms` (what the hits'
+stored runs took) gauges. The `vx.run` span carries the same counts
 (`vx.run.hit_count`, `vx.run.up_to_date_count`, `vx.run.restored_local_count`,
 `vx.run.restored_remote_count`).
 The counts are DELTA sums over the run's own interval (start to end), so a
@@ -119,18 +137,42 @@ backend adds runs rather than reading each as the series' new total.
 **Metrics per task**, as gauges keyed by `cicd.pipeline.task.name`,
 `vx.task.project` and `vx.task.task`: at each task's end
 `vx.task.duration` (ms, with `vx.cache.source`), and for a task the runner
-measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes); a
-skipped task sends none. While a task runs, its process tree is sampled
+measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes), on a
+hit `vx.task.time_saved` (ms), and `vx.task.admission_held` (ms) when an
+`admit` policy held it; a skipped task sends none. While a task runs, its process tree is sampled
 each second: `vx.task.cpu_usage` (cores busy since the last sample, 1 =
 one core) and `vx.task.memory` (resident bytes). Sampling runs only while
 metrics export; a remote task, or a task under 1 s, has no samples.
 
 **Logs per run** (on by default): the captured output tail of each executed
-task, as one log record linked to its task span (unlinked when traces are
-off: the span is never exported). Build output can hold
+task, as one log record at the task's end, linked to its task span (unlinked
+when traces are off: the span is never exported). Build output can hold
 secrets, and this sends it to the collector; `logs: false` or
 `OTEL_LOGS_EXPORTER=none` turns it off.
 
+**One run, three signals, linked.** Every request's resource names the run
+(`service.instance.id` and `cicd.pipeline.run.id`, its id), the host
+(`host.name`, `host.arch`, `os.type`) and the commit (`vcs.ref.head.*`),
+under `OTEL_RESOURCE_ATTRIBUTES`, which wins. Each task metric point carries
+its task's span as an exemplar, and each log record its span, so a chart
+opens the trace and a span opens its output.
+
+## Live export
+
+```ts
+otel({ live: process.env.CI === 'true' })
+```
+
+With `live: true` each task's span, metrics and output tail are sent as the
+task ends, its process samples as they are taken, and a log record
+(`event.name` `vx.run.start` or `vx.task.start`, linked to the trace) as the
+run and each task start; the `vx.run` span, the stage spans and the run's
+metrics follow at the end. A dashboard follows a CI run while it runs. One
+send is in flight at a time and takes everything that ended meanwhile; a
+refused send warns once per signal for the run. Cost on 60 tasks against a
+local collector: within the noise of run-to-run wall time (min of 6: 1,101 ms
+off, 1,108 ms on). Off by default.
+
 ## Behavior note
 
 This replaces core's previous hardcoded OTel emit, which fired automatically

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-otel",
-  "version": "0.0.519",
+  "version": "0.0.521",
   "license": "MIT",
   "description": "OpenTelemetry exporter for vx — a telemetry plugin mapping runs to OTLP traces + metrics (CI/CD + VCS semantic conventions).",
   "keywords": [
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.519"
+    "@vzn/vx": "^0.0.521"
   },
   "repository": {
     "type": "git",

package/src/otlp.ts

@@ -57,6 +57,11 @@ const SEMCONV = {
   vcsHeadName: 'vcs.ref.head.name',
   serviceName: 'service.name',
   serviceVersion: 'service.version',
+  serviceInstanceId: 'service.instance.id',
+  hostName: 'host.name',
+  hostArch: 'host.arch',
+  osType: 'os.type',
+  eventName: 'event.name',
 } as const
 
 /**
@@ -126,6 +131,17 @@ const VX_ATTR = {
   taskOutputs: 'vx.task.outputs',
   peakRssBytes: 'vx.peak_rss_bytes',
   taskAttempts: 'vx.task.attempts',
+  taskAttempt: 'vx.task.attempt',
+  taskCommand: 'vx.task.command',
+  taskFlakyPasses: 'vx.task.flaky.passes',
+  taskFlakyFailures: 'vx.task.flaky.failures',
+  taskAdmissionHeldMs: 'vx.task.admission_held_ms',
+  storedDurationMs: 'vx.cache.stored_duration_ms',
+  storedCpuMs: 'vx.cache.stored_cpu_ms',
+  storedPeakRssBytes: 'vx.cache.stored_peak_rss_bytes',
+  sandboxViolation: 'vx.sandbox.violation',
+  sandboxViolationsDropped: 'vx.sandbox.violations_dropped',
+  stageName: 'vx.stage.name',
   wallclockStartNs: 'vx.task.wallclock_start_ns',
   wallclockEndNs: 'vx.task.wallclock_end_ns',
   // task log records (the OTel Logs signal)
@@ -151,6 +167,95 @@ export interface OtlpSpan {
   endTimeUnixNano: string
   attributes: KeyValue[]
   status: { code: number }
+  events?: OtlpSpanEvent[]
+  links?: OtlpSpanLink[]
+}
+
+export interface OtlpSpanEvent {
+  timeUnixNano: string
+  name: string
+  attributes: KeyValue[]
+}
+
+export interface OtlpSpanLink {
+  traceId: string
+  spanId: string
+  attributes: KeyValue[]
+}
+
+/**
+ * A task span's events: each attempt that failed and was run again
+ * (`vx.task.retry`, at its end), vx's own deadline killing the task
+ * (`vx.task.timeout`, at the span's end), and each sandbox violation
+ * (`vx.sandbox.violation`, its line).
+ */
+export function taskSpanEvents(t: TaskTelemetry, endUnixNano: string): OtlpSpanEvent[] {
+  const events: OtlpSpanEvent[] = (t.failedAttempts ?? []).map((a, i) => ({
+    timeUnixNano: String(Math.trunc(a.endedAt) * 1_000_000),
+    name: 'vx.task.retry',
+    attributes: [
+      intAttr(VX_ATTR.taskAttempt, i + 1),
+      intAttr(VX_ATTR.taskExitCode, a.exitCode),
+      ...(a.timedOut === true ? [boolAttr(VX_ATTR.taskTimedOut, true)] : []),
+    ],
+  }))
+  if (t.timedOut === true) {
+    events.push({ timeUnixNano: endUnixNano, name: 'vx.task.timeout', attributes: [] })
+  }
+  const lines = t.sandboxViolationLines ?? []
+  for (const line of lines.slice(0, MAX_VIOLATION_EVENTS)) {
+    events.push({
+      timeUnixNano: endUnixNano,
+      name: VX_ATTR.sandboxViolation,
+      attributes: [strAttr(VX_ATTR.sandboxViolation, line)],
+    })
+  }
+  if (lines.length > MAX_VIOLATION_EVENTS) {
+    events.push({
+      timeUnixNano: endUnixNano,
+      name: VX_ATTR.sandboxViolationsDropped,
+      attributes: [intAttr(VX_ATTR.sandboxViolationsDropped, lines.length - MAX_VIOLATION_EVENTS)],
+    })
+  }
+  return events
+}
+
+/** A collector keeps 128 events per span by default; the rest say how many they were. */
+const MAX_VIOLATION_EVENTS = 100
+
+/**
+ * A child span of the run for one of its stages (`VX_TIMING`'s marks):
+ * where the time before the first task and after the last one went. Named
+ * by the stage, a fixed set. The stages ahead of the run lock end before
+ * the run span starts.
+ */
+export function stageSpan(
+  traceId: string,
+  parentSpanId: string,
+  stage: { name: string; startedAt: number; endedAt: number },
+  spanId: string,
+): OtlpSpan {
+  return {
+    traceId,
+    spanId,
+    parentSpanId,
+    name: stage.name,
+    kind: SPAN_KIND_INTERNAL,
+    startTimeUnixNano: exactNanos(stage.startedAt),
+    endTimeUnixNano: exactNanos(stage.endedAt),
+    attributes: [strAttr(VX_ATTR.stageName, stage.name)],
+    status: { code: STATUS_UNSET },
+  }
+}
+
+/** Epoch ms with a fraction, as OTLP's int64 nanoseconds: a stage can be under 1 ms. */
+function exactNanos(ms: number): string {
+  return String(BigInt(Math.round(ms * 1000)) * 1000n)
+}
+
+/** A link from a task span to the span of a task it waited on. */
+export function dependencyLink(traceId: string, spanId: string, taskId: string): OtlpSpanLink {
+  return { traceId, spanId, attributes: [strAttr(SEMCONV.taskName, taskId)] }
 }
 
 // --- attribute mapping (pure) ------------------------------------------
@@ -175,6 +280,24 @@ function resourceAttributes(
   ]
 }
 
+/**
+ * The run as resource attributes, so every signal it sends (spans, metric
+ * points, log records) names the same run and host: a backend joins them on
+ * `service.instance.id`. Under `OTEL_RESOURCE_ATTRIBUTES`, which wins.
+ */
+export function runResource(run: RunContextRecord): Record<string, string> {
+  const r: Record<string, string> = {
+    [SEMCONV.serviceInstanceId]: run.runId,
+    [SEMCONV.pipelineRunId]: run.runId,
+    [SEMCONV.osType]: run.os === 'win32' ? 'windows' : run.os,
+    [SEMCONV.hostArch]: run.arch === 'x64' ? 'amd64' : run.arch,
+  }
+  if (run.host !== null) r[SEMCONV.hostName] = run.host
+  if (run.commitSha !== null) r[SEMCONV.vcsHeadRevision] = run.commitSha
+  if (run.branch !== null) r[SEMCONV.vcsHeadName] = run.branch
+  return r
+}
+
 /**
  * Attributes for the root `vx.run` span.
  *
@@ -253,6 +376,8 @@ interface TaskSpanRunContext {
   workspaceId: string
   /** The run's canonical start (epoch ms) — the storage key's base. */
   startedAt: number
+  /** The task's command, from its start record; a task that never started has none. */
+  command?: string
 }
 
 /**
@@ -283,8 +408,8 @@ function runResult(t: TaskTelemetry): string {
  * `started_at` from them is computing a dedup key that must match the value
  * the native ingest path derives, to the millisecond.
  *
- * Nothing here is unbounded: every value is an id, an enum or a number, so a
- * collector's attribute-value limit cannot cut one. The largest thing this
+ * Nothing here is unbounded: every value is an id, an enum, a number or the
+ * task's command line, so a collector's attribute-value limit cannot cut one. The largest thing this
  * exporter ships is a task's captured tail, which travels as a LOG record
  * (`vx.log.*`) carrying its own full length, so a cut there is visible.
  */
@@ -325,6 +450,20 @@ export function taskSpanAttributes(t: TaskTelemetry, run: TaskSpanRunContext): K
   if (t.outputs !== undefined) attrs.push(strAttr(VX_ATTR.taskOutputs, t.outputs))
   if (t.peakRssBytes !== undefined) attrs.push(intAttr(VX_ATTR.peakRssBytes, t.peakRssBytes))
   if (t.attempts !== undefined) attrs.push(intAttr(VX_ATTR.taskAttempts, t.attempts))
+  if (run.command !== undefined) attrs.push(strAttr(VX_ATTR.taskCommand, run.command))
+  if (t.flaky !== undefined) {
+    attrs.push(
+      intAttr(VX_ATTR.taskFlakyPasses, t.flaky.passes),
+      intAttr(VX_ATTR.taskFlakyFailures, t.flaky.failures),
+    )
+  }
+  if (t.storedDurationMs !== undefined)
+    attrs.push(intAttr(VX_ATTR.storedDurationMs, t.storedDurationMs))
+  if (t.storedCpuMs !== undefined) attrs.push(intAttr(VX_ATTR.storedCpuMs, t.storedCpuMs))
+  if (t.storedPeakRssBytes !== undefined)
+    attrs.push(intAttr(VX_ATTR.storedPeakRssBytes, t.storedPeakRssBytes))
+  if (t.admissionHeldMs !== undefined)
+    attrs.push(intAttr(VX_ATTR.taskAdmissionHeldMs, t.admissionHeldMs))
   if (t.wallclockStartNs !== undefined)
     attrs.push(int64Attr(VX_ATTR.wallclockStartNs, t.wallclockStartNs))
   if (t.wallclockEndNs !== undefined)
@@ -411,6 +550,11 @@ export function buildMetricsRequest(
               ]),
               sum('vx.tasks.cache_up_to_date', [point(summary.upToDateCount)]),
               gauge('vx.run.duration_ms', summary.totalDurationMs),
+              // What the run's cache hits skipped: the stored runs' time.
+              gauge(
+                'vx.run.time_saved_ms',
+                summary.tasks.reduce((n, t) => n + (t.storedDurationMs ?? 0), 0),
+              ),
               ...taskMetrics(points),
             ],
           },
@@ -424,7 +568,7 @@ export function buildMetricsRequest(
  * A per-task metric point: a task's totals at its end, or one live sample
  * of its process tree while it runs.
  */
-export type TaskMetricPoint =
+export type TaskMetricPoint = (
   | { kind: 'end'; task: TaskTelemetry; startUnixNano: string; endUnixNano: string }
   | {
       kind: 'sample'
@@ -436,6 +580,10 @@ export type TaskMetricPoint =
       cpuUsage: number
       rssBytes: number
     }
+) & {
+  /** The task's span: each point carries it as an exemplar, a link from the chart to the trace. */
+  span?: { traceId: string; spanId: string }
+}
 
 /**
  * An ExportMetricsServiceRequest of per-task gauges alone: the points past
@@ -460,9 +608,11 @@ export function buildTaskMetricsRequest(
 /**
  * Per-task gauges, keyed by task. At each task's end: its duration, CPU time and peak memory (CPU and memory as
  * the runner measured them at exit; a task it did not measure, a cache hit or
- * a remote run, sends no point for them). While a task runs: its process
+ * a remote run, sends no point for them), on a hit the time it saved, and
+ * how long an `admit` policy held it. While a task runs: its process
  * tree's CPU usage and memory, once per sample. A task's numbers otherwise
- * live only on its span, which no metrics backend charts.
+ * live only on its span, which no metrics backend charts. Each point names
+ * the task's span as an exemplar.
  */
 function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
   const series = new Map<string, { unit: string; dataPoints: unknown[] }>()
@@ -476,12 +626,17 @@ function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
     strAttr(VX_ATTR.taskProject, project),
     strAttr(VX_ATTR.taskTask, task),
   ]
+  const exemplars = (p: TaskMetricPoint, value: number, timeUnixNano: string) =>
+    p.span === undefined
+      ? {}
+      : { exemplars: [{ timeUnixNano, asDouble: value, filteredAttributes: [], ...p.span }] }
   for (const p of points) {
     if (p.kind === 'sample') {
       const at = (value: number) => ({
         asDouble: value,
         timeUnixNano: p.timeUnixNano,
         attributes: taskAttrs(p.taskId, p.project, p.task),
+        ...exemplars(p, value, p.timeUnixNano),
       })
       add('vx.task.cpu_usage', '1', at(p.cpuUsage))
       add('vx.task.memory', 'By', at(p.rssBytes))
@@ -496,10 +651,13 @@ function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
         ...taskAttrs(t.taskId, t.project, t.task),
         strAttr(VX_ATTR.cacheSource, t.cacheSource),
       ],
+      ...exemplars(p, value, p.endUnixNano),
     })
     add('vx.task.duration', 'ms', at(t.durationMs))
     if (t.cpuMs !== undefined) add('vx.task.cpu_time', 'ms', at(t.cpuMs))
     if (t.peakRssBytes !== undefined) add('vx.task.peak_memory', 'By', at(t.peakRssBytes))
+    if (t.storedDurationMs !== undefined) add('vx.task.time_saved', 'ms', at(t.storedDurationMs))
+    if (t.admissionHeldMs !== undefined) add('vx.task.admission_held', 'ms', at(t.admissionHeldMs))
   }
   return [...series].map(([name, { unit, dataPoints }]) => ({ name, unit, gauge: { dataPoints } }))
 }
@@ -545,6 +703,8 @@ export function buildLogsRequest(args: {
   workspaceId: string
   entries: readonly TaskLogEntry[]
   timeUnixNano: string
+  /** When the task ended; `timeUnixNano` for one with no known end. */
+  timeFor?: (taskId: string) => string | undefined
   traceId?: string
   spanIdFor?: (taskId: string) => string | undefined
   resource?: Readonly<Record<string, string>>
@@ -561,8 +721,9 @@ export function buildLogsRequest(args: {
     ]
     if (e.hash !== undefined) attrs.push(strAttr(VX_ATTR.taskHash, e.hash))
     const spanId = args.spanIdFor?.(e.taskId)
+    const time = args.timeFor?.(e.taskId) ?? args.timeUnixNano
     return {
-      timeUnixNano: args.timeUnixNano,
+      timeUnixNano: time,
       observedTimeUnixNano: args.timeUnixNano,
       severityNumber: failed ? SEVERITY_ERROR : SEVERITY_INFO,
       severityText: failed ? 'ERROR' : 'INFO',
@@ -583,3 +744,53 @@ export function buildLogsRequest(args: {
     ],
   }
 }
+
+/** A lifecycle moment, sent as it happens in live mode: a run or a task started. */
+export interface LiveEvent {
+  name: 'vx.run.start' | 'vx.task.start'
+  timeUnixNano: string
+  body: string
+  taskId?: string
+  spanId?: string
+}
+
+/**
+ * Live mode's lifecycle records, one log record each, linked to the run's
+ * trace (and the task's span): what a dashboard shows as running before any
+ * span, which is sent only once it ends, exists.
+ */
+export function buildEventLogsRequest(args: {
+  serviceName: string
+  vxVersion: string
+  runId: string
+  workspaceId: string
+  events: readonly LiveEvent[]
+  traceId?: string
+  resource?: Readonly<Record<string, string>>
+}): unknown {
+  const logRecords: OtlpLogRecord[] = args.events.map((e) => ({
+    timeUnixNano: e.timeUnixNano,
+    observedTimeUnixNano: e.timeUnixNano,
+    severityNumber: SEVERITY_INFO,
+    severityText: 'INFO',
+    body: { stringValue: e.body },
+    attributes: [
+      strAttr(SEMCONV.eventName, e.name),
+      strAttr(SEMCONV.pipelineRunId, args.runId),
+      strAttr(VX_ATTR.workspaceId, args.workspaceId),
+      ...(e.taskId !== undefined ? [strAttr(SEMCONV.taskName, e.taskId)] : []),
+    ],
+    ...(args.traceId ? { traceId: args.traceId } : {}),
+    ...(args.traceId && e.spanId ? { spanId: e.spanId } : {}),
+  }))
+  return {
+    resourceLogs: [
+      {
+        resource: {
+          attributes: resourceAttributes(args.serviceName, args.vxVersion, args.resource),
+        },
+        scopeLogs: [{ scope: { name: 'vx', version: args.vxVersion }, logRecords }],
+      },
+    ],
+  }
+}

package/src/plugin.ts

@@ -17,7 +17,7 @@ import {
   type VxPlugin,
 } from '@vzn/vx'
 import { readFileSync } from 'node:fs'
-import { OtelSink, type OtlpTls, type PostFn } from './sink.js'
+import type { OtelSinkConfig, OtlpTls, PostFn } from './sink.js'
 
 export interface OtelPluginOptions {
   /** OTLP base endpoint. Falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`. */
@@ -40,6 +40,12 @@ export interface OtelPluginOptions {
    * Set false (or `OTEL_LOGS_EXPORTER=none`) to export traces + metrics only.
    */
   logs?: boolean
+  /**
+   * Send each task's span, metrics and log as it ends, and a log record as
+   * the run and each task start, instead of everything at the run's end: a
+   * dashboard follows the run while it runs. Default: false.
+   */
+  live?: boolean
   /** Per-request timeout (ms). Falls back to `OTEL_EXPORTER_OTLP_TIMEOUT`, else 15000. */
   timeoutMs?: number
   /**
@@ -166,7 +172,7 @@ export function resolveOtelConfig(
   opts: OtelPluginOptions,
   env: Record<string, string | undefined>,
   warn?: (m: string) => void,
-): ConstructorParameters<typeof OtelSink>[0] | undefined {
+): OtelSinkConfig | undefined {
   const base = present(opts.endpoint) ?? present(env['OTEL_EXPORTER_OTLP_ENDPOINT'])
   const tracesUrl =
     present(opts.tracesEndpoint) ??
@@ -354,6 +360,7 @@ export function resolveOtelConfig(
     ...(tracesWanted && tracesUrl !== undefined ? {} : { tracesEnabled: false }),
     metricsEnabled: metricsWanted && metricsUrl !== undefined,
     logsEnabled: logsWanted && logsUrl !== undefined,
+    ...(opts.live === true ? { live: true } : {}),
     timeoutMs,
     // A signal's own `OTEL_EXPORTER_OTLP_<SIGNAL>_TIMEOUT` wins over the
     // shared one; it was not read (F-49). The option tops both.
@@ -377,6 +384,7 @@ const OTEL_PLUGIN_KEYS: PluginOptionKinds<OtelPluginOptions> = {
   headers: 'object',
   metrics: 'boolean',
   logs: 'boolean',
+  live: 'boolean',
   timeoutMs: 'number',
   compression: 'string',
   post: 'function',
@@ -392,10 +400,12 @@ const OTEL_PLUGIN_KEYS: PluginOptionKinds<OtelPluginOptions> = {
 export function otel(opts: OtelPluginOptions = {}): VxPlugin {
   refuseUnknownOptions('otel()', opts, OTEL_PLUGIN_KEYS)
   return definePlugin(import.meta, {
-    telemetry(ctx: TelemetryContext): TelemetrySink | undefined {
+    telemetry(ctx: TelemetryContext): Promise<TelemetrySink> | undefined {
       const config = resolveOtelConfig(opts, process.env, (m) => ctx.warn(m))
       if (config === undefined) return undefined
-      return new OtelSink(config)
+      // The exporter loads only when it will export: a declined otel() is
+      // read by every run, and the sink was most of its import cost.
+      return import('./sink.js').then(({ OtelSink }) => new OtelSink(config))
     },
   })
 }

package/src/sink.ts

@@ -1,10 +1,10 @@
 // The OTLP telemetry sink — accumulates a run's records into OTLP trace +
-// metric payloads and ships them over HTTP/JSON. Observe-only: it implements
+// metric payloads and ships them over HTTP/JSON, at the run's end or, live,
+// as each task ends. Observe-only: it implements
 // core's TelemetrySink (records in, nothing out). Never-fail: every network
 // error is swallowed, the POST is time-bounded, and the upload is idempotent —
 // a down collector can never affect a run.
 
-import { randomBytes } from 'node:crypto'
 import { TaskLogBuffer } from '@vzn/vx'
 import type {
   RunContextRecord,
@@ -13,15 +13,22 @@ import type {
   TelemetryRecord,
   TelemetrySink,
 } from '@vzn/vx'
+import type { TaskLogEntry } from '@vzn/vx'
 import {
+  buildEventLogsRequest,
   buildLogsRequest,
   buildMetricsRequest,
   buildTaskMetricsRequest,
   buildTraceRequest,
+  type LiveEvent,
   type OtlpSpan,
   type TaskMetricPoint,
+  runResource,
   runSpanAttributes,
   SPAN_KIND_INTERNAL,
+  dependencyLink,
+  stageSpan,
+  taskSpanEvents,
   runStatusCode,
   taskSpanAttributes,
   taskStatusCode,
@@ -62,6 +69,8 @@ export interface OtelSinkConfig {
   gzip?: readonly OtelSignal[]
   /** Per signal: a collector's CA, and a client certificate for mutual TLS. */
   tls?: Partial<Record<OtelSignal, OtlpTls>>
+  /** Send each task's span, metrics and log as it ends, not all at the run's end. */
+  live?: boolean
   /** False under `OTEL_TRACES_EXPORTER=none`; absent is true. */
   tracesEnabled?: boolean
   metricsEnabled: boolean
@@ -247,7 +256,7 @@ function partialSuccess(body: string): string | undefined {
 }
 
 function genId(bytes: number): string {
-  return randomBytes(bytes).toString('hex')
+  return Buffer.from(crypto.getRandomValues(new Uint8Array(bytes))).toString('hex')
 }
 
 /** ms → unix-nano decimal string (OTLP int64-as-string). */
@@ -276,6 +285,16 @@ export class OtelSink implements TelemetrySink {
   private readonly lastSample = new Map<string, { ts: number; cpuMs: number }>()
   private readonly taskSpanId = new Map<string, string>()
   private readonly taskStartNano = new Map<string, string>()
+  private readonly taskDeps = new Map<string, readonly string[]>()
+  private readonly taskCommand = new Map<string, string>()
+  private readonly taskEndNano = new Map<string, string>()
+  /** Resource attributes: the run's, under `OTEL_RESOURCE_ATTRIBUTES`. */
+  private resource: Readonly<Record<string, string>> = {}
+  // Live mode: what ended since the last send; one send in flight at a time.
+  private readonly liveLogs: TaskLogEntry[] = []
+  private readonly liveEvents: LiveEvent[] = []
+  private pumping: Promise<void> | undefined
+  private readonly warnedLive = new Set<OtelSignal>()
   private summary: RunSummaryRecord | undefined
   private uploaded = false
   // Core's own bounded capture buffer, so which task's output survives a
@@ -298,6 +317,7 @@ export class OtelSink implements TelemetrySink {
       tls: config.tls ?? {},
       headers: config.headers,
       signalHeaders: config.signalHeaders ?? {},
+      live: config.live === true,
       tracesEnabled: config.tracesEnabled !== false,
       metricsEnabled: config.metricsEnabled,
       logsEnabled: config.logsEnabled,
@@ -327,16 +347,34 @@ export class OtelSink implements TelemetrySink {
         // it is what the summary reports and what a receiver stores.
         this.rootStartNano = nanos(record.startedAt)
         this.runStartedAt = record.startedAt
+        this.resource = { ...runResource(record.run), ...this.cfg.resource }
+        this.liveEvent({
+          name: 'vx.run.start',
+          timeUnixNano: this.rootStartNano,
+          body: `${record.run.command}: ${record.total} tasks`,
+        })
         return
-      case 'task.start':
-        this.taskSpanId.set(record.taskId, genId(8))
+      case 'task.start': {
+        const spanId = genId(8)
+        this.taskSpanId.set(record.taskId, spanId)
         this.taskStartNano.set(record.taskId, nanos(record.ts))
+        if (record.command !== undefined) this.taskCommand.set(record.taskId, record.command)
+        if (record.dependsOn !== undefined) this.taskDeps.set(record.taskId, record.dependsOn)
+        this.liveEvent({
+          name: 'vx.task.start',
+          timeUnixNano: nanos(record.ts),
+          body:
+            record.command === undefined ? record.taskId : `${record.taskId}: ${record.command}`,
+          taskId: record.taskId,
+          spanId,
+        })
         this.taskNames.set(record.taskId, {
           project: record.project,
           task: record.task,
           startMs: record.ts,
         })
         return
+      }
       case 'task.sample': {
         // Usage is the CPU the tree spent since the last look, over the
         // wall time between: the first look counts from the task's start.
@@ -355,14 +393,27 @@ export class OtelSink implements TelemetrySink {
           timeUnixNano: nanos(record.ts),
           cpuUsage,
           rssBytes: record.rssBytes,
+          ...this.spanOf(record.taskId),
         })
+        this.kick()
         return
       }
       case 'task.log':
         this.logs.append(record.taskId, record.chunk)
         return
       case 'task.end': {
-        const spanId = this.taskSpanId.get(record.taskId) ?? genId(8)
+        // A task that never started (a skip) still gets a span, and a later
+        // task that waited on it links to it.
+        let spanId = this.taskSpanId.get(record.taskId)
+        if (spanId === undefined) {
+          spanId = genId(8)
+          this.taskSpanId.set(record.taskId, spanId)
+        }
+        const links = (this.taskDeps.get(record.taskId) ?? []).flatMap((dep) => {
+          const depSpan = this.taskSpanId.get(dep)
+          return depSpan === undefined ? [] : [dependencyLink(this.traceId, depSpan, dep)]
+        })
+        const events = taskSpanEvents(record, nanos(record.ts))
         const startNano =
           this.taskStartNano.get(record.taskId) ?? nanos(record.ts - record.durationMs)
         const t: TaskTelemetry = record
@@ -378,8 +429,13 @@ export class OtelSink implements TelemetrySink {
             runId: this.runId,
             workspaceId: this.run?.workspaceId ?? '',
             startedAt: this.runStartedAt,
+            ...(this.taskCommand.has(record.taskId)
+              ? { command: this.taskCommand.get(record.taskId)! }
+              : {}),
           }),
           status: { code: taskStatusCode(t) },
+          ...(events.length > 0 ? { events } : {}),
+          ...(links.length > 0 ? { links } : {}),
         })
         // A skipped task never ran: a zero duration would read as a fast one.
         if (record.status !== 'skipped') {
@@ -388,14 +444,21 @@ export class OtelSink implements TelemetrySink {
             task: t,
             startUnixNano: startNano,
             endUnixNano: nanos(record.ts),
+            ...this.spanOf(record.taskId),
           })
         }
         this.lastSample.delete(record.taskId)
+        this.taskEndNano.set(record.taskId, nanos(record.ts))
         // Decides retention: a cache hit's bytes belong to the run that
         // executed them, so only an executed success/failure keeps a tail.
         if (this.cfg.logsEnabled) {
           this.logs.finish(record.taskId, record.status, record.cacheSource, record.hash)
+          if (this.cfg.live) {
+            const entry = this.logs.takeEntry(record.taskId)
+            if (entry !== undefined) this.liveLogs.push(entry)
+          }
         }
+        this.kick()
         return
       }
       case 'run.end':
@@ -407,6 +470,79 @@ export class OtelSink implements TelemetrySink {
   /** Core's flush deadline, passed to every POST. */
   private deadline: AbortSignal | undefined
 
+  /** The task's span, for its metric points' exemplars; none with traces off. */
+  private spanOf(taskId: string): { span?: { traceId: string; spanId: string } } {
+    const spanId = this.taskSpanId.get(taskId)
+    return this.cfg.tracesEnabled && this.traceId && spanId !== undefined
+      ? { span: { traceId: this.traceId, spanId } }
+      : {}
+  }
+
+  private liveEvent(event: LiveEvent): void {
+    if (this.cfg.live && this.cfg.logsEnabled) this.liveEvents.push(event)
+  }
+
+  /** Live mode: send what is waiting, unless a send is already on its way (it takes it next). */
+  private kick(): void {
+    if (!this.cfg.live || this.pumping !== undefined || this.uploaded) return
+    this.pumping = Promise.resolve()
+      .then(() => this.pump())
+      .finally(() => {
+        this.pumping = undefined
+      })
+  }
+
+  private async pump(): Promise<void> {
+    while (
+      this.spans.length + this.taskPoints.length + this.liveLogs.length + this.liveEvents.length >
+      0
+    ) {
+      const spans = this.spans.splice(0)
+      const points = this.taskPoints.splice(0)
+      const logs = this.liveLogs.splice(0)
+      const events = this.liveEvents.splice(0)
+      await Promise.all([
+        this.shipSpans(spans, true),
+        this.shipPoints(points),
+        this.shipLiveLogs(logs, events),
+      ])
+    }
+  }
+
+  private async shipPoints(points: TaskMetricPoint[]): Promise<void> {
+    if (!this.cfg.metricsEnabled || points.length === 0) return
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const bodies = requestBodies(points, (group) =>
+      JSON.stringify(
+        buildTaskMetricsRequest(this.cfg.serviceName, vxVersion, group, this.resource),
+      ),
+    )
+    await this.send('metrics', this.cfg.metricsUrl, bodies, true)
+  }
+
+  private async shipLiveLogs(logs: TaskLogEntry[], events: LiveEvent[]): Promise<void> {
+    if (!this.cfg.logsEnabled) return
+    const bodies = [
+      ...(logs.length > 0 ? this.logBodies(logs) : []),
+      ...(events.length > 0
+        ? requestBodies(events, (group) =>
+            JSON.stringify(
+              buildEventLogsRequest({
+                serviceName: this.cfg.serviceName,
+                vxVersion: this.run?.vxVersion ?? '0.0.0',
+                runId: this.runId,
+                workspaceId: this.run?.workspaceId ?? '',
+                events: group,
+                resource: this.resource,
+                ...(this.cfg.tracesEnabled && this.traceId ? { traceId: this.traceId } : {}),
+              }),
+            ),
+          )
+        : []),
+    ]
+    if (bodies.length > 0) await this.send('logs', this.cfg.logsUrl, bodies, true)
+  }
+
   onRunSummary(summary: RunSummaryRecord): void {
     this.summary = summary
   }
@@ -415,6 +551,11 @@ export class OtelSink implements TelemetrySink {
     if (this.uploaded) return
     this.uploaded = true
     this.deadline = signal
+    // Live: what is on its way lands before the rest, which the run's end sends.
+    if (this.cfg.live) {
+      await this.pumping
+      await this.pump()
+    }
     // Finalize the root span now that the run is over (run.end set the end).
     if (this.run !== undefined && this.traceId) {
       // Prefer the summary's own start/end: they are the run's canonical
@@ -437,17 +578,25 @@ export class OtelSink implements TelemetrySink {
         attributes: runSpanAttributes(this.run, this.summary),
         status: { code: runStatusCode(this.summary) },
       })
+      const stages = (this.summary?.stages ?? []).map((stage) =>
+        stageSpan(this.traceId, this.rootSpanId, stage, genId(8)),
+      )
+      this.spans.splice(1, 0, ...stages)
     }
-    const vxVersion = this.run?.vxVersion ?? '0.0.0'
-    await Promise.all([this.shipTraces(vxVersion), this.shipMetrics(), this.shipLogs(vxVersion)])
+    await Promise.all([this.shipTraces(), this.shipMetrics(), this.shipLogs()])
   }
 
-  private async shipTraces(vxVersion: string): Promise<void> {
-    if (this.cfg.tracesEnabled === false || this.spans.length === 0) return
-    const bodies = requestBodies(this.spans, (spans) =>
-      JSON.stringify(buildTraceRequest(this.cfg.serviceName, vxVersion, spans, this.cfg.resource)),
+  private async shipTraces(): Promise<void> {
+    await this.shipSpans(this.spans, false)
+  }
+
+  private async shipSpans(spans: OtlpSpan[], live: boolean): Promise<void> {
+    if (this.cfg.tracesEnabled === false || spans.length === 0) return
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const bodies = requestBodies(spans, (group) =>
+      JSON.stringify(buildTraceRequest(this.cfg.serviceName, vxVersion, group, this.resource)),
     )
-    await this.send('traces', this.cfg.tracesUrl, bodies)
+    await this.send('traces', this.cfg.tracesUrl, bodies, live)
   }
 
   private async shipMetrics(): Promise<void> {
@@ -463,12 +612,7 @@ export class OtelSink implements TelemetrySink {
             points[0] === first
               ? this.runMetricsBody(summary, points)
               : JSON.stringify(
-                  buildTaskMetricsRequest(
-                    this.cfg.serviceName,
-                    vxVersion,
-                    points,
-                    this.cfg.resource,
-                  ),
+                  buildTaskMetricsRequest(this.cfg.serviceName, vxVersion, points, this.resource),
                 ),
           )
     await this.send('metrics', this.cfg.metricsUrl, bodies)
@@ -481,27 +625,32 @@ export class OtelSink implements TelemetrySink {
         summary,
         nanos(summary.endedAt),
         nanos(summary.startedAt),
-        this.cfg.resource,
+        this.resource,
         points,
       ),
     )
   }
 
-  private async shipLogs(vxVersion: string): Promise<void> {
+  private async shipLogs(): Promise<void> {
     if (!this.cfg.logsEnabled) return
-    const workspaceId = this.run?.workspaceId ?? ''
-    const bundle = this.logs.drain(this.runId, workspaceId)
+    const bundle = this.logs.drain(this.runId, this.run?.workspaceId ?? '')
     if (bundle.tasks.length === 0) return
-    const bodies = requestBodies(bundle.tasks, (entries) =>
+    await this.send('logs', this.cfg.logsUrl, this.logBodies(bundle.tasks))
+  }
+
+  private logBodies(tasks: readonly TaskLogEntry[]): string[] {
+    const now = nanos(this.summary?.endedAt ?? Date.now())
+    return requestBodies(tasks, (entries) =>
       JSON.stringify(
         buildLogsRequest({
           serviceName: this.cfg.serviceName,
-          resource: this.cfg.resource,
-          vxVersion,
+          resource: this.resource,
+          vxVersion: this.run?.vxVersion ?? '0.0.0',
           runId: this.runId,
-          workspaceId,
+          workspaceId: this.run?.workspaceId ?? '',
           entries,
-          timeUnixNano: nanos(this.summary?.endedAt ?? Date.now()),
+          timeUnixNano: now,
+          timeFor: (taskId) => this.taskEndNano.get(taskId),
           // Traces off: the trace and its spans are never exported, and a
           // record naming them links nowhere (F-49).
           ...(this.cfg.tracesEnabled && this.traceId
@@ -510,7 +659,6 @@ export class OtelSink implements TelemetrySink {
         }),
       ),
     )
-    await this.send('logs', this.cfg.logsUrl, bodies)
   }
 
   /** The transport, under the signal's own timeout when it has one (F-49). */
@@ -519,8 +667,16 @@ export class OtelSink implements TelemetrySink {
     return own === undefined || this.injected ? this.cfg.post : defaultPost(own)
   }
 
-  /** POSTs every body at once; one warning per signal, however many fail. */
-  private async send(signal: OtelSignal, url: string, bodies: readonly string[]): Promise<void> {
+  /**
+   * POSTs every body at once; one warning per signal, however many fail, and
+   * live sends (`once`) one per signal for the whole run.
+   */
+  private async send(
+    signal: OtelSignal,
+    url: string,
+    bodies: readonly string[],
+    once = false,
+  ): Promise<void> {
     const gzip = this.cfg.gzip.includes(signal)
     const headers: Record<string, string> = {
       'content-type': 'application/json',
@@ -545,6 +701,10 @@ export class OtelSink implements TelemetrySink {
     ).filter((m) => m !== undefined)
     // export is fully optional — a down collector never affects a run
     if (failed.length === 0) return
+    if (once) {
+      if (this.warnedLive.has(signal)) return
+      this.warnedLive.add(signal)
+    }
     // Name the URL: three signals ship concurrently and each is caught
     // here on its own, so a bare "export failed" cannot tell a down
     // collector from one misconfigured signal endpoint.