@vzn/vx-otel 0.0.520 → 0.0.521

package/README.md

@@ -103,13 +103,20 @@ otel({
   `vx.task.hash`, duration, CPU ms, peak RSS, and on a skipped task its root
   blocker (`vx.task.blocked_by`), on a timed-out one `vx.task.timed_out`, on
   a sandboxed one its violation count (`vx.task.sandbox_violations`), on a
-  persistent one that never became ready why (`vx.task.not_ready`). A
+  persistent one that never became ready why (`vx.task.not_ready`), its
+  command (`vx.task.command`, secrets masked), on a flaky one its record
+  (`vx.task.flaky.passes`, `vx.task.flaky.failures`), on a hit what the
+  stored run took (`vx.cache.stored_duration_ms`, `vx.cache.stored_cpu_ms`,
+  `vx.cache.stored_peak_rss_bytes`), and how long an `admit` policy held it
+  (`vx.task.admission_held_ms`). A
   failed task sets span status
   `ERROR`. A task span links to the spans of the tasks it waited on (a
   group seen through to the tasks behind it), and carries an event per
   attempt that failed and was run again (`vx.task.retry`, with
-  `vx.task.attempt` and `vx.task.exit_code`) and one when vx's own
-  timeout killed it (`vx.task.timeout`);
+  `vx.task.attempt` and `vx.task.exit_code`), one when vx's own
+  timeout killed it (`vx.task.timeout`), and one per sandbox violation
+  (`vx.sandbox.violation`, its line; past 100, `vx.sandbox.violations_dropped`
+  says how many more);
 - a child span per stage of the run, named by the stage (`startup`,
   `load configs`, `git enumeration`, `classify + probe`, `run graph`,
   `record history`, …, the stages `VX_TIMING` prints), with
@@ -120,7 +127,8 @@ otel({
 **Metrics per run**: `vx.tasks.total`, `vx.tasks.failed`,
 `vx.tasks.cache_hits{source=local|remote}`, what those hits did to the disk as
 `vx.tasks.cache_restored{source=local|remote}` and `vx.tasks.cache_up_to_date`,
-and the `vx.run.duration_ms` gauge. The `vx.run` span carries the same counts
+and the `vx.run.duration_ms` and `vx.run.time_saved_ms` (what the hits'
+stored runs took) gauges. The `vx.run` span carries the same counts
 (`vx.run.hit_count`, `vx.run.up_to_date_count`, `vx.run.restored_local_count`,
 `vx.run.restored_remote_count`).
 The counts are DELTA sums over the run's own interval (start to end), so a
@@ -129,18 +137,42 @@ backend adds runs rather than reading each as the series' new total.
 **Metrics per task**, as gauges keyed by `cicd.pipeline.task.name`,
 `vx.task.project` and `vx.task.task`: at each task's end
 `vx.task.duration` (ms, with `vx.cache.source`), and for a task the runner
-measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes); a
-skipped task sends none. While a task runs, its process tree is sampled
+measured `vx.task.cpu_time` (ms) and `vx.task.peak_memory` (bytes), on a
+hit `vx.task.time_saved` (ms), and `vx.task.admission_held` (ms) when an
+`admit` policy held it; a skipped task sends none. While a task runs, its process tree is sampled
 each second: `vx.task.cpu_usage` (cores busy since the last sample, 1 =
 one core) and `vx.task.memory` (resident bytes). Sampling runs only while
 metrics export; a remote task, or a task under 1 s, has no samples.
 
 **Logs per run** (on by default): the captured output tail of each executed
-task, as one log record linked to its task span (unlinked when traces are
-off: the span is never exported). Build output can hold
+task, as one log record at the task's end, linked to its task span (unlinked
+when traces are off: the span is never exported). Build output can hold
 secrets, and this sends it to the collector; `logs: false` or
 `OTEL_LOGS_EXPORTER=none` turns it off.
 
+**One run, three signals, linked.** Every request's resource names the run
+(`service.instance.id` and `cicd.pipeline.run.id`, its id), the host
+(`host.name`, `host.arch`, `os.type`) and the commit (`vcs.ref.head.*`),
+under `OTEL_RESOURCE_ATTRIBUTES`, which wins. Each task metric point carries
+its task's span as an exemplar, and each log record its span, so a chart
+opens the trace and a span opens its output.
+
+## Live export
+
+```ts
+otel({ live: process.env.CI === 'true' })
+```
+
+With `live: true` each task's span, metrics and output tail are sent as the
+task ends, its process samples as they are taken, and a log record
+(`event.name` `vx.run.start` or `vx.task.start`, linked to the trace) as the
+run and each task start; the `vx.run` span, the stage spans and the run's
+metrics follow at the end. A dashboard follows a CI run while it runs. One
+send is in flight at a time and takes everything that ended meanwhile; a
+refused send warns once per signal for the run. Cost on 60 tasks against a
+local collector: within the noise of run-to-run wall time (min of 6: 1,101 ms
+off, 1,108 ms on). Off by default.
+
 ## Behavior note
 
 This replaces core's previous hardcoded OTel emit, which fired automatically

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-otel",
-  "version": "0.0.520",
+  "version": "0.0.521",
   "license": "MIT",
   "description": "OpenTelemetry exporter for vx — a telemetry plugin mapping runs to OTLP traces + metrics (CI/CD + VCS semantic conventions).",
   "keywords": [
@@ -34,7 +34,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.520"
+    "@vzn/vx": "^0.0.521"
   },
   "repository": {
     "type": "git",

package/src/otlp.ts

@@ -57,6 +57,11 @@ const SEMCONV = {
   vcsHeadName: 'vcs.ref.head.name',
   serviceName: 'service.name',
   serviceVersion: 'service.version',
+  serviceInstanceId: 'service.instance.id',
+  hostName: 'host.name',
+  hostArch: 'host.arch',
+  osType: 'os.type',
+  eventName: 'event.name',
 } as const
 
 /**
@@ -127,6 +132,15 @@ const VX_ATTR = {
   peakRssBytes: 'vx.peak_rss_bytes',
   taskAttempts: 'vx.task.attempts',
   taskAttempt: 'vx.task.attempt',
+  taskCommand: 'vx.task.command',
+  taskFlakyPasses: 'vx.task.flaky.passes',
+  taskFlakyFailures: 'vx.task.flaky.failures',
+  taskAdmissionHeldMs: 'vx.task.admission_held_ms',
+  storedDurationMs: 'vx.cache.stored_duration_ms',
+  storedCpuMs: 'vx.cache.stored_cpu_ms',
+  storedPeakRssBytes: 'vx.cache.stored_peak_rss_bytes',
+  sandboxViolation: 'vx.sandbox.violation',
+  sandboxViolationsDropped: 'vx.sandbox.violations_dropped',
   stageName: 'vx.stage.name',
   wallclockStartNs: 'vx.task.wallclock_start_ns',
   wallclockEndNs: 'vx.task.wallclock_end_ns',
@@ -171,8 +185,9 @@ export interface OtlpSpanLink {
 
 /**
  * A task span's events: each attempt that failed and was run again
- * (`vx.task.retry`, at its end), and vx's own deadline killing the task
- * (`vx.task.timeout`, at the span's end).
+ * (`vx.task.retry`, at its end), vx's own deadline killing the task
+ * (`vx.task.timeout`, at the span's end), and each sandbox violation
+ * (`vx.sandbox.violation`, its line).
  */
 export function taskSpanEvents(t: TaskTelemetry, endUnixNano: string): OtlpSpanEvent[] {
   const events: OtlpSpanEvent[] = (t.failedAttempts ?? []).map((a, i) => ({
@@ -187,9 +202,27 @@ export function taskSpanEvents(t: TaskTelemetry, endUnixNano: string): OtlpSpanE
   if (t.timedOut === true) {
     events.push({ timeUnixNano: endUnixNano, name: 'vx.task.timeout', attributes: [] })
   }
+  const lines = t.sandboxViolationLines ?? []
+  for (const line of lines.slice(0, MAX_VIOLATION_EVENTS)) {
+    events.push({
+      timeUnixNano: endUnixNano,
+      name: VX_ATTR.sandboxViolation,
+      attributes: [strAttr(VX_ATTR.sandboxViolation, line)],
+    })
+  }
+  if (lines.length > MAX_VIOLATION_EVENTS) {
+    events.push({
+      timeUnixNano: endUnixNano,
+      name: VX_ATTR.sandboxViolationsDropped,
+      attributes: [intAttr(VX_ATTR.sandboxViolationsDropped, lines.length - MAX_VIOLATION_EVENTS)],
+    })
+  }
   return events
 }
 
+/** A collector keeps 128 events per span by default; the rest say how many they were. */
+const MAX_VIOLATION_EVENTS = 100
+
 /**
  * A child span of the run for one of its stages (`VX_TIMING`'s marks):
  * where the time before the first task and after the last one went. Named
@@ -247,6 +280,24 @@ function resourceAttributes(
   ]
 }
 
+/**
+ * The run as resource attributes, so every signal it sends (spans, metric
+ * points, log records) names the same run and host: a backend joins them on
+ * `service.instance.id`. Under `OTEL_RESOURCE_ATTRIBUTES`, which wins.
+ */
+export function runResource(run: RunContextRecord): Record<string, string> {
+  const r: Record<string, string> = {
+    [SEMCONV.serviceInstanceId]: run.runId,
+    [SEMCONV.pipelineRunId]: run.runId,
+    [SEMCONV.osType]: run.os === 'win32' ? 'windows' : run.os,
+    [SEMCONV.hostArch]: run.arch === 'x64' ? 'amd64' : run.arch,
+  }
+  if (run.host !== null) r[SEMCONV.hostName] = run.host
+  if (run.commitSha !== null) r[SEMCONV.vcsHeadRevision] = run.commitSha
+  if (run.branch !== null) r[SEMCONV.vcsHeadName] = run.branch
+  return r
+}
+
 /**
  * Attributes for the root `vx.run` span.
  *
@@ -325,6 +376,8 @@ interface TaskSpanRunContext {
   workspaceId: string
   /** The run's canonical start (epoch ms) — the storage key's base. */
   startedAt: number
+  /** The task's command, from its start record; a task that never started has none. */
+  command?: string
 }
 
 /**
@@ -355,8 +408,8 @@ function runResult(t: TaskTelemetry): string {
  * `started_at` from them is computing a dedup key that must match the value
  * the native ingest path derives, to the millisecond.
  *
- * Nothing here is unbounded: every value is an id, an enum or a number, so a
- * collector's attribute-value limit cannot cut one. The largest thing this
+ * Nothing here is unbounded: every value is an id, an enum, a number or the
+ * task's command line, so a collector's attribute-value limit cannot cut one. The largest thing this
  * exporter ships is a task's captured tail, which travels as a LOG record
  * (`vx.log.*`) carrying its own full length, so a cut there is visible.
  */
@@ -397,6 +450,20 @@ export function taskSpanAttributes(t: TaskTelemetry, run: TaskSpanRunContext): K
   if (t.outputs !== undefined) attrs.push(strAttr(VX_ATTR.taskOutputs, t.outputs))
   if (t.peakRssBytes !== undefined) attrs.push(intAttr(VX_ATTR.peakRssBytes, t.peakRssBytes))
   if (t.attempts !== undefined) attrs.push(intAttr(VX_ATTR.taskAttempts, t.attempts))
+  if (run.command !== undefined) attrs.push(strAttr(VX_ATTR.taskCommand, run.command))
+  if (t.flaky !== undefined) {
+    attrs.push(
+      intAttr(VX_ATTR.taskFlakyPasses, t.flaky.passes),
+      intAttr(VX_ATTR.taskFlakyFailures, t.flaky.failures),
+    )
+  }
+  if (t.storedDurationMs !== undefined)
+    attrs.push(intAttr(VX_ATTR.storedDurationMs, t.storedDurationMs))
+  if (t.storedCpuMs !== undefined) attrs.push(intAttr(VX_ATTR.storedCpuMs, t.storedCpuMs))
+  if (t.storedPeakRssBytes !== undefined)
+    attrs.push(intAttr(VX_ATTR.storedPeakRssBytes, t.storedPeakRssBytes))
+  if (t.admissionHeldMs !== undefined)
+    attrs.push(intAttr(VX_ATTR.taskAdmissionHeldMs, t.admissionHeldMs))
   if (t.wallclockStartNs !== undefined)
     attrs.push(int64Attr(VX_ATTR.wallclockStartNs, t.wallclockStartNs))
   if (t.wallclockEndNs !== undefined)
@@ -483,6 +550,11 @@ export function buildMetricsRequest(
               ]),
               sum('vx.tasks.cache_up_to_date', [point(summary.upToDateCount)]),
               gauge('vx.run.duration_ms', summary.totalDurationMs),
+              // What the run's cache hits skipped: the stored runs' time.
+              gauge(
+                'vx.run.time_saved_ms',
+                summary.tasks.reduce((n, t) => n + (t.storedDurationMs ?? 0), 0),
+              ),
               ...taskMetrics(points),
             ],
           },
@@ -496,7 +568,7 @@ export function buildMetricsRequest(
  * A per-task metric point: a task's totals at its end, or one live sample
  * of its process tree while it runs.
  */
-export type TaskMetricPoint =
+export type TaskMetricPoint = (
   | { kind: 'end'; task: TaskTelemetry; startUnixNano: string; endUnixNano: string }
   | {
       kind: 'sample'
@@ -508,6 +580,10 @@ export type TaskMetricPoint =
       cpuUsage: number
       rssBytes: number
     }
+) & {
+  /** The task's span: each point carries it as an exemplar, a link from the chart to the trace. */
+  span?: { traceId: string; spanId: string }
+}
 
 /**
  * An ExportMetricsServiceRequest of per-task gauges alone: the points past
@@ -532,9 +608,11 @@ export function buildTaskMetricsRequest(
 /**
  * Per-task gauges, keyed by task. At each task's end: its duration, CPU time and peak memory (CPU and memory as
  * the runner measured them at exit; a task it did not measure, a cache hit or
- * a remote run, sends no point for them). While a task runs: its process
+ * a remote run, sends no point for them), on a hit the time it saved, and
+ * how long an `admit` policy held it. While a task runs: its process
  * tree's CPU usage and memory, once per sample. A task's numbers otherwise
- * live only on its span, which no metrics backend charts.
+ * live only on its span, which no metrics backend charts. Each point names
+ * the task's span as an exemplar.
  */
 function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
   const series = new Map<string, { unit: string; dataPoints: unknown[] }>()
@@ -548,12 +626,17 @@ function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
     strAttr(VX_ATTR.taskProject, project),
     strAttr(VX_ATTR.taskTask, task),
   ]
+  const exemplars = (p: TaskMetricPoint, value: number, timeUnixNano: string) =>
+    p.span === undefined
+      ? {}
+      : { exemplars: [{ timeUnixNano, asDouble: value, filteredAttributes: [], ...p.span }] }
   for (const p of points) {
     if (p.kind === 'sample') {
       const at = (value: number) => ({
         asDouble: value,
         timeUnixNano: p.timeUnixNano,
         attributes: taskAttrs(p.taskId, p.project, p.task),
+        ...exemplars(p, value, p.timeUnixNano),
       })
       add('vx.task.cpu_usage', '1', at(p.cpuUsage))
       add('vx.task.memory', 'By', at(p.rssBytes))
@@ -568,10 +651,13 @@ function taskMetrics(points: readonly TaskMetricPoint[]): unknown[] {
         ...taskAttrs(t.taskId, t.project, t.task),
         strAttr(VX_ATTR.cacheSource, t.cacheSource),
       ],
+      ...exemplars(p, value, p.endUnixNano),
     })
     add('vx.task.duration', 'ms', at(t.durationMs))
     if (t.cpuMs !== undefined) add('vx.task.cpu_time', 'ms', at(t.cpuMs))
     if (t.peakRssBytes !== undefined) add('vx.task.peak_memory', 'By', at(t.peakRssBytes))
+    if (t.storedDurationMs !== undefined) add('vx.task.time_saved', 'ms', at(t.storedDurationMs))
+    if (t.admissionHeldMs !== undefined) add('vx.task.admission_held', 'ms', at(t.admissionHeldMs))
   }
   return [...series].map(([name, { unit, dataPoints }]) => ({ name, unit, gauge: { dataPoints } }))
 }
@@ -617,6 +703,8 @@ export function buildLogsRequest(args: {
   workspaceId: string
   entries: readonly TaskLogEntry[]
   timeUnixNano: string
+  /** When the task ended; `timeUnixNano` for one with no known end. */
+  timeFor?: (taskId: string) => string | undefined
   traceId?: string
   spanIdFor?: (taskId: string) => string | undefined
   resource?: Readonly<Record<string, string>>
@@ -633,8 +721,9 @@ export function buildLogsRequest(args: {
     ]
     if (e.hash !== undefined) attrs.push(strAttr(VX_ATTR.taskHash, e.hash))
     const spanId = args.spanIdFor?.(e.taskId)
+    const time = args.timeFor?.(e.taskId) ?? args.timeUnixNano
     return {
-      timeUnixNano: args.timeUnixNano,
+      timeUnixNano: time,
       observedTimeUnixNano: args.timeUnixNano,
       severityNumber: failed ? SEVERITY_ERROR : SEVERITY_INFO,
       severityText: failed ? 'ERROR' : 'INFO',
@@ -655,3 +744,53 @@ export function buildLogsRequest(args: {
     ],
   }
 }
+
+/** A lifecycle moment, sent as it happens in live mode: a run or a task started. */
+export interface LiveEvent {
+  name: 'vx.run.start' | 'vx.task.start'
+  timeUnixNano: string
+  body: string
+  taskId?: string
+  spanId?: string
+}
+
+/**
+ * Live mode's lifecycle records, one log record each, linked to the run's
+ * trace (and the task's span): what a dashboard shows as running before any
+ * span, which is sent only once it ends, exists.
+ */
+export function buildEventLogsRequest(args: {
+  serviceName: string
+  vxVersion: string
+  runId: string
+  workspaceId: string
+  events: readonly LiveEvent[]
+  traceId?: string
+  resource?: Readonly<Record<string, string>>
+}): unknown {
+  const logRecords: OtlpLogRecord[] = args.events.map((e) => ({
+    timeUnixNano: e.timeUnixNano,
+    observedTimeUnixNano: e.timeUnixNano,
+    severityNumber: SEVERITY_INFO,
+    severityText: 'INFO',
+    body: { stringValue: e.body },
+    attributes: [
+      strAttr(SEMCONV.eventName, e.name),
+      strAttr(SEMCONV.pipelineRunId, args.runId),
+      strAttr(VX_ATTR.workspaceId, args.workspaceId),
+      ...(e.taskId !== undefined ? [strAttr(SEMCONV.taskName, e.taskId)] : []),
+    ],
+    ...(args.traceId ? { traceId: args.traceId } : {}),
+    ...(args.traceId && e.spanId ? { spanId: e.spanId } : {}),
+  }))
+  return {
+    resourceLogs: [
+      {
+        resource: {
+          attributes: resourceAttributes(args.serviceName, args.vxVersion, args.resource),
+        },
+        scopeLogs: [{ scope: { name: 'vx', version: args.vxVersion }, logRecords }],
+      },
+    ],
+  }
+}

package/src/plugin.ts

@@ -17,7 +17,7 @@ import {
   type VxPlugin,
 } from '@vzn/vx'
 import { readFileSync } from 'node:fs'
-import { OtelSink, type OtlpTls, type PostFn } from './sink.js'
+import type { OtelSinkConfig, OtlpTls, PostFn } from './sink.js'
 
 export interface OtelPluginOptions {
   /** OTLP base endpoint. Falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`. */
@@ -40,6 +40,12 @@ export interface OtelPluginOptions {
    * Set false (or `OTEL_LOGS_EXPORTER=none`) to export traces + metrics only.
    */
   logs?: boolean
+  /**
+   * Send each task's span, metrics and log as it ends, and a log record as
+   * the run and each task start, instead of everything at the run's end: a
+   * dashboard follows the run while it runs. Default: false.
+   */
+  live?: boolean
   /** Per-request timeout (ms). Falls back to `OTEL_EXPORTER_OTLP_TIMEOUT`, else 15000. */
   timeoutMs?: number
   /**
@@ -166,7 +172,7 @@ export function resolveOtelConfig(
   opts: OtelPluginOptions,
   env: Record<string, string | undefined>,
   warn?: (m: string) => void,
-): ConstructorParameters<typeof OtelSink>[0] | undefined {
+): OtelSinkConfig | undefined {
   const base = present(opts.endpoint) ?? present(env['OTEL_EXPORTER_OTLP_ENDPOINT'])
   const tracesUrl =
     present(opts.tracesEndpoint) ??
@@ -354,6 +360,7 @@ export function resolveOtelConfig(
     ...(tracesWanted && tracesUrl !== undefined ? {} : { tracesEnabled: false }),
     metricsEnabled: metricsWanted && metricsUrl !== undefined,
     logsEnabled: logsWanted && logsUrl !== undefined,
+    ...(opts.live === true ? { live: true } : {}),
     timeoutMs,
     // A signal's own `OTEL_EXPORTER_OTLP_<SIGNAL>_TIMEOUT` wins over the
     // shared one; it was not read (F-49). The option tops both.
@@ -377,6 +384,7 @@ const OTEL_PLUGIN_KEYS: PluginOptionKinds<OtelPluginOptions> = {
   headers: 'object',
   metrics: 'boolean',
   logs: 'boolean',
+  live: 'boolean',
   timeoutMs: 'number',
   compression: 'string',
   post: 'function',
@@ -392,10 +400,12 @@ const OTEL_PLUGIN_KEYS: PluginOptionKinds<OtelPluginOptions> = {
 export function otel(opts: OtelPluginOptions = {}): VxPlugin {
   refuseUnknownOptions('otel()', opts, OTEL_PLUGIN_KEYS)
   return definePlugin(import.meta, {
-    telemetry(ctx: TelemetryContext): TelemetrySink | undefined {
+    telemetry(ctx: TelemetryContext): Promise<TelemetrySink> | undefined {
       const config = resolveOtelConfig(opts, process.env, (m) => ctx.warn(m))
       if (config === undefined) return undefined
-      return new OtelSink(config)
+      // The exporter loads only when it will export: a declined otel() is
+      // read by every run, and the sink was most of its import cost.
+      return import('./sink.js').then(({ OtelSink }) => new OtelSink(config))
     },
   })
 }

package/src/sink.ts

@@ -1,10 +1,10 @@
 // The OTLP telemetry sink — accumulates a run's records into OTLP trace +
-// metric payloads and ships them over HTTP/JSON. Observe-only: it implements
+// metric payloads and ships them over HTTP/JSON, at the run's end or, live,
+// as each task ends. Observe-only: it implements
 // core's TelemetrySink (records in, nothing out). Never-fail: every network
 // error is swallowed, the POST is time-bounded, and the upload is idempotent —
 // a down collector can never affect a run.
 
-import { randomBytes } from 'node:crypto'
 import { TaskLogBuffer } from '@vzn/vx'
 import type {
   RunContextRecord,
@@ -13,13 +13,17 @@ import type {
   TelemetryRecord,
   TelemetrySink,
 } from '@vzn/vx'
+import type { TaskLogEntry } from '@vzn/vx'
 import {
+  buildEventLogsRequest,
   buildLogsRequest,
   buildMetricsRequest,
   buildTaskMetricsRequest,
   buildTraceRequest,
+  type LiveEvent,
   type OtlpSpan,
   type TaskMetricPoint,
+  runResource,
   runSpanAttributes,
   SPAN_KIND_INTERNAL,
   dependencyLink,
@@ -65,6 +69,8 @@ export interface OtelSinkConfig {
   gzip?: readonly OtelSignal[]
   /** Per signal: a collector's CA, and a client certificate for mutual TLS. */
   tls?: Partial<Record<OtelSignal, OtlpTls>>
+  /** Send each task's span, metrics and log as it ends, not all at the run's end. */
+  live?: boolean
   /** False under `OTEL_TRACES_EXPORTER=none`; absent is true. */
   tracesEnabled?: boolean
   metricsEnabled: boolean
@@ -250,7 +256,7 @@ function partialSuccess(body: string): string | undefined {
 }
 
 function genId(bytes: number): string {
-  return randomBytes(bytes).toString('hex')
+  return Buffer.from(crypto.getRandomValues(new Uint8Array(bytes))).toString('hex')
 }
 
 /** ms → unix-nano decimal string (OTLP int64-as-string). */
@@ -280,6 +286,15 @@ export class OtelSink implements TelemetrySink {
   private readonly taskSpanId = new Map<string, string>()
   private readonly taskStartNano = new Map<string, string>()
   private readonly taskDeps = new Map<string, readonly string[]>()
+  private readonly taskCommand = new Map<string, string>()
+  private readonly taskEndNano = new Map<string, string>()
+  /** Resource attributes: the run's, under `OTEL_RESOURCE_ATTRIBUTES`. */
+  private resource: Readonly<Record<string, string>> = {}
+  // Live mode: what ended since the last send; one send in flight at a time.
+  private readonly liveLogs: TaskLogEntry[] = []
+  private readonly liveEvents: LiveEvent[] = []
+  private pumping: Promise<void> | undefined
+  private readonly warnedLive = new Set<OtelSignal>()
   private summary: RunSummaryRecord | undefined
   private uploaded = false
   // Core's own bounded capture buffer, so which task's output survives a
@@ -302,6 +317,7 @@ export class OtelSink implements TelemetrySink {
       tls: config.tls ?? {},
       headers: config.headers,
       signalHeaders: config.signalHeaders ?? {},
+      live: config.live === true,
       tracesEnabled: config.tracesEnabled !== false,
       metricsEnabled: config.metricsEnabled,
       logsEnabled: config.logsEnabled,
@@ -331,17 +347,34 @@ export class OtelSink implements TelemetrySink {
         // it is what the summary reports and what a receiver stores.
         this.rootStartNano = nanos(record.startedAt)
         this.runStartedAt = record.startedAt
+        this.resource = { ...runResource(record.run), ...this.cfg.resource }
+        this.liveEvent({
+          name: 'vx.run.start',
+          timeUnixNano: this.rootStartNano,
+          body: `${record.run.command}: ${record.total} tasks`,
+        })
         return
-      case 'task.start':
-        this.taskSpanId.set(record.taskId, genId(8))
+      case 'task.start': {
+        const spanId = genId(8)
+        this.taskSpanId.set(record.taskId, spanId)
         this.taskStartNano.set(record.taskId, nanos(record.ts))
+        if (record.command !== undefined) this.taskCommand.set(record.taskId, record.command)
         if (record.dependsOn !== undefined) this.taskDeps.set(record.taskId, record.dependsOn)
+        this.liveEvent({
+          name: 'vx.task.start',
+          timeUnixNano: nanos(record.ts),
+          body:
+            record.command === undefined ? record.taskId : `${record.taskId}: ${record.command}`,
+          taskId: record.taskId,
+          spanId,
+        })
         this.taskNames.set(record.taskId, {
           project: record.project,
           task: record.task,
           startMs: record.ts,
         })
         return
+      }
       case 'task.sample': {
         // Usage is the CPU the tree spent since the last look, over the
         // wall time between: the first look counts from the task's start.
@@ -360,7 +393,9 @@ export class OtelSink implements TelemetrySink {
           timeUnixNano: nanos(record.ts),
           cpuUsage,
           rssBytes: record.rssBytes,
+          ...this.spanOf(record.taskId),
         })
+        this.kick()
         return
       }
       case 'task.log':
@@ -394,6 +429,9 @@ export class OtelSink implements TelemetrySink {
             runId: this.runId,
             workspaceId: this.run?.workspaceId ?? '',
             startedAt: this.runStartedAt,
+            ...(this.taskCommand.has(record.taskId)
+              ? { command: this.taskCommand.get(record.taskId)! }
+              : {}),
           }),
           status: { code: taskStatusCode(t) },
           ...(events.length > 0 ? { events } : {}),
@@ -406,14 +444,21 @@ export class OtelSink implements TelemetrySink {
             task: t,
             startUnixNano: startNano,
             endUnixNano: nanos(record.ts),
+            ...this.spanOf(record.taskId),
           })
         }
         this.lastSample.delete(record.taskId)
+        this.taskEndNano.set(record.taskId, nanos(record.ts))
         // Decides retention: a cache hit's bytes belong to the run that
         // executed them, so only an executed success/failure keeps a tail.
         if (this.cfg.logsEnabled) {
           this.logs.finish(record.taskId, record.status, record.cacheSource, record.hash)
+          if (this.cfg.live) {
+            const entry = this.logs.takeEntry(record.taskId)
+            if (entry !== undefined) this.liveLogs.push(entry)
+          }
         }
+        this.kick()
         return
       }
       case 'run.end':
@@ -425,6 +470,79 @@ export class OtelSink implements TelemetrySink {
   /** Core's flush deadline, passed to every POST. */
   private deadline: AbortSignal | undefined
 
+  /** The task's span, for its metric points' exemplars; none with traces off. */
+  private spanOf(taskId: string): { span?: { traceId: string; spanId: string } } {
+    const spanId = this.taskSpanId.get(taskId)
+    return this.cfg.tracesEnabled && this.traceId && spanId !== undefined
+      ? { span: { traceId: this.traceId, spanId } }
+      : {}
+  }
+
+  private liveEvent(event: LiveEvent): void {
+    if (this.cfg.live && this.cfg.logsEnabled) this.liveEvents.push(event)
+  }
+
+  /** Live mode: send what is waiting, unless a send is already on its way (it takes it next). */
+  private kick(): void {
+    if (!this.cfg.live || this.pumping !== undefined || this.uploaded) return
+    this.pumping = Promise.resolve()
+      .then(() => this.pump())
+      .finally(() => {
+        this.pumping = undefined
+      })
+  }
+
+  private async pump(): Promise<void> {
+    while (
+      this.spans.length + this.taskPoints.length + this.liveLogs.length + this.liveEvents.length >
+      0
+    ) {
+      const spans = this.spans.splice(0)
+      const points = this.taskPoints.splice(0)
+      const logs = this.liveLogs.splice(0)
+      const events = this.liveEvents.splice(0)
+      await Promise.all([
+        this.shipSpans(spans, true),
+        this.shipPoints(points),
+        this.shipLiveLogs(logs, events),
+      ])
+    }
+  }
+
+  private async shipPoints(points: TaskMetricPoint[]): Promise<void> {
+    if (!this.cfg.metricsEnabled || points.length === 0) return
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const bodies = requestBodies(points, (group) =>
+      JSON.stringify(
+        buildTaskMetricsRequest(this.cfg.serviceName, vxVersion, group, this.resource),
+      ),
+    )
+    await this.send('metrics', this.cfg.metricsUrl, bodies, true)
+  }
+
+  private async shipLiveLogs(logs: TaskLogEntry[], events: LiveEvent[]): Promise<void> {
+    if (!this.cfg.logsEnabled) return
+    const bodies = [
+      ...(logs.length > 0 ? this.logBodies(logs) : []),
+      ...(events.length > 0
+        ? requestBodies(events, (group) =>
+            JSON.stringify(
+              buildEventLogsRequest({
+                serviceName: this.cfg.serviceName,
+                vxVersion: this.run?.vxVersion ?? '0.0.0',
+                runId: this.runId,
+                workspaceId: this.run?.workspaceId ?? '',
+                events: group,
+                resource: this.resource,
+                ...(this.cfg.tracesEnabled && this.traceId ? { traceId: this.traceId } : {}),
+              }),
+            ),
+          )
+        : []),
+    ]
+    if (bodies.length > 0) await this.send('logs', this.cfg.logsUrl, bodies, true)
+  }
+
   onRunSummary(summary: RunSummaryRecord): void {
     this.summary = summary
   }
@@ -433,6 +551,11 @@ export class OtelSink implements TelemetrySink {
     if (this.uploaded) return
     this.uploaded = true
     this.deadline = signal
+    // Live: what is on its way lands before the rest, which the run's end sends.
+    if (this.cfg.live) {
+      await this.pumping
+      await this.pump()
+    }
     // Finalize the root span now that the run is over (run.end set the end).
     if (this.run !== undefined && this.traceId) {
       // Prefer the summary's own start/end: they are the run's canonical
@@ -460,16 +583,20 @@ export class OtelSink implements TelemetrySink {
       )
       this.spans.splice(1, 0, ...stages)
     }
-    const vxVersion = this.run?.vxVersion ?? '0.0.0'
-    await Promise.all([this.shipTraces(vxVersion), this.shipMetrics(), this.shipLogs(vxVersion)])
+    await Promise.all([this.shipTraces(), this.shipMetrics(), this.shipLogs()])
+  }
+
+  private async shipTraces(): Promise<void> {
+    await this.shipSpans(this.spans, false)
   }
 
-  private async shipTraces(vxVersion: string): Promise<void> {
-    if (this.cfg.tracesEnabled === false || this.spans.length === 0) return
-    const bodies = requestBodies(this.spans, (spans) =>
-      JSON.stringify(buildTraceRequest(this.cfg.serviceName, vxVersion, spans, this.cfg.resource)),
+  private async shipSpans(spans: OtlpSpan[], live: boolean): Promise<void> {
+    if (this.cfg.tracesEnabled === false || spans.length === 0) return
+    const vxVersion = this.run?.vxVersion ?? '0.0.0'
+    const bodies = requestBodies(spans, (group) =>
+      JSON.stringify(buildTraceRequest(this.cfg.serviceName, vxVersion, group, this.resource)),
     )
-    await this.send('traces', this.cfg.tracesUrl, bodies)
+    await this.send('traces', this.cfg.tracesUrl, bodies, live)
   }
 
   private async shipMetrics(): Promise<void> {
@@ -485,12 +612,7 @@ export class OtelSink implements TelemetrySink {
             points[0] === first
               ? this.runMetricsBody(summary, points)
               : JSON.stringify(
-                  buildTaskMetricsRequest(
-                    this.cfg.serviceName,
-                    vxVersion,
-                    points,
-                    this.cfg.resource,
-                  ),
+                  buildTaskMetricsRequest(this.cfg.serviceName, vxVersion, points, this.resource),
                 ),
           )
     await this.send('metrics', this.cfg.metricsUrl, bodies)
@@ -503,27 +625,32 @@ export class OtelSink implements TelemetrySink {
         summary,
         nanos(summary.endedAt),
         nanos(summary.startedAt),
-        this.cfg.resource,
+        this.resource,
         points,
       ),
     )
   }
 
-  private async shipLogs(vxVersion: string): Promise<void> {
+  private async shipLogs(): Promise<void> {
     if (!this.cfg.logsEnabled) return
-    const workspaceId = this.run?.workspaceId ?? ''
-    const bundle = this.logs.drain(this.runId, workspaceId)
+    const bundle = this.logs.drain(this.runId, this.run?.workspaceId ?? '')
     if (bundle.tasks.length === 0) return
-    const bodies = requestBodies(bundle.tasks, (entries) =>
+    await this.send('logs', this.cfg.logsUrl, this.logBodies(bundle.tasks))
+  }
+
+  private logBodies(tasks: readonly TaskLogEntry[]): string[] {
+    const now = nanos(this.summary?.endedAt ?? Date.now())
+    return requestBodies(tasks, (entries) =>
       JSON.stringify(
         buildLogsRequest({
           serviceName: this.cfg.serviceName,
-          resource: this.cfg.resource,
-          vxVersion,
+          resource: this.resource,
+          vxVersion: this.run?.vxVersion ?? '0.0.0',
           runId: this.runId,
-          workspaceId,
+          workspaceId: this.run?.workspaceId ?? '',
           entries,
-          timeUnixNano: nanos(this.summary?.endedAt ?? Date.now()),
+          timeUnixNano: now,
+          timeFor: (taskId) => this.taskEndNano.get(taskId),
           // Traces off: the trace and its spans are never exported, and a
           // record naming them links nowhere (F-49).
           ...(this.cfg.tracesEnabled && this.traceId
@@ -532,7 +659,6 @@ export class OtelSink implements TelemetrySink {
         }),
       ),
     )
-    await this.send('logs', this.cfg.logsUrl, bodies)
   }
 
   /** The transport, under the signal's own timeout when it has one (F-49). */
@@ -541,8 +667,16 @@ export class OtelSink implements TelemetrySink {
     return own === undefined || this.injected ? this.cfg.post : defaultPost(own)
   }
 
-  /** POSTs every body at once; one warning per signal, however many fail. */
-  private async send(signal: OtelSignal, url: string, bodies: readonly string[]): Promise<void> {
+  /**
+   * POSTs every body at once; one warning per signal, however many fail, and
+   * live sends (`once`) one per signal for the whole run.
+   */
+  private async send(
+    signal: OtelSignal,
+    url: string,
+    bodies: readonly string[],
+    once = false,
+  ): Promise<void> {
     const gzip = this.cfg.gzip.includes(signal)
     const headers: Record<string, string> = {
       'content-type': 'application/json',
@@ -567,6 +701,10 @@ export class OtelSink implements TelemetrySink {
     ).filter((m) => m !== undefined)
     // export is fully optional — a down collector never affects a run
     if (failed.length === 0) return
+    if (once) {
+      if (this.warnedLive.has(signal)) return
+      this.warnedLive.add(signal)
+    }
     // Name the URL: three signals ship concurrently and each is caught
     // here on its own, so a bare "export failed" cannot tell a down
     // collector from one misconfigured signal endpoint.