@ciphyrshq/sdk 2.6.0 → 3.0.0

package/src/tracer.js

@@ -1,4 +1,15 @@
-import { randomUUID } from 'node:crypto';
+import { randomBytes } from 'node:crypto';
+import { availableParallelism } from 'node:os';
+import { spanStorage, traceStorage } from './context.js';
+import { extract, registerInternalOrigin, setDefaultProject, autoInstrument } from './propagation.js';
+import { remoteParent } from './context.js';
+
+// W3C-shaped ids (32 / 16 hex). `traceparent` accepts nothing else, and
+// generating them in that shape is what lets the header carry the REAL ids
+// instead of a hash of them — so an OpenTelemetry-instrumented peer links to
+// this exact trace. (randomUUID's dashes made it unusable as a W3C id.)
+const newTraceId = () => randomBytes(16).toString('hex');
+const newSpanId  = () => randomBytes(8).toString('hex');
 
 // ─────────────────────────────────────────────────────────────────────────────
 // CiphyrsTracer — buffered span/metric collection with auto-flush
@@ -12,6 +23,14 @@ export class CiphyrsTracer {
   #spanBuffer = [];
   #metricBuffer = [];
   #flushInterval;
+  #heartbeatInterval;
+  #knownAgents = new Set();
+  #spansSinceBeat = 0;
+  #errorsSinceBeat = 0;
+  #startedAt = Date.now();
+  #lastCpu = null;
+  #lastCpuAt = null;
+  #lastLag = null;
 
   /**
    * @param {import('./client.js').CiphyrsClient} client
@@ -19,29 +38,160 @@ export class CiphyrsTracer {
    * @param {string}  opts.projectName      — Project name for grouping traces
    * @param {number}  [opts.flushIntervalMs=5000] — Auto-flush interval in ms
    * @param {number}  [opts.batchSize=50]   — Flush when buffer reaches this size
+   * @param {string}  [opts.agentName]      — The agent this process runs. Optional
+   *   (agents are also discovered from your spans), but naming it here makes the
+   *   agent and its heartbeat visible in the fleet before it handles a request.
+   * @param {number}  [opts.heartbeatIntervalMs=60000] — Liveness beat for every
+   *   agent this tracer has seen. 0 disables. The interval is sent with each
+   *   beat so the server sizes THIS agent's down window to it rather than
+   *   applying one global threshold to the whole fleet.
+   * @param {boolean} [opts.heartbeatMetrics=true] — Ship process metrics (RSS,
+   *   event-loop lag, error rate) with each beat, so the fleet can show
+   *   `degraded` before `down`.
+   * @param {boolean} [opts.propagate=true] — Inject W3C traceparent/baggage into
+   *   outbound fetch/http calls made inside a span, so agents in other processes
+   *   join the same trace and the topology edge forms. See propagation.js.
    */
-  constructor(client, { projectName, flushIntervalMs = 5000, batchSize = 50 } = {}) {
+  constructor(client, {
+    projectName, flushIntervalMs = 5000, batchSize = 50,
+    agentName = null, heartbeatIntervalMs = 60_000, heartbeatMetrics = true,
+    propagate = true,
+  } = {}) {
     this.#client = client;
     this.#projectName = projectName;
-    this.#config = { flushIntervalMs, batchSize };
+    this.#config = { flushIntervalMs, batchSize, agentName, heartbeatIntervalMs, heartbeatMetrics };
+    if (agentName) this.#knownAgents.add(agentName);
     this.#flushInterval = setInterval(() => this.flush().catch(() => {}), flushIntervalMs);
     // Prevent the interval from keeping the process alive
     if (this.#flushInterval.unref) this.#flushInterval.unref();
+
+    if (propagate) {
+      setDefaultProject(projectName);
+      if (client?._baseUrl) registerInternalOrigin(client._baseUrl);
+      autoInstrument();
+    }
+    if (heartbeatIntervalMs > 0 && typeof this.#client?.reportHeartbeat === 'function') {
+      this.#startHeartbeat(heartbeatIntervalMs);
+    }
+  }
+
+  /** Agents this tracer has seen (configured, or observed in a span). */
+  get knownAgents() { return [...this.#knownAgents]; }
+
+  #startHeartbeat(intervalMs) {
+    // Heartbeats did not exist in this SDK at all, so a Node agent could never
+    // report liveness: the fleet had to infer health from span traffic, which
+    // makes a quiet agent indistinguishable from a dead one. On by default, and
+    // the first beat is immediate so a process shows up at start-up rather than
+    // one interval later.
+    const beat = () => {
+      for (const agentName of this.#knownAgents) {
+        this.#client.reportHeartbeat({
+          agentName,
+          status: 'up',
+          projectName: this.#projectName,
+          heartbeatIntervalS: Math.round(intervalMs / 1000),
+          metrics: this.#config.heartbeatMetrics ? this.collectMetrics() : undefined,
+        }).catch(() => {});   // a beat must never take down the agent
+      }
+    };
+    setTimeout(beat, 0).unref?.();
+    this.#heartbeatInterval = setInterval(beat, intervalMs);
+    if (this.#heartbeatInterval.unref) this.#heartbeatInterval.unref();
   }
 
   /**
-   * Start a new trace.
+   * Process metrics to ship with a heartbeat. The server turns a
+   * struggling-but-alive process into `degraded`, so the fleet shows trouble
+   * BEFORE the agent stops answering. Only keys the server whitelists are sent.
+   */
+  collectMetrics() {
+    const m = {
+      uptime_s: Math.round((Date.now() - this.#startedAt) / 100) / 10,
+      runtime: `node ${process.versions?.node || '?'}`,
+    };
+    const spans = this.#spansSinceBeat;
+    const errors = this.#errorsSinceBeat;
+    this.#spansSinceBeat = 0;
+    this.#errorsSinceBeat = 0;
+    m.spans_since_last = spans;
+    m.errors_since_last = errors;
+    if (spans > 0) m.error_rate = Math.round((errors / spans) * 1000) / 1000;
+    try {
+      m.rss_mb = Math.round((process.memoryUsage().rss / (1024 * 1024)) * 10) / 10;
+      m.heap_mb = Math.round((process.memoryUsage().heapUsed / (1024 * 1024)) * 10) / 10;
+    } catch { /* not Node — skip */ }
+    try {
+      // Event-loop lag: how late a zero-delay timer actually ran. This is the
+      // signal that separates "busy" from "wedged" in a Node agent.
+      const lag = this.#lastLag;
+      if (typeof lag === 'number') m.event_loop_lag_ms = Math.round(lag * 100) / 100;
+      const t0 = process.hrtime.bigint();
+      setTimeout(() => {
+        this.#lastLag = Number(process.hrtime.bigint() - t0) / 1e6;
+      }, 0).unref?.();
+    } catch { /* skip */ }
+    try {
+      // CPU is a RATE, so it needs a baseline and a window wide enough to
+      // divide by. The first beat fires immediately at start-up, where the
+      // window is ~0ms and cumulative CPU is not: dividing them reported
+      // 1371% and, because a high cpu_pct is one of the server's degraded
+      // signals, marked a perfectly healthy agent degraded the moment it
+      // booted. So: no baseline or no real window means no number.
+      const now = Date.now();
+      const prev = this.#lastCpu;
+      const prevAt = this.#lastCpuAt;
+      this.#lastCpu = process.cpuUsage();
+      this.#lastCpuAt = now;
+      if (prev && prevAt && now - prevAt >= 500) {
+        const usage = process.cpuUsage(prev);
+        // Normalised by core count, so 100 means "this box is saturated"
+        // rather than "one thread is busy" — a 16-core host legitimately
+        // reports 800% per-core for work that is using half the machine.
+        const cores = Math.max(1, availableParallelism());
+        const windowUs = (now - prevAt) * 1000 * cores;
+        const pct = ((usage.user + usage.system) / windowUs) * 100;
+        m.cpu_pct = Math.min(100, Math.max(0, Math.round(pct * 10) / 10));
+      }
+    } catch { /* skip */ }
+    return m;
+  }
+
+  /**
+   * Start a new trace — or CONTINUE the caller's, when this process is serving
+   * a request that carried trace context (expressMiddleware, fastifyPlugin, or
+   * an explicit `withRemoteContext(headers, fn)`). Without continuation every
+   * service began its own trace and the graph showed disconnected agents even
+   * with headers flowing correctly.
+   *
    * @param {string} name — Human-readable trace name
    * @param {object} [opts]
    * @param {string} [opts.traceId] — Provide your own trace ID, or one is generated
+   * @param {object} [opts.headers] — Inbound request headers, to continue their trace
    * @returns {Trace}
    */
   trace(name, opts = {}) {
     return new Trace(this, name, opts);
   }
 
+  /**
+   * Run `fn` inside a trace, with the trace active for everything it awaits.
+   * The scoped form — prefer it when you can:
+   *
+   *   await tracer.withTrace('handle order', async (t) => {
+   *     await t.span('RouterAgent').run(async () => { … })
+   *   })
+   */
+  async withTrace(name, fn, opts = {}) {
+    const t = this.trace(name, opts);
+    return traceStorage.run({ trace_id: t.traceId, name }, () => fn(t));
+  }
+
   /** @internal — called by Span.end() to enqueue completed spans */
   _enqueueSpan(span) {
+    if (span?.agent_name) this.#knownAgents.add(span.agent_name);
+    this.#spansSinceBeat++;
+    if (span?.status === 'error') this.#errorsSinceBeat++;
     this.#spanBuffer.push(span);
     if (this.#spanBuffer.length >= this.#config.batchSize) {
       this.flush().catch(() => {});
@@ -85,7 +235,8 @@ export class CiphyrsTracer {
             { name: this.#projectName },
             { trace_id: traceId, name: traceSpans[0]?.trace_name || traceId },
             traceSpans,
-          ).catch(err => console.error('Ciphyrs flush error:', err.message)),
+          ).catch(err => this.#requeue(this.#spanBuffer, traceSpans,
+            `trace ${traceId}`, err)),
         );
       }
     }
@@ -95,19 +246,53 @@ export class CiphyrsTracer {
         this.#client._request(`${this.#client._baseUrl}/v1/observe/metrics`, {
           method: 'POST',
           body: { metrics },
-        }).catch(err => console.error('Ciphyrs metrics flush error:', err.message)),
+        }).catch(err => this.#requeue(this.#metricBuffer, metrics, 'metrics', err)),
       );
     }
 
     await Promise.allSettled(promises);
   }
 
+  /**
+   * Put a failed batch back rather than dropping it.
+   *
+   * The buffers are spliced empty BEFORE the send, so the previous `.catch`
+   * that only logged meant every failure the client's retries could not
+   * absorb destroyed that batch for good. The customer saw one line of
+   * console noise and a permanent hole in their traces.
+   *
+   * sdk-node's client.js already did it this way; this brings the published
+   * SDK in line. APPEND rather than prepend: spans are posted one trace at a
+   * time, so with traces A and B both failing, prepending would leave the
+   * buffer holding B before A and hand a recovered collector a reversed
+   * timeline.
+   */
+  #requeue(buffer, items, what, err) {
+    buffer.push(...items);
+    // Bounded, or an unreachable collector grows this without limit inside
+    // the customer's process. Oldest go first: during an incident the recent
+    // spans are the ones worth keeping.
+    const cap = Math.max(this.#config.batchSize * 20, 1000);
+    let dropped = 0;
+    if (buffer.length > cap) {
+      dropped = buffer.length - cap;
+      buffer.splice(0, dropped);
+    }
+    console.error(
+      `[ciphyrs] ${what}: flush failed (${err.message}); `
+      + (dropped
+        ? `buffer full, dropped ${dropped} oldest item(s)`
+        : `${items.length} item(s) re-queued for the next flush`),
+    );
+  }
+
   /**
    * Stop the auto-flush timer and flush any remaining data.
    * Always call this before process exit to avoid data loss.
    */
   async shutdown() {
     clearInterval(this.#flushInterval);
+    if (this.#heartbeatInterval) clearInterval(this.#heartbeatInterval);
     await this.flush();
   }
 }
@@ -121,16 +306,41 @@ export class Trace {
   #traceId;
   #name;
   #startTime;
+  #remoteParentSpanId;
+  #remoteAgent;
+  #ownSpanIds = new Set();
 
   constructor(tracer, name, opts = {}) {
     this.#tracer = tracer;
-    this.#traceId = opts.traceId || randomUUID();
+    // Continue the caller's trace when one is active for this async context
+    // (or was handed in as `opts.headers`), unless an explicit traceId wins.
+    const remote = opts.headers ? extract(opts.headers) : remoteParent();
+    this.#traceId = opts.traceId || remote?.trace_id || newTraceId();
+    this.#remoteParentSpanId = (!opts.traceId && remote?.span_id) || null;
+    this.#remoteAgent = remote?.agent_name || null;
     this.#name = name;
     this.#startTime = Date.now();
   }
 
   get traceId() { return this.#traceId; }
   get name()    { return this.#name; }
+  /** True when this trace continues one started by another agent/process. */
+  get isContinuation() { return this.#remoteParentSpanId !== null; }
+  /** The agent that called us, when the inbound request named one. */
+  get callerAgent() { return this.#remoteAgent; }
+  /** @internal — Span.end() reports back so sibling spans can nest correctly. */
+  _ownSpan(id) { this.#ownSpanIds.add(id); }
+  /** @internal — the parent a new span should use when none was given. */
+  _defaultParent() {
+    const ambient = spanStorage.getStore();
+    if (ambient?.span_id && this.#ownSpanIds.has(ambient.span_id)) return ambient.span_id;
+    return this.#remoteParentSpanId;
+  }
+
+  /** Run `fn` with this trace active for everything it awaits. */
+  run(fn) {
+    return traceStorage.run({ trace_id: this.#traceId, name: this.#name }, () => fn(this));
+  }
 
   /**
    * Create a new span within this trace.
@@ -142,10 +352,20 @@ export class Trace {
    * @returns {Span}
    */
   span(name, opts = {}) {
-    return new Span(this.#tracer, this.#traceId, this.#name, name, opts);
+    const parentSpanId = opts.parentSpanId !== undefined ? opts.parentSpanId : this._defaultParent();
+    const s = new Span(this.#tracer, this.#traceId, this.#name, name, { ...opts, parentSpanId });
+    this._ownSpan(s.spanId);
+    // Auto-enter so the SDK's long-standing style nests without a callback:
+    //   const outer = t.span('Router'); const inner = t.span('Billing');
+    // `inner` is a child of `outer`, and an HTTP call made between them
+    // carries `outer`'s (then `inner`'s) trace headers. end() unwinds.
+    // Pass `{ scoped: true }` to opt out and manage context with run().
+    if (opts.scoped !== true) s.enter();
+    return s;
   }
 }
 
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Span — a single timed operation within a trace
 // ─────────────────────────────────────────────────────────────────────────────
@@ -154,12 +374,15 @@ export class Span {
   #tracer;
   #data;
   #startTime;
+  #ended = false;
+  #entered = false;
+  #previousStore = undefined;
 
   constructor(tracer, traceId, traceName, name, opts = {}) {
     this.#tracer = tracer;
     this.#startTime = Date.now();
     this.#data = {
-      span_id: randomUUID(),
+      span_id: newSpanId(),
       trace_id: traceId,
       trace_name: traceName,
       parent_span_id: opts.parentSpanId || null,
@@ -177,6 +400,49 @@ export class Span {
   }
 
   get spanId() { return this.#data.span_id; }
+  get agentName() { return this.#data.agent_name; }
+
+  /**
+   * Run `fn` with this span active, then end it — the scoped form, and the one
+   * to prefer: everything `fn` awaits sees this span as its parent, and an
+   * outbound HTTP call made inside it carries this span's trace headers.
+   * The span is ended (and marked errored) even if `fn` throws.
+   */
+  async run(fn) {
+    // `kind` rides in the ambient store so a caller inside this span can tell
+    // WHAT span it is in, not merely that one is open. protectTool needs that
+    // distinction: the gateway matches a tool span to its gate call by span
+    // id, so sending the id of an enclosing AGENT span can never match while
+    // still telling the gateway this producer is correlated.
+    const store = { trace_id: this.#data.trace_id, span_id: this.#data.span_id, agent_name: this.#data.agent_name, kind: this.#data.kind };
+    try {
+      return await spanStorage.run(store, () => fn(this));
+    } catch (err) {
+      this.setError(err?.message || String(err));
+      throw err;
+    } finally {
+      if (!this.#ended) this.end();
+    }
+  }
+
+  /**
+   * Make this span the ambient one for the rest of the current async context,
+   * without a callback. This is what keeps the SDK's long-standing
+   * `const s = trace.span(); … s.end()` style nesting correctly — `end()`
+   * restores whatever was ambient before.
+   */
+  enter() {
+    if (this.#entered) return this;
+    this.#entered = true;
+    this.#previousStore = spanStorage.getStore();
+    spanStorage.enterWith({
+      trace_id: this.#data.trace_id, span_id: this.#data.span_id, agent_name: this.#data.agent_name,
+      // Same store shape as run() above — the unscoped style must not be the
+      // one that leaves protectTool unable to tell a tool span from an agent span.
+      kind: this.#data.kind,
+    });
+    return this;
+  }
 
   /** Set the input text for this span. */
   setInput(text)  { this.#data.input_text = text; return this; }
@@ -211,9 +477,12 @@ export class Span {
    * @returns {this}
    */
   end() {
+    if (this.#ended) return this;      // double end() must not double-count
+    this.#ended = true;
     this.#data.started_at = new Date(this.#startTime).toISOString();
     this.#data.ended_at = new Date().toISOString();
     this.#data.duration_ms = Date.now() - this.#startTime;
+    if (this.#entered) spanStorage.enterWith(this.#previousStore);
     this.#tracer._enqueueSpan(this.#data);
     return this;
   }

package/src/tracer.test.js

@@ -0,0 +1,194 @@
+/**
+ * CiphyrsTracer — buffered span and metric ingestion.
+ * Run with: npm test
+ *
+ * This is the only route spans, traces and metrics take to the platform, and
+ * its failure mode is silence: when ingestion breaks nothing errors, data just
+ * never arrives, and an empty dashboard is indistinguishable from a quiet day.
+ *
+ * The buffers are spliced empty BEFORE the send, so a `.catch` that only
+ * logged meant any failure the client's retries could not absorb destroyed
+ * that batch permanently. sdk-node's client.js already re-queued; this package
+ * did not, so the same customer got different durability from two Ciphyrs
+ * SDKs doing the same job.
+ */
+
+// No test in this package may open a socket — see no-network.test-helper.js.
+// Imported per file because node --test runs each file in its own process.
+import './no-network.test-helper.js'
+import { describe, it, beforeEach, afterEach } from 'node:test'
+import assert from 'node:assert/strict'
+import { CiphyrsTracer } from './tracer.js'
+
+// console.error is the delivery mechanism for "your telemetry is not arriving",
+// so it is asserted on rather than silenced.
+let logged
+let realError
+beforeEach(() => { logged = []; realError = console.error; console.error = (m) => logged.push(String(m)) })
+afterEach(() => { console.error = realError })
+
+function fakeClient({ failIngest = false, failMetrics = false } = {}) {
+  const calls = { ingest: [], metrics: [] }
+  return {
+    calls,
+    _baseUrl: 'https://api',
+    trace: {
+      ingest: async (project, trace, spans) => {
+        calls.ingest.push({ trace, spans })
+        if (failIngest) throw new Error('collector unreachable')
+        return { ok: true }
+      },
+    },
+    _request: async (url, opts) => {
+      calls.metrics.push(opts.body)
+      if (failMetrics) throw new Error('collector unreachable')
+      return { ok: true }
+    },
+  }
+}
+
+const tracerFor = (client, opts = {}) =>
+  // flushIntervalMs is large so the timer never races an assertion; every
+  // flush below is explicit.
+  new CiphyrsTracer(client, { projectName: 'billing', flushIntervalMs: 3_600_000, ...opts })
+
+// Reaches the private #spanBuffer the only way a test legitimately can:
+// enqueue, flush, and observe what a subsequent flush still tries to send.
+const pendingSpanCount = async (tracer, client) => {
+  const before = client.calls.ingest.length
+  await tracer.flush()
+  return client.calls.ingest.slice(before).reduce((n, c) => n + c.spans.length, 0)
+}
+
+describe('CiphyrsTracer — a successful flush', () => {
+  it('sends buffered spans grouped into one call per trace', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    t._enqueueSpan({ trace_id: 'A', name: 'a2' })
+    t._enqueueSpan({ trace_id: 'B', name: 'b1' })
+    await t.flush()
+    assert.equal(client.calls.ingest.length, 2, 'traces must not be merged')
+    assert.deepEqual(client.calls.ingest.map(c => c.spans.length), [2, 1])
+  })
+
+  it('sends buffered metrics', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    t.emitMetric('token_cost', 0.42, { unit: 'usd' })
+    await t.flush()
+    assert.equal(client.calls.metrics[0].metrics[0].name, 'token_cost')
+  })
+
+  it('empties the buffers, so nothing is sent twice', async () => {
+    const client = fakeClient()
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    t.emitMetric('m', 1)
+    await t.flush()
+    await t.flush()
+    assert.equal(client.calls.ingest.length, 1)
+    assert.equal(client.calls.metrics.length, 1)
+  })
+
+  it('does nothing when there is nothing buffered', async () => {
+    const client = fakeClient()
+    await tracerFor(client).flush()
+    assert.equal(client.calls.ingest.length + client.calls.metrics.length, 0)
+  })
+})
+
+describe('CiphyrsTracer — a failed flush must not destroy the batch', () => {
+  it('re-queues spans and says so', async () => {
+    const client = fakeClient({ failIngest: true })
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    await t.flush()
+    assert.match(logged.join('\n'), /re-queued/)
+    assert.equal(await pendingSpanCount(t, client), 1, 'the span was destroyed')
+  })
+
+  it('re-queues metrics and says so', async () => {
+    const client = fakeClient({ failMetrics: true })
+    const t = tracerFor(client)
+    t.emitMetric('token_cost', 0.42)
+    await t.flush()
+    const before = client.calls.metrics.length
+    await t.flush()
+    assert.ok(client.calls.metrics.length > before, 'the metric was destroyed')
+    assert.equal(client.calls.metrics.at(-1).metrics[0].name, 'token_cost')
+  })
+
+  it('delivers the batch once the collector comes back', async () => {
+    // Re-queueing is only worth anything if a later attempt picks it up.
+    const client = fakeClient({ failIngest: true })
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    await t.flush()
+    client.trace.ingest = async (p, trace, spans) => {
+      client.calls.ingest.push({ trace, spans }); return { ok: true }
+    }
+    await t.flush()
+    assert.equal(client.calls.ingest.at(-1).spans[0].name, 'a1')
+    // ...and is then gone, not resent forever.
+    const n = client.calls.ingest.length
+    await t.flush()
+    assert.equal(client.calls.ingest.length, n)
+  })
+
+  it('keeps two failed traces in the order they were produced', async () => {
+    // Spans post one trace at a time, so the second failure lands on top of
+    // the first — the one case where re-queueing at the front would silently
+    // reverse the timeline.
+    const client = fakeClient({ failIngest: true })
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    t._enqueueSpan({ trace_id: 'B', name: 'b1' })
+    await t.flush()
+    const before = client.calls.ingest.length
+    await t.flush()
+    const names = client.calls.ingest.slice(before).flatMap(c => c.spans.map(s => s.name))
+    assert.deepEqual(names, ['a1', 'b1'])
+  })
+
+  it('bounds the re-queue so an outage cannot exhaust memory', async () => {
+    // A tracing SDK that OOMs the process it observes is worse than one that
+    // drops telemetry — but it has to say what it dropped.
+    const client = fakeClient({ failIngest: true })
+    const t = tracerFor(client, { batchSize: 5 })   // cap = max(5*20, 1000) = 1000
+    for (let i = 0; i < 1200; i++) t._enqueueSpan({ trace_id: 'A', name: `s${i}` })
+    await t.flush()
+    assert.match(logged.join('\n'), /dropped \d+ oldest/)
+    assert.ok(await pendingSpanCount(t, client) <= 1000)
+  })
+
+  it('drops the oldest, keeping the most recent spans', async () => {
+    const client = fakeClient({ failIngest: true })
+    const t = tracerFor(client, { batchSize: 5 })
+    for (let i = 0; i < 1200; i++) t._enqueueSpan({ trace_id: 'A', name: `s${i}` })
+    await t.flush()
+    const before = client.calls.ingest.length
+    await t.flush()
+    const names = client.calls.ingest.slice(before).flatMap(c => c.spans.map(s => s.name))
+    assert.equal(names.at(-1), 's1199', 'threw away the newest instead of the oldest')
+    assert.ok(!names.includes('s0'))
+  })
+
+  it('a metrics failure does not take the spans down with it', async () => {
+    const client = fakeClient({ failMetrics: true })
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    t.emitMetric('m', 1)
+    await t.flush()
+    assert.equal(client.calls.ingest.length, 1, 'spans were sent')
+    assert.equal(await pendingSpanCount(t, client), 0, 'spans were needlessly re-queued')
+  })
+
+  it('never rejects — observability must not crash the observed process', async () => {
+    const client = fakeClient({ failIngest: true, failMetrics: true })
+    const t = tracerFor(client)
+    t._enqueueSpan({ trace_id: 'A', name: 'a1' })
+    t.emitMetric('m', 1)
+    await t.flush()   // must resolve
+  })
+})